| <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> |
| <html lang="en"> |
| <head> |
| <title>Source code</title> |
| <link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style"> |
| </head> |
| <body> |
| <div class="sourceContainer"> |
| <pre><span class="sourceLineNo">001</span>/**<a name="line.1"></a> |
| <span class="sourceLineNo">002</span> * Licensed to the Apache Software Foundation (ASF) under one<a name="line.2"></a> |
| <span class="sourceLineNo">003</span> * or more contributor license agreements. See the NOTICE file<a name="line.3"></a> |
| <span class="sourceLineNo">004</span> * distributed with this work for additional information<a name="line.4"></a> |
| <span class="sourceLineNo">005</span> * regarding copyright ownership. The ASF licenses this file<a name="line.5"></a> |
| <span class="sourceLineNo">006</span> * to you under the Apache License, Version 2.0 (the<a name="line.6"></a> |
| <span class="sourceLineNo">007</span> * "License"); you may not use this file except in compliance<a name="line.7"></a> |
| <span class="sourceLineNo">008</span> * with the License. You may obtain a copy of the License at<a name="line.8"></a> |
| <span class="sourceLineNo">009</span> *<a name="line.9"></a> |
| <span class="sourceLineNo">010</span> * http://www.apache.org/licenses/LICENSE-2.0<a name="line.10"></a> |
| <span class="sourceLineNo">011</span> *<a name="line.11"></a> |
| <span class="sourceLineNo">012</span> * Unless required by applicable law or agreed to in writing, software<a name="line.12"></a> |
| <span class="sourceLineNo">013</span> * distributed under the License is distributed on an "AS IS" BASIS,<a name="line.13"></a> |
| <span class="sourceLineNo">014</span> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.<a name="line.14"></a> |
| <span class="sourceLineNo">015</span> * See the License for the specific language governing permissions and<a name="line.15"></a> |
| <span class="sourceLineNo">016</span> * limitations under the License.<a name="line.16"></a> |
| <span class="sourceLineNo">017</span> */<a name="line.17"></a> |
| <span class="sourceLineNo">018</span>package org.apache.hadoop.hbase.regionserver;<a name="line.18"></a> |
| <span class="sourceLineNo">019</span><a name="line.19"></a> |
| <span class="sourceLineNo">020</span>import static org.apache.hadoop.hbase.HConstants.REPLICATION_SCOPE_LOCAL;<a name="line.20"></a> |
| <span class="sourceLineNo">021</span>import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY;<a name="line.21"></a> |
| <span class="sourceLineNo">022</span>import static org.apache.hadoop.hbase.util.ConcurrentMapUtils.computeIfAbsent;<a name="line.22"></a> |
| <span class="sourceLineNo">023</span><a name="line.23"></a> |
| <span class="sourceLineNo">024</span>import edu.umd.cs.findbugs.annotations.Nullable;<a name="line.24"></a> |
| <span class="sourceLineNo">025</span>import java.io.EOFException;<a name="line.25"></a> |
| <span class="sourceLineNo">026</span>import java.io.FileNotFoundException;<a name="line.26"></a> |
| <span class="sourceLineNo">027</span>import java.io.IOException;<a name="line.27"></a> |
| <span class="sourceLineNo">028</span>import java.io.InterruptedIOException;<a name="line.28"></a> |
| <span class="sourceLineNo">029</span>import java.lang.reflect.Constructor;<a name="line.29"></a> |
| <span class="sourceLineNo">030</span>import java.nio.ByteBuffer;<a name="line.30"></a> |
| <span class="sourceLineNo">031</span>import java.nio.charset.StandardCharsets;<a name="line.31"></a> |
| <span class="sourceLineNo">032</span>import java.text.ParseException;<a name="line.32"></a> |
| <span class="sourceLineNo">033</span>import java.util.ArrayList;<a name="line.33"></a> |
| <span class="sourceLineNo">034</span>import java.util.Arrays;<a name="line.34"></a> |
| <span class="sourceLineNo">035</span>import java.util.Collection;<a name="line.35"></a> |
| <span class="sourceLineNo">036</span>import java.util.Collections;<a name="line.36"></a> |
| <span class="sourceLineNo">037</span>import java.util.HashMap;<a name="line.37"></a> |
| <span class="sourceLineNo">038</span>import java.util.HashSet;<a name="line.38"></a> |
| <span class="sourceLineNo">039</span>import java.util.Iterator;<a name="line.39"></a> |
| <span class="sourceLineNo">040</span>import java.util.List;<a name="line.40"></a> |
| <span class="sourceLineNo">041</span>import java.util.Map;<a name="line.41"></a> |
| <span class="sourceLineNo">042</span>import java.util.Map.Entry;<a name="line.42"></a> |
| <span class="sourceLineNo">043</span>import java.util.NavigableMap;<a name="line.43"></a> |
| <span class="sourceLineNo">044</span>import java.util.NavigableSet;<a name="line.44"></a> |
| <span class="sourceLineNo">045</span>import java.util.Objects;<a name="line.45"></a> |
| <span class="sourceLineNo">046</span>import java.util.Optional;<a name="line.46"></a> |
| <span class="sourceLineNo">047</span>import java.util.RandomAccess;<a name="line.47"></a> |
| <span class="sourceLineNo">048</span>import java.util.Set;<a name="line.48"></a> |
| <span class="sourceLineNo">049</span>import java.util.TreeMap;<a name="line.49"></a> |
| <span class="sourceLineNo">050</span>import java.util.UUID;<a name="line.50"></a> |
| <span class="sourceLineNo">051</span>import java.util.concurrent.Callable;<a name="line.51"></a> |
| <span class="sourceLineNo">052</span>import java.util.concurrent.CompletionService;<a name="line.52"></a> |
| <span class="sourceLineNo">053</span>import java.util.concurrent.ConcurrentHashMap;<a name="line.53"></a> |
| <span class="sourceLineNo">054</span>import java.util.concurrent.ConcurrentMap;<a name="line.54"></a> |
| <span class="sourceLineNo">055</span>import java.util.concurrent.ConcurrentSkipListMap;<a name="line.55"></a> |
| <span class="sourceLineNo">056</span>import java.util.concurrent.ExecutionException;<a name="line.56"></a> |
| <span class="sourceLineNo">057</span>import java.util.concurrent.ExecutorCompletionService;<a name="line.57"></a> |
| <span class="sourceLineNo">058</span>import java.util.concurrent.ExecutorService;<a name="line.58"></a> |
| <span class="sourceLineNo">059</span>import java.util.concurrent.Executors;<a name="line.59"></a> |
| <span class="sourceLineNo">060</span>import java.util.concurrent.Future;<a name="line.60"></a> |
| <span class="sourceLineNo">061</span>import java.util.concurrent.FutureTask;<a name="line.61"></a> |
| <span class="sourceLineNo">062</span>import java.util.concurrent.ThreadFactory;<a name="line.62"></a> |
| <span class="sourceLineNo">063</span>import java.util.concurrent.ThreadPoolExecutor;<a name="line.63"></a> |
| <span class="sourceLineNo">064</span>import java.util.concurrent.TimeUnit;<a name="line.64"></a> |
| <span class="sourceLineNo">065</span>import java.util.concurrent.TimeoutException;<a name="line.65"></a> |
| <span class="sourceLineNo">066</span>import java.util.concurrent.atomic.AtomicBoolean;<a name="line.66"></a> |
| <span class="sourceLineNo">067</span>import java.util.concurrent.atomic.AtomicInteger;<a name="line.67"></a> |
| <span class="sourceLineNo">068</span>import java.util.concurrent.atomic.LongAdder;<a name="line.68"></a> |
| <span class="sourceLineNo">069</span>import java.util.concurrent.locks.Lock;<a name="line.69"></a> |
| <span class="sourceLineNo">070</span>import java.util.concurrent.locks.ReadWriteLock;<a name="line.70"></a> |
| <span class="sourceLineNo">071</span>import java.util.concurrent.locks.ReentrantReadWriteLock;<a name="line.71"></a> |
| <span class="sourceLineNo">072</span>import java.util.function.Function;<a name="line.72"></a> |
| <span class="sourceLineNo">073</span>import java.util.stream.Collectors;<a name="line.73"></a> |
| <span class="sourceLineNo">074</span>import java.util.stream.Stream;<a name="line.74"></a> |
| <span class="sourceLineNo">075</span>import org.apache.hadoop.conf.Configuration;<a name="line.75"></a> |
| <span class="sourceLineNo">076</span>import org.apache.hadoop.fs.FileStatus;<a name="line.76"></a> |
| <span class="sourceLineNo">077</span>import org.apache.hadoop.fs.FileSystem;<a name="line.77"></a> |
| <span class="sourceLineNo">078</span>import org.apache.hadoop.fs.LocatedFileStatus;<a name="line.78"></a> |
| <span class="sourceLineNo">079</span>import org.apache.hadoop.fs.Path;<a name="line.79"></a> |
| <span class="sourceLineNo">080</span>import org.apache.hadoop.hbase.Cell;<a name="line.80"></a> |
| <span class="sourceLineNo">081</span>import org.apache.hadoop.hbase.CellBuilderType;<a name="line.81"></a> |
| <span class="sourceLineNo">082</span>import org.apache.hadoop.hbase.CellComparator;<a name="line.82"></a> |
| <span class="sourceLineNo">083</span>import org.apache.hadoop.hbase.CellComparatorImpl;<a name="line.83"></a> |
| <span class="sourceLineNo">084</span>import org.apache.hadoop.hbase.CellScanner;<a name="line.84"></a> |
| <span class="sourceLineNo">085</span>import org.apache.hadoop.hbase.CellUtil;<a name="line.85"></a> |
| <span class="sourceLineNo">086</span>import org.apache.hadoop.hbase.CompareOperator;<a name="line.86"></a> |
| <span class="sourceLineNo">087</span>import org.apache.hadoop.hbase.CompoundConfiguration;<a name="line.87"></a> |
| <span class="sourceLineNo">088</span>import org.apache.hadoop.hbase.DoNotRetryIOException;<a name="line.88"></a> |
| <span class="sourceLineNo">089</span>import org.apache.hadoop.hbase.DroppedSnapshotException;<a name="line.89"></a> |
| <span class="sourceLineNo">090</span>import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;<a name="line.90"></a> |
| <span class="sourceLineNo">091</span>import org.apache.hadoop.hbase.HConstants;<a name="line.91"></a> |
| <span class="sourceLineNo">092</span>import org.apache.hadoop.hbase.HConstants.OperationStatusCode;<a name="line.92"></a> |
| <span class="sourceLineNo">093</span>import org.apache.hadoop.hbase.HDFSBlocksDistribution;<a name="line.93"></a> |
| <span class="sourceLineNo">094</span>import org.apache.hadoop.hbase.KeyValue;<a name="line.94"></a> |
| <span class="sourceLineNo">095</span>import org.apache.hadoop.hbase.MetaCellComparator;<a name="line.95"></a> |
| <span class="sourceLineNo">096</span>import org.apache.hadoop.hbase.NamespaceDescriptor;<a name="line.96"></a> |
| <span class="sourceLineNo">097</span>import org.apache.hadoop.hbase.NotServingRegionException;<a name="line.97"></a> |
| <span class="sourceLineNo">098</span>import org.apache.hadoop.hbase.PrivateCellUtil;<a name="line.98"></a> |
| <span class="sourceLineNo">099</span>import org.apache.hadoop.hbase.RegionTooBusyException;<a name="line.99"></a> |
| <span class="sourceLineNo">100</span>import org.apache.hadoop.hbase.Tag;<a name="line.100"></a> |
| <span class="sourceLineNo">101</span>import org.apache.hadoop.hbase.TagUtil;<a name="line.101"></a> |
| <span class="sourceLineNo">102</span>import org.apache.hadoop.hbase.client.Append;<a name="line.102"></a> |
| <span class="sourceLineNo">103</span>import org.apache.hadoop.hbase.client.CheckAndMutate;<a name="line.103"></a> |
| <span class="sourceLineNo">104</span>import org.apache.hadoop.hbase.client.CheckAndMutateResult;<a name="line.104"></a> |
| <span class="sourceLineNo">105</span>import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;<a name="line.105"></a> |
| <span class="sourceLineNo">106</span>import org.apache.hadoop.hbase.client.CompactionState;<a name="line.106"></a> |
| <span class="sourceLineNo">107</span>import org.apache.hadoop.hbase.client.Delete;<a name="line.107"></a> |
| <span class="sourceLineNo">108</span>import org.apache.hadoop.hbase.client.Durability;<a name="line.108"></a> |
| <span class="sourceLineNo">109</span>import org.apache.hadoop.hbase.client.Get;<a name="line.109"></a> |
| <span class="sourceLineNo">110</span>import org.apache.hadoop.hbase.client.Increment;<a name="line.110"></a> |
| <span class="sourceLineNo">111</span>import org.apache.hadoop.hbase.client.IsolationLevel;<a name="line.111"></a> |
| <span class="sourceLineNo">112</span>import org.apache.hadoop.hbase.client.Mutation;<a name="line.112"></a> |
| <span class="sourceLineNo">113</span>import org.apache.hadoop.hbase.client.Put;<a name="line.113"></a> |
| <span class="sourceLineNo">114</span>import org.apache.hadoop.hbase.client.RegionInfo;<a name="line.114"></a> |
| <span class="sourceLineNo">115</span>import org.apache.hadoop.hbase.client.RegionReplicaUtil;<a name="line.115"></a> |
| <span class="sourceLineNo">116</span>import org.apache.hadoop.hbase.client.Result;<a name="line.116"></a> |
| <span class="sourceLineNo">117</span>import org.apache.hadoop.hbase.client.Row;<a name="line.117"></a> |
| <span class="sourceLineNo">118</span>import org.apache.hadoop.hbase.client.RowMutations;<a name="line.118"></a> |
| <span class="sourceLineNo">119</span>import org.apache.hadoop.hbase.client.Scan;<a name="line.119"></a> |
| <span class="sourceLineNo">120</span>import org.apache.hadoop.hbase.client.TableDescriptor;<a name="line.120"></a> |
| <span class="sourceLineNo">121</span>import org.apache.hadoop.hbase.client.TableDescriptorBuilder;<a name="line.121"></a> |
| <span class="sourceLineNo">122</span>import org.apache.hadoop.hbase.conf.ConfigurationManager;<a name="line.122"></a> |
| <span class="sourceLineNo">123</span>import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver;<a name="line.123"></a> |
| <span class="sourceLineNo">124</span>import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;<a name="line.124"></a> |
| <span class="sourceLineNo">125</span>import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;<a name="line.125"></a> |
| <span class="sourceLineNo">126</span>import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;<a name="line.126"></a> |
| <span class="sourceLineNo">127</span>import org.apache.hadoop.hbase.exceptions.TimeoutIOException;<a name="line.127"></a> |
| <span class="sourceLineNo">128</span>import org.apache.hadoop.hbase.exceptions.UnknownProtocolException;<a name="line.128"></a> |
| <span class="sourceLineNo">129</span>import org.apache.hadoop.hbase.filter.BinaryComparator;<a name="line.129"></a> |
| <span class="sourceLineNo">130</span>import org.apache.hadoop.hbase.filter.ByteArrayComparable;<a name="line.130"></a> |
| <span class="sourceLineNo">131</span>import org.apache.hadoop.hbase.filter.Filter;<a name="line.131"></a> |
| <span class="sourceLineNo">132</span>import org.apache.hadoop.hbase.io.HFileLink;<a name="line.132"></a> |
| <span class="sourceLineNo">133</span>import org.apache.hadoop.hbase.io.HeapSize;<a name="line.133"></a> |
| <span class="sourceLineNo">134</span>import org.apache.hadoop.hbase.io.TimeRange;<a name="line.134"></a> |
| <span class="sourceLineNo">135</span>import org.apache.hadoop.hbase.io.hfile.BlockCache;<a name="line.135"></a> |
| <span class="sourceLineNo">136</span>import org.apache.hadoop.hbase.io.hfile.HFile;<a name="line.136"></a> |
| <span class="sourceLineNo">137</span>import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;<a name="line.137"></a> |
| <span class="sourceLineNo">138</span>import org.apache.hadoop.hbase.ipc.RpcCall;<a name="line.138"></a> |
| <span class="sourceLineNo">139</span>import org.apache.hadoop.hbase.ipc.RpcServer;<a name="line.139"></a> |
| <span class="sourceLineNo">140</span>import org.apache.hadoop.hbase.mob.MobFileCache;<a name="line.140"></a> |
| <span class="sourceLineNo">141</span>import org.apache.hadoop.hbase.monitoring.MonitoredTask;<a name="line.141"></a> |
| <span class="sourceLineNo">142</span>import org.apache.hadoop.hbase.monitoring.TaskMonitor;<a name="line.142"></a> |
| <span class="sourceLineNo">143</span>import org.apache.hadoop.hbase.quotas.RegionServerSpaceQuotaManager;<a name="line.143"></a> |
| <span class="sourceLineNo">144</span>import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.WriteEntry;<a name="line.144"></a> |
| <span class="sourceLineNo">145</span>import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;<a name="line.145"></a> |
| <span class="sourceLineNo">146</span>import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;<a name="line.146"></a> |
| <span class="sourceLineNo">147</span>import org.apache.hadoop.hbase.regionserver.compactions.ForbidMajorCompactionChecker;<a name="line.147"></a> |
| <span class="sourceLineNo">148</span>import org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory;<a name="line.148"></a> |
| <span class="sourceLineNo">149</span>import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;<a name="line.149"></a> |
| <span class="sourceLineNo">150</span>import org.apache.hadoop.hbase.regionserver.throttle.StoreHotnessProtector;<a name="line.150"></a> |
| <span class="sourceLineNo">151</span>import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;<a name="line.151"></a> |
| <span class="sourceLineNo">152</span>import org.apache.hadoop.hbase.regionserver.wal.WALUtil;<a name="line.152"></a> |
| <span class="sourceLineNo">153</span>import org.apache.hadoop.hbase.replication.ReplicationUtils;<a name="line.153"></a> |
| <span class="sourceLineNo">154</span>import org.apache.hadoop.hbase.replication.regionserver.ReplicationObserver;<a name="line.154"></a> |
| <span class="sourceLineNo">155</span>import org.apache.hadoop.hbase.security.User;<a name="line.155"></a> |
| <span class="sourceLineNo">156</span>import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;<a name="line.156"></a> |
| <span class="sourceLineNo">157</span>import org.apache.hadoop.hbase.snapshot.SnapshotManifest;<a name="line.157"></a> |
| <span class="sourceLineNo">158</span>import org.apache.hadoop.hbase.trace.TraceUtil;<a name="line.158"></a> |
| <span class="sourceLineNo">159</span>import org.apache.hadoop.hbase.util.Bytes;<a name="line.159"></a> |
| <span class="sourceLineNo">160</span>import org.apache.hadoop.hbase.util.CancelableProgressable;<a name="line.160"></a> |
| <span class="sourceLineNo">161</span>import org.apache.hadoop.hbase.util.ClassSize;<a name="line.161"></a> |
| <span class="sourceLineNo">162</span>import org.apache.hadoop.hbase.util.CommonFSUtils;<a name="line.162"></a> |
| <span class="sourceLineNo">163</span>import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;<a name="line.163"></a> |
| <span class="sourceLineNo">164</span>import org.apache.hadoop.hbase.util.FSUtils;<a name="line.164"></a> |
| <span class="sourceLineNo">165</span>import org.apache.hadoop.hbase.util.HashedBytes;<a name="line.165"></a> |
| <span class="sourceLineNo">166</span>import org.apache.hadoop.hbase.util.NonceKey;<a name="line.166"></a> |
| <span class="sourceLineNo">167</span>import org.apache.hadoop.hbase.util.Pair;<a name="line.167"></a> |
| <span class="sourceLineNo">168</span>import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;<a name="line.168"></a> |
| <span class="sourceLineNo">169</span>import org.apache.hadoop.hbase.util.TableDescriptorChecker;<a name="line.169"></a> |
| <span class="sourceLineNo">170</span>import org.apache.hadoop.hbase.util.Threads;<a name="line.170"></a> |
| <span class="sourceLineNo">171</span>import org.apache.hadoop.hbase.wal.WAL;<a name="line.171"></a> |
| <span class="sourceLineNo">172</span>import org.apache.hadoop.hbase.wal.WALEdit;<a name="line.172"></a> |
| <span class="sourceLineNo">173</span>import org.apache.hadoop.hbase.wal.WALFactory;<a name="line.173"></a> |
| <span class="sourceLineNo">174</span>import org.apache.hadoop.hbase.wal.WALKey;<a name="line.174"></a> |
| <span class="sourceLineNo">175</span>import org.apache.hadoop.hbase.wal.WALKeyImpl;<a name="line.175"></a> |
| <span class="sourceLineNo">176</span>import org.apache.hadoop.hbase.wal.WALSplitUtil;<a name="line.176"></a> |
| <span class="sourceLineNo">177</span>import org.apache.hadoop.hbase.wal.WALSplitUtil.MutationReplay;<a name="line.177"></a> |
| <span class="sourceLineNo">178</span>import org.apache.hadoop.util.StringUtils;<a name="line.178"></a> |
| <span class="sourceLineNo">179</span>import org.apache.htrace.core.TraceScope;<a name="line.179"></a> |
| <span class="sourceLineNo">180</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.180"></a> |
| <span class="sourceLineNo">181</span>import org.slf4j.Logger;<a name="line.181"></a> |
| <span class="sourceLineNo">182</span>import org.slf4j.LoggerFactory;<a name="line.182"></a> |
| <span class="sourceLineNo">183</span><a name="line.183"></a> |
| <span class="sourceLineNo">184</span>import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;<a name="line.184"></a> |
| <span class="sourceLineNo">185</span>import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;<a name="line.185"></a> |
| <span class="sourceLineNo">186</span>import org.apache.hbase.thirdparty.com.google.common.collect.Lists;<a name="line.186"></a> |
| <span class="sourceLineNo">187</span>import org.apache.hbase.thirdparty.com.google.common.collect.Maps;<a name="line.187"></a> |
| <span class="sourceLineNo">188</span>import org.apache.hbase.thirdparty.com.google.common.io.Closeables;<a name="line.188"></a> |
| <span class="sourceLineNo">189</span>import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.MethodDescriptor;<a name="line.189"></a> |
| <span class="sourceLineNo">190</span>import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.ServiceDescriptor;<a name="line.190"></a> |
| <span class="sourceLineNo">191</span>import org.apache.hbase.thirdparty.com.google.protobuf.Message;<a name="line.191"></a> |
| <span class="sourceLineNo">192</span>import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;<a name="line.192"></a> |
| <span class="sourceLineNo">193</span>import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;<a name="line.193"></a> |
| <span class="sourceLineNo">194</span>import org.apache.hbase.thirdparty.com.google.protobuf.Service;<a name="line.194"></a> |
| <span class="sourceLineNo">195</span>import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;<a name="line.195"></a> |
| <span class="sourceLineNo">196</span>import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;<a name="line.196"></a> |
| <span class="sourceLineNo">197</span>import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;<a name="line.197"></a> |
| <span class="sourceLineNo">198</span><a name="line.198"></a> |
| <span class="sourceLineNo">199</span>import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;<a name="line.199"></a> |
| <span class="sourceLineNo">200</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;<a name="line.200"></a> |
| <span class="sourceLineNo">201</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceCall;<a name="line.201"></a> |
| <span class="sourceLineNo">202</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionLoad;<a name="line.202"></a> |
| <span class="sourceLineNo">203</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.StoreSequenceId;<a name="line.203"></a> |
| <span class="sourceLineNo">204</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;<a name="line.204"></a> |
| <span class="sourceLineNo">205</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;<a name="line.205"></a> |
| <span class="sourceLineNo">206</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor;<a name="line.206"></a> |
| <span class="sourceLineNo">207</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor;<a name="line.207"></a> |
| <span class="sourceLineNo">208</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;<a name="line.208"></a> |
| <span class="sourceLineNo">209</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;<a name="line.209"></a> |
| <span class="sourceLineNo">210</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor;<a name="line.210"></a> |
| <span class="sourceLineNo">211</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;<a name="line.211"></a> |
| <span class="sourceLineNo">212</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor;<a name="line.212"></a> |
| <span class="sourceLineNo">213</span><a name="line.213"></a> |
| <span class="sourceLineNo">214</span>/**<a name="line.214"></a> |
| <span class="sourceLineNo">215</span> * Regions store data for a certain region of a table. It stores all columns<a name="line.215"></a> |
| <span class="sourceLineNo">216</span> * for each row. A given table consists of one or more Regions.<a name="line.216"></a> |
| <span class="sourceLineNo">217</span> *<a name="line.217"></a> |
| <span class="sourceLineNo">218</span> * <p>An Region is defined by its table and its key extent.<a name="line.218"></a> |
| <span class="sourceLineNo">219</span> *<a name="line.219"></a> |
| <span class="sourceLineNo">220</span> * <p>Locking at the Region level serves only one purpose: preventing the<a name="line.220"></a> |
| <span class="sourceLineNo">221</span> * region from being closed (and consequently split) while other operations<a name="line.221"></a> |
| <span class="sourceLineNo">222</span> * are ongoing. Each row level operation obtains both a row lock and a region<a name="line.222"></a> |
| <span class="sourceLineNo">223</span> * read lock for the duration of the operation. While a scanner is being<a name="line.223"></a> |
| <span class="sourceLineNo">224</span> * constructed, getScanner holds a read lock. If the scanner is successfully<a name="line.224"></a> |
| <span class="sourceLineNo">225</span> * constructed, it holds a read lock until it is closed. A close takes out a<a name="line.225"></a> |
| <span class="sourceLineNo">226</span> * write lock and consequently will block for ongoing operations and will block<a name="line.226"></a> |
| <span class="sourceLineNo">227</span> * new operations from starting while the close is in progress.<a name="line.227"></a> |
| <span class="sourceLineNo">228</span> */<a name="line.228"></a> |
| <span class="sourceLineNo">229</span>@SuppressWarnings("deprecation")<a name="line.229"></a> |
| <span class="sourceLineNo">230</span>@InterfaceAudience.Private<a name="line.230"></a> |
| <span class="sourceLineNo">231</span>public class HRegion implements HeapSize, PropagatingConfigurationObserver, Region {<a name="line.231"></a> |
| <span class="sourceLineNo">232</span> private static final Logger LOG = LoggerFactory.getLogger(HRegion.class);<a name="line.232"></a> |
| <span class="sourceLineNo">233</span><a name="line.233"></a> |
| <span class="sourceLineNo">234</span> public static final String LOAD_CFS_ON_DEMAND_CONFIG_KEY =<a name="line.234"></a> |
| <span class="sourceLineNo">235</span> "hbase.hregion.scan.loadColumnFamiliesOnDemand";<a name="line.235"></a> |
| <span class="sourceLineNo">236</span><a name="line.236"></a> |
| <span class="sourceLineNo">237</span> public static final String HBASE_MAX_CELL_SIZE_KEY = "hbase.server.keyvalue.maxsize";<a name="line.237"></a> |
| <span class="sourceLineNo">238</span> public static final int DEFAULT_MAX_CELL_SIZE = 10485760;<a name="line.238"></a> |
| <span class="sourceLineNo">239</span><a name="line.239"></a> |
| <span class="sourceLineNo">240</span> public static final String HBASE_REGIONSERVER_MINIBATCH_SIZE =<a name="line.240"></a> |
| <span class="sourceLineNo">241</span> "hbase.regionserver.minibatch.size";<a name="line.241"></a> |
| <span class="sourceLineNo">242</span> public static final int DEFAULT_HBASE_REGIONSERVER_MINIBATCH_SIZE = 20000;<a name="line.242"></a> |
| <span class="sourceLineNo">243</span><a name="line.243"></a> |
| <span class="sourceLineNo">244</span> public static final String WAL_HSYNC_CONF_KEY = "hbase.wal.hsync";<a name="line.244"></a> |
| <span class="sourceLineNo">245</span> public static final boolean DEFAULT_WAL_HSYNC = false;<a name="line.245"></a> |
| <span class="sourceLineNo">246</span><a name="line.246"></a> |
| <span class="sourceLineNo">247</span> /**<a name="line.247"></a> |
| <span class="sourceLineNo">248</span> * This is for for using HRegion as a local storage, where we may put the recovered edits in a<a name="line.248"></a> |
| <span class="sourceLineNo">249</span> * special place. Once this is set, we will only replay the recovered edits under this directory<a name="line.249"></a> |
| <span class="sourceLineNo">250</span> * and ignore the original replay directory configs.<a name="line.250"></a> |
| <span class="sourceLineNo">251</span> */<a name="line.251"></a> |
| <span class="sourceLineNo">252</span> public static final String SPECIAL_RECOVERED_EDITS_DIR =<a name="line.252"></a> |
| <span class="sourceLineNo">253</span> "hbase.hregion.special.recovered.edits.dir";<a name="line.253"></a> |
| <span class="sourceLineNo">254</span><a name="line.254"></a> |
| <span class="sourceLineNo">255</span> /**<a name="line.255"></a> |
| <span class="sourceLineNo">256</span> * Whether to use {@link MetaCellComparator} even if we are not meta region. Used when creating<a name="line.256"></a> |
| <span class="sourceLineNo">257</span> * master local region.<a name="line.257"></a> |
| <span class="sourceLineNo">258</span> */<a name="line.258"></a> |
| <span class="sourceLineNo">259</span> public static final String USE_META_CELL_COMPARATOR = "hbase.region.use.meta.cell.comparator";<a name="line.259"></a> |
| <span class="sourceLineNo">260</span><a name="line.260"></a> |
| <span class="sourceLineNo">261</span> public static final boolean DEFAULT_USE_META_CELL_COMPARATOR = false;<a name="line.261"></a> |
| <span class="sourceLineNo">262</span><a name="line.262"></a> |
| <span class="sourceLineNo">263</span> final AtomicBoolean closed = new AtomicBoolean(false);<a name="line.263"></a> |
| <span class="sourceLineNo">264</span><a name="line.264"></a> |
| <span class="sourceLineNo">265</span> /* Closing can take some time; use the closing flag if there is stuff we don't<a name="line.265"></a> |
| <span class="sourceLineNo">266</span> * want to do while in closing state; e.g. like offer this region up to the<a name="line.266"></a> |
| <span class="sourceLineNo">267</span> * master as a region to close if the carrying regionserver is overloaded.<a name="line.267"></a> |
| <span class="sourceLineNo">268</span> * Once set, it is never cleared.<a name="line.268"></a> |
| <span class="sourceLineNo">269</span> */<a name="line.269"></a> |
| <span class="sourceLineNo">270</span> final AtomicBoolean closing = new AtomicBoolean(false);<a name="line.270"></a> |
| <span class="sourceLineNo">271</span><a name="line.271"></a> |
| <span class="sourceLineNo">272</span> /**<a name="line.272"></a> |
| <span class="sourceLineNo">273</span> * The max sequence id of flushed data on this region. There is no edit in memory that is<a name="line.273"></a> |
| <span class="sourceLineNo">274</span> * less that this sequence id.<a name="line.274"></a> |
| <span class="sourceLineNo">275</span> */<a name="line.275"></a> |
| <span class="sourceLineNo">276</span> private volatile long maxFlushedSeqId = HConstants.NO_SEQNUM;<a name="line.276"></a> |
| <span class="sourceLineNo">277</span><a name="line.277"></a> |
| <span class="sourceLineNo">278</span> /**<a name="line.278"></a> |
| <span class="sourceLineNo">279</span> * Record the sequence id of last flush operation. Can be in advance of<a name="line.279"></a> |
| <span class="sourceLineNo">280</span> * {@link #maxFlushedSeqId} when flushing a single column family. In this case,<a name="line.280"></a> |
| <span class="sourceLineNo">281</span> * {@link #maxFlushedSeqId} will be older than the oldest edit in memory.<a name="line.281"></a> |
| <span class="sourceLineNo">282</span> */<a name="line.282"></a> |
| <span class="sourceLineNo">283</span> private volatile long lastFlushOpSeqId = HConstants.NO_SEQNUM;<a name="line.283"></a> |
| <span class="sourceLineNo">284</span><a name="line.284"></a> |
| <span class="sourceLineNo">285</span> /**<a name="line.285"></a> |
| <span class="sourceLineNo">286</span> * The sequence id of the last replayed open region event from the primary region. This is used<a name="line.286"></a> |
| <span class="sourceLineNo">287</span> * to skip entries before this due to the possibility of replay edits coming out of order from<a name="line.287"></a> |
| <span class="sourceLineNo">288</span> * replication.<a name="line.288"></a> |
| <span class="sourceLineNo">289</span> */<a name="line.289"></a> |
| <span class="sourceLineNo">290</span> protected volatile long lastReplayedOpenRegionSeqId = -1L;<a name="line.290"></a> |
| <span class="sourceLineNo">291</span> protected volatile long lastReplayedCompactionSeqId = -1L;<a name="line.291"></a> |
| <span class="sourceLineNo">292</span><a name="line.292"></a> |
| <span class="sourceLineNo">293</span> //////////////////////////////////////////////////////////////////////////////<a name="line.293"></a> |
| <span class="sourceLineNo">294</span> // Members<a name="line.294"></a> |
| <span class="sourceLineNo">295</span> //////////////////////////////////////////////////////////////////////////////<a name="line.295"></a> |
| <span class="sourceLineNo">296</span><a name="line.296"></a> |
| <span class="sourceLineNo">297</span> // map from a locked row to the context for that lock including:<a name="line.297"></a> |
| <span class="sourceLineNo">298</span> // - CountDownLatch for threads waiting on that row<a name="line.298"></a> |
| <span class="sourceLineNo">299</span> // - the thread that owns the lock (allow reentrancy)<a name="line.299"></a> |
| <span class="sourceLineNo">300</span> // - reference count of (reentrant) locks held by the thread<a name="line.300"></a> |
| <span class="sourceLineNo">301</span> // - the row itself<a name="line.301"></a> |
| <span class="sourceLineNo">302</span> private final ConcurrentHashMap<HashedBytes, RowLockContext> lockedRows =<a name="line.302"></a> |
| <span class="sourceLineNo">303</span> new ConcurrentHashMap<>();<a name="line.303"></a> |
| <span class="sourceLineNo">304</span><a name="line.304"></a> |
| <span class="sourceLineNo">305</span> protected final Map<byte[], HStore> stores =<a name="line.305"></a> |
| <span class="sourceLineNo">306</span> new ConcurrentSkipListMap<>(Bytes.BYTES_RAWCOMPARATOR);<a name="line.306"></a> |
| <span class="sourceLineNo">307</span><a name="line.307"></a> |
| <span class="sourceLineNo">308</span> // TODO: account for each registered handler in HeapSize computation<a name="line.308"></a> |
| <span class="sourceLineNo">309</span> private Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();<a name="line.309"></a> |
| <span class="sourceLineNo">310</span><a name="line.310"></a> |
| <span class="sourceLineNo">311</span> // Track data size in all memstores<a name="line.311"></a> |
| <span class="sourceLineNo">312</span> private final MemStoreSizing memStoreSizing = new ThreadSafeMemStoreSizing();<a name="line.312"></a> |
| <span class="sourceLineNo">313</span> RegionServicesForStores regionServicesForStores;<a name="line.313"></a> |
| <span class="sourceLineNo">314</span><a name="line.314"></a> |
| <span class="sourceLineNo">315</span> // Debug possible data loss due to WAL off<a name="line.315"></a> |
| <span class="sourceLineNo">316</span> final LongAdder numMutationsWithoutWAL = new LongAdder();<a name="line.316"></a> |
| <span class="sourceLineNo">317</span> final LongAdder dataInMemoryWithoutWAL = new LongAdder();<a name="line.317"></a> |
| <span class="sourceLineNo">318</span><a name="line.318"></a> |
| <span class="sourceLineNo">319</span> // Debug why CAS operations are taking a while.<a name="line.319"></a> |
| <span class="sourceLineNo">320</span> final LongAdder checkAndMutateChecksPassed = new LongAdder();<a name="line.320"></a> |
| <span class="sourceLineNo">321</span> final LongAdder checkAndMutateChecksFailed = new LongAdder();<a name="line.321"></a> |
| <span class="sourceLineNo">322</span><a name="line.322"></a> |
| <span class="sourceLineNo">323</span> // Number of requests<a name="line.323"></a> |
| <span class="sourceLineNo">324</span> // Count rows for scan<a name="line.324"></a> |
| <span class="sourceLineNo">325</span> final LongAdder readRequestsCount = new LongAdder();<a name="line.325"></a> |
| <span class="sourceLineNo">326</span> final LongAdder cpRequestsCount = new LongAdder();<a name="line.326"></a> |
| <span class="sourceLineNo">327</span> final LongAdder filteredReadRequestsCount = new LongAdder();<a name="line.327"></a> |
| <span class="sourceLineNo">328</span> // Count rows for multi row mutations<a name="line.328"></a> |
| <span class="sourceLineNo">329</span> final LongAdder writeRequestsCount = new LongAdder();<a name="line.329"></a> |
| <span class="sourceLineNo">330</span><a name="line.330"></a> |
| <span class="sourceLineNo">331</span> // Number of requests blocked by memstore size.<a name="line.331"></a> |
| <span class="sourceLineNo">332</span> private final LongAdder blockedRequestsCount = new LongAdder();<a name="line.332"></a> |
| <span class="sourceLineNo">333</span><a name="line.333"></a> |
| <span class="sourceLineNo">334</span> // Compaction LongAdders<a name="line.334"></a> |
| <span class="sourceLineNo">335</span> final LongAdder compactionsFinished = new LongAdder();<a name="line.335"></a> |
| <span class="sourceLineNo">336</span> final LongAdder compactionsFailed = new LongAdder();<a name="line.336"></a> |
| <span class="sourceLineNo">337</span> final LongAdder compactionNumFilesCompacted = new LongAdder();<a name="line.337"></a> |
| <span class="sourceLineNo">338</span> final LongAdder compactionNumBytesCompacted = new LongAdder();<a name="line.338"></a> |
| <span class="sourceLineNo">339</span> final LongAdder compactionsQueued = new LongAdder();<a name="line.339"></a> |
| <span class="sourceLineNo">340</span> final LongAdder flushesQueued = new LongAdder();<a name="line.340"></a> |
| <span class="sourceLineNo">341</span><a name="line.341"></a> |
| <span class="sourceLineNo">342</span> private BlockCache blockCache;<a name="line.342"></a> |
| <span class="sourceLineNo">343</span> private MobFileCache mobFileCache;<a name="line.343"></a> |
| <span class="sourceLineNo">344</span> private final WAL wal;<a name="line.344"></a> |
| <span class="sourceLineNo">345</span> private final HRegionFileSystem fs;<a name="line.345"></a> |
| <span class="sourceLineNo">346</span> protected final Configuration conf;<a name="line.346"></a> |
| <span class="sourceLineNo">347</span> private final Configuration baseConf;<a name="line.347"></a> |
| <span class="sourceLineNo">348</span> private final int rowLockWaitDuration;<a name="line.348"></a> |
| <span class="sourceLineNo">349</span> static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000;<a name="line.349"></a> |
| <span class="sourceLineNo">350</span><a name="line.350"></a> |
| <span class="sourceLineNo">351</span> private Path regionDir;<a name="line.351"></a> |
| <span class="sourceLineNo">352</span> private FileSystem walFS;<a name="line.352"></a> |
| <span class="sourceLineNo">353</span><a name="line.353"></a> |
| <span class="sourceLineNo">354</span> // set to true if the region is restored from snapshot<a name="line.354"></a> |
| <span class="sourceLineNo">355</span> private boolean isRestoredRegion = false;<a name="line.355"></a> |
| <span class="sourceLineNo">356</span><a name="line.356"></a> |
| <span class="sourceLineNo">357</span> public void setRestoredRegion(boolean restoredRegion) {<a name="line.357"></a> |
| <span class="sourceLineNo">358</span> isRestoredRegion = restoredRegion;<a name="line.358"></a> |
| <span class="sourceLineNo">359</span> }<a name="line.359"></a> |
| <span class="sourceLineNo">360</span><a name="line.360"></a> |
| <span class="sourceLineNo">361</span> // The internal wait duration to acquire a lock before read/update<a name="line.361"></a> |
| <span class="sourceLineNo">362</span> // from the region. It is not per row. The purpose of this wait time<a name="line.362"></a> |
| <span class="sourceLineNo">363</span> // is to avoid waiting a long time while the region is busy, so that<a name="line.363"></a> |
| <span class="sourceLineNo">364</span> // we can release the IPC handler soon enough to improve the<a name="line.364"></a> |
| <span class="sourceLineNo">365</span> // availability of the region server. It can be adjusted by<a name="line.365"></a> |
| <span class="sourceLineNo">366</span> // tuning configuration "hbase.busy.wait.duration".<a name="line.366"></a> |
| <span class="sourceLineNo">367</span> final long busyWaitDuration;<a name="line.367"></a> |
| <span class="sourceLineNo">368</span> static final long DEFAULT_BUSY_WAIT_DURATION = HConstants.DEFAULT_HBASE_RPC_TIMEOUT;<a name="line.368"></a> |
| <span class="sourceLineNo">369</span><a name="line.369"></a> |
| <span class="sourceLineNo">370</span> // If updating multiple rows in one call, wait longer,<a name="line.370"></a> |
| <span class="sourceLineNo">371</span> // i.e. waiting for busyWaitDuration * # of rows. However,<a name="line.371"></a> |
| <span class="sourceLineNo">372</span> // we can limit the max multiplier.<a name="line.372"></a> |
| <span class="sourceLineNo">373</span> final int maxBusyWaitMultiplier;<a name="line.373"></a> |
| <span class="sourceLineNo">374</span><a name="line.374"></a> |
| <span class="sourceLineNo">375</span> // Max busy wait duration. There is no point to wait longer than the RPC<a name="line.375"></a> |
| <span class="sourceLineNo">376</span> // purge timeout, when a RPC call will be terminated by the RPC engine.<a name="line.376"></a> |
| <span class="sourceLineNo">377</span> final long maxBusyWaitDuration;<a name="line.377"></a> |
| <span class="sourceLineNo">378</span><a name="line.378"></a> |
| <span class="sourceLineNo">379</span> // Max cell size. If nonzero, the maximum allowed size for any given cell<a name="line.379"></a> |
| <span class="sourceLineNo">380</span> // in bytes<a name="line.380"></a> |
| <span class="sourceLineNo">381</span> final long maxCellSize;<a name="line.381"></a> |
| <span class="sourceLineNo">382</span><a name="line.382"></a> |
| <span class="sourceLineNo">383</span> // Number of mutations for minibatch processing.<a name="line.383"></a> |
| <span class="sourceLineNo">384</span> private final int miniBatchSize;<a name="line.384"></a> |
| <span class="sourceLineNo">385</span><a name="line.385"></a> |
| <span class="sourceLineNo">386</span> // negative number indicates infinite timeout<a name="line.386"></a> |
| <span class="sourceLineNo">387</span> static final long DEFAULT_ROW_PROCESSOR_TIMEOUT = 60 * 1000L;<a name="line.387"></a> |
| <span class="sourceLineNo">388</span> final ExecutorService rowProcessorExecutor = Executors.newCachedThreadPool();<a name="line.388"></a> |
| <span class="sourceLineNo">389</span><a name="line.389"></a> |
| <span class="sourceLineNo">390</span> final ConcurrentHashMap<RegionScanner, Long> scannerReadPoints;<a name="line.390"></a> |
| <span class="sourceLineNo">391</span><a name="line.391"></a> |
| <span class="sourceLineNo">392</span> /**<a name="line.392"></a> |
| <span class="sourceLineNo">393</span> * The sequence ID that was enLongAddered when this region was opened.<a name="line.393"></a> |
| <span class="sourceLineNo">394</span> */<a name="line.394"></a> |
| <span class="sourceLineNo">395</span> private long openSeqNum = HConstants.NO_SEQNUM;<a name="line.395"></a> |
| <span class="sourceLineNo">396</span><a name="line.396"></a> |
| <span class="sourceLineNo">397</span> /**<a name="line.397"></a> |
| <span class="sourceLineNo">398</span> * The default setting for whether to enable on-demand CF loading for<a name="line.398"></a> |
| <span class="sourceLineNo">399</span> * scan requests to this region. Requests can override it.<a name="line.399"></a> |
| <span class="sourceLineNo">400</span> */<a name="line.400"></a> |
| <span class="sourceLineNo">401</span> private boolean isLoadingCfsOnDemandDefault = false;<a name="line.401"></a> |
| <span class="sourceLineNo">402</span><a name="line.402"></a> |
| <span class="sourceLineNo">403</span> private final AtomicInteger majorInProgress = new AtomicInteger(0);<a name="line.403"></a> |
| <span class="sourceLineNo">404</span> private final AtomicInteger minorInProgress = new AtomicInteger(0);<a name="line.404"></a> |
| <span class="sourceLineNo">405</span><a name="line.405"></a> |
| <span class="sourceLineNo">406</span> //<a name="line.406"></a> |
| <span class="sourceLineNo">407</span> // Context: During replay we want to ensure that we do not lose any data. So, we<a name="line.407"></a> |
| <span class="sourceLineNo">408</span> // have to be conservative in how we replay wals. For each store, we calculate<a name="line.408"></a> |
| <span class="sourceLineNo">409</span> // the maxSeqId up to which the store was flushed. And, skip the edits which<a name="line.409"></a> |
| <span class="sourceLineNo">410</span> // are equal to or lower than maxSeqId for each store.<a name="line.410"></a> |
| <span class="sourceLineNo">411</span> // The following map is populated when opening the region<a name="line.411"></a> |
| <span class="sourceLineNo">412</span> Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.412"></a> |
| <span class="sourceLineNo">413</span><a name="line.413"></a> |
| <span class="sourceLineNo">414</span> /** Saved state from replaying prepare flush cache */<a name="line.414"></a> |
| <span class="sourceLineNo">415</span> private PrepareFlushResult prepareFlushResult = null;<a name="line.415"></a> |
| <span class="sourceLineNo">416</span><a name="line.416"></a> |
| <span class="sourceLineNo">417</span> private volatile ConfigurationManager configurationManager;<a name="line.417"></a> |
| <span class="sourceLineNo">418</span><a name="line.418"></a> |
| <span class="sourceLineNo">419</span> // Used for testing.<a name="line.419"></a> |
| <span class="sourceLineNo">420</span> private volatile Long timeoutForWriteLock = null;<a name="line.420"></a> |
| <span class="sourceLineNo">421</span><a name="line.421"></a> |
| <span class="sourceLineNo">422</span> private final CellComparator cellComparator;<a name="line.422"></a> |
| <span class="sourceLineNo">423</span><a name="line.423"></a> |
| <span class="sourceLineNo">424</span> /**<a name="line.424"></a> |
| <span class="sourceLineNo">425</span> * @return The smallest mvcc readPoint across all the scanners in this<a name="line.425"></a> |
| <span class="sourceLineNo">426</span> * region. Writes older than this readPoint, are included in every<a name="line.426"></a> |
| <span class="sourceLineNo">427</span> * read operation.<a name="line.427"></a> |
| <span class="sourceLineNo">428</span> */<a name="line.428"></a> |
| <span class="sourceLineNo">429</span> public long getSmallestReadPoint() {<a name="line.429"></a> |
| <span class="sourceLineNo">430</span> long minimumReadPoint;<a name="line.430"></a> |
| <span class="sourceLineNo">431</span> // We need to ensure that while we are calculating the smallestReadPoint<a name="line.431"></a> |
| <span class="sourceLineNo">432</span> // no new RegionScanners can grab a readPoint that we are unaware of.<a name="line.432"></a> |
| <span class="sourceLineNo">433</span> // We achieve this by synchronizing on the scannerReadPoints object.<a name="line.433"></a> |
| <span class="sourceLineNo">434</span> synchronized (scannerReadPoints) {<a name="line.434"></a> |
| <span class="sourceLineNo">435</span> minimumReadPoint = mvcc.getReadPoint();<a name="line.435"></a> |
| <span class="sourceLineNo">436</span> for (Long readPoint : this.scannerReadPoints.values()) {<a name="line.436"></a> |
| <span class="sourceLineNo">437</span> if (readPoint < minimumReadPoint) {<a name="line.437"></a> |
| <span class="sourceLineNo">438</span> minimumReadPoint = readPoint;<a name="line.438"></a> |
| <span class="sourceLineNo">439</span> }<a name="line.439"></a> |
| <span class="sourceLineNo">440</span> }<a name="line.440"></a> |
| <span class="sourceLineNo">441</span> }<a name="line.441"></a> |
| <span class="sourceLineNo">442</span> return minimumReadPoint;<a name="line.442"></a> |
| <span class="sourceLineNo">443</span> }<a name="line.443"></a> |
| <span class="sourceLineNo">444</span><a name="line.444"></a> |
| <span class="sourceLineNo">445</span> /*<a name="line.445"></a> |
| <span class="sourceLineNo">446</span> * Data structure of write state flags used coordinating flushes,<a name="line.446"></a> |
| <span class="sourceLineNo">447</span> * compactions and closes.<a name="line.447"></a> |
| <span class="sourceLineNo">448</span> */<a name="line.448"></a> |
| <span class="sourceLineNo">449</span> static class WriteState {<a name="line.449"></a> |
| <span class="sourceLineNo">450</span> // Set while a memstore flush is happening.<a name="line.450"></a> |
| <span class="sourceLineNo">451</span> volatile boolean flushing = false;<a name="line.451"></a> |
| <span class="sourceLineNo">452</span> // Set when a flush has been requested.<a name="line.452"></a> |
| <span class="sourceLineNo">453</span> volatile boolean flushRequested = false;<a name="line.453"></a> |
| <span class="sourceLineNo">454</span> // Number of compactions running.<a name="line.454"></a> |
| <span class="sourceLineNo">455</span> AtomicInteger compacting = new AtomicInteger(0);<a name="line.455"></a> |
| <span class="sourceLineNo">456</span> // Gets set in close. If set, cannot compact or flush again.<a name="line.456"></a> |
| <span class="sourceLineNo">457</span> volatile boolean writesEnabled = true;<a name="line.457"></a> |
| <span class="sourceLineNo">458</span> // Set if region is read-only<a name="line.458"></a> |
| <span class="sourceLineNo">459</span> volatile boolean readOnly = false;<a name="line.459"></a> |
| <span class="sourceLineNo">460</span> // whether the reads are enabled. This is different than readOnly, because readOnly is<a name="line.460"></a> |
| <span class="sourceLineNo">461</span> // static in the lifetime of the region, while readsEnabled is dynamic<a name="line.461"></a> |
| <span class="sourceLineNo">462</span> volatile boolean readsEnabled = true;<a name="line.462"></a> |
| <span class="sourceLineNo">463</span><a name="line.463"></a> |
| <span class="sourceLineNo">464</span> /**<a name="line.464"></a> |
| <span class="sourceLineNo">465</span> * Set flags that make this region read-only.<a name="line.465"></a> |
| <span class="sourceLineNo">466</span> *<a name="line.466"></a> |
| <span class="sourceLineNo">467</span> * @param onOff flip value for region r/o setting<a name="line.467"></a> |
| <span class="sourceLineNo">468</span> */<a name="line.468"></a> |
| <span class="sourceLineNo">469</span> synchronized void setReadOnly(final boolean onOff) {<a name="line.469"></a> |
| <span class="sourceLineNo">470</span> this.writesEnabled = !onOff;<a name="line.470"></a> |
| <span class="sourceLineNo">471</span> this.readOnly = onOff;<a name="line.471"></a> |
| <span class="sourceLineNo">472</span> }<a name="line.472"></a> |
| <span class="sourceLineNo">473</span><a name="line.473"></a> |
| <span class="sourceLineNo">474</span> boolean isReadOnly() {<a name="line.474"></a> |
| <span class="sourceLineNo">475</span> return this.readOnly;<a name="line.475"></a> |
| <span class="sourceLineNo">476</span> }<a name="line.476"></a> |
| <span class="sourceLineNo">477</span><a name="line.477"></a> |
| <span class="sourceLineNo">478</span> boolean isFlushRequested() {<a name="line.478"></a> |
| <span class="sourceLineNo">479</span> return this.flushRequested;<a name="line.479"></a> |
| <span class="sourceLineNo">480</span> }<a name="line.480"></a> |
| <span class="sourceLineNo">481</span><a name="line.481"></a> |
| <span class="sourceLineNo">482</span> void setReadsEnabled(boolean readsEnabled) {<a name="line.482"></a> |
| <span class="sourceLineNo">483</span> this.readsEnabled = readsEnabled;<a name="line.483"></a> |
| <span class="sourceLineNo">484</span> }<a name="line.484"></a> |
| <span class="sourceLineNo">485</span><a name="line.485"></a> |
| <span class="sourceLineNo">486</span> static final long HEAP_SIZE = ClassSize.align(<a name="line.486"></a> |
| <span class="sourceLineNo">487</span> ClassSize.OBJECT + 5 * Bytes.SIZEOF_BOOLEAN);<a name="line.487"></a> |
| <span class="sourceLineNo">488</span> }<a name="line.488"></a> |
| <span class="sourceLineNo">489</span><a name="line.489"></a> |
| <span class="sourceLineNo">490</span> /**<a name="line.490"></a> |
| <span class="sourceLineNo">491</span> * Objects from this class are created when flushing to describe all the different states that<a name="line.491"></a> |
| <span class="sourceLineNo">492</span> * that method ends up in. The Result enum describes those states. The sequence id should only<a name="line.492"></a> |
| <span class="sourceLineNo">493</span> * be specified if the flush was successful, and the failure message should only be specified<a name="line.493"></a> |
| <span class="sourceLineNo">494</span> * if it didn't flush.<a name="line.494"></a> |
| <span class="sourceLineNo">495</span> */<a name="line.495"></a> |
| <span class="sourceLineNo">496</span> public static class FlushResultImpl implements FlushResult {<a name="line.496"></a> |
| <span class="sourceLineNo">497</span> final Result result;<a name="line.497"></a> |
| <span class="sourceLineNo">498</span> final String failureReason;<a name="line.498"></a> |
| <span class="sourceLineNo">499</span> final long flushSequenceId;<a name="line.499"></a> |
| <span class="sourceLineNo">500</span> final boolean wroteFlushWalMarker;<a name="line.500"></a> |
| <span class="sourceLineNo">501</span><a name="line.501"></a> |
| <span class="sourceLineNo">502</span> /**<a name="line.502"></a> |
| <span class="sourceLineNo">503</span> * Convenience constructor to use when the flush is successful, the failure message is set to<a name="line.503"></a> |
| <span class="sourceLineNo">504</span> * null.<a name="line.504"></a> |
| <span class="sourceLineNo">505</span> * @param result Expecting FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_COMPACTION_NEEDED.<a name="line.505"></a> |
| <span class="sourceLineNo">506</span> * @param flushSequenceId Generated sequence id that comes right after the edits in the<a name="line.506"></a> |
| <span class="sourceLineNo">507</span> * memstores.<a name="line.507"></a> |
| <span class="sourceLineNo">508</span> */<a name="line.508"></a> |
| <span class="sourceLineNo">509</span> FlushResultImpl(Result result, long flushSequenceId) {<a name="line.509"></a> |
| <span class="sourceLineNo">510</span> this(result, flushSequenceId, null, false);<a name="line.510"></a> |
| <span class="sourceLineNo">511</span> assert result == Result.FLUSHED_NO_COMPACTION_NEEDED || result == Result<a name="line.511"></a> |
| <span class="sourceLineNo">512</span> .FLUSHED_COMPACTION_NEEDED;<a name="line.512"></a> |
| <span class="sourceLineNo">513</span> }<a name="line.513"></a> |
| <span class="sourceLineNo">514</span><a name="line.514"></a> |
| <span class="sourceLineNo">515</span> /**<a name="line.515"></a> |
| <span class="sourceLineNo">516</span> * Convenience constructor to use when we cannot flush.<a name="line.516"></a> |
| <span class="sourceLineNo">517</span> * @param result Expecting CANNOT_FLUSH_MEMSTORE_EMPTY or CANNOT_FLUSH.<a name="line.517"></a> |
| <span class="sourceLineNo">518</span> * @param failureReason Reason why we couldn't flush.<a name="line.518"></a> |
| <span class="sourceLineNo">519</span> */<a name="line.519"></a> |
| <span class="sourceLineNo">520</span> FlushResultImpl(Result result, String failureReason, boolean wroteFlushMarker) {<a name="line.520"></a> |
| <span class="sourceLineNo">521</span> this(result, -1, failureReason, wroteFlushMarker);<a name="line.521"></a> |
| <span class="sourceLineNo">522</span> assert result == Result.CANNOT_FLUSH_MEMSTORE_EMPTY || result == Result.CANNOT_FLUSH;<a name="line.522"></a> |
| <span class="sourceLineNo">523</span> }<a name="line.523"></a> |
| <span class="sourceLineNo">524</span><a name="line.524"></a> |
| <span class="sourceLineNo">525</span> /**<a name="line.525"></a> |
| <span class="sourceLineNo">526</span> * Constructor with all the parameters.<a name="line.526"></a> |
| <span class="sourceLineNo">527</span> * @param result Any of the Result.<a name="line.527"></a> |
| <span class="sourceLineNo">528</span> * @param flushSequenceId Generated sequence id if the memstores were flushed else -1.<a name="line.528"></a> |
| <span class="sourceLineNo">529</span> * @param failureReason Reason why we couldn't flush, or null.<a name="line.529"></a> |
| <span class="sourceLineNo">530</span> */<a name="line.530"></a> |
| <span class="sourceLineNo">531</span> FlushResultImpl(Result result, long flushSequenceId, String failureReason,<a name="line.531"></a> |
| <span class="sourceLineNo">532</span> boolean wroteFlushMarker) {<a name="line.532"></a> |
| <span class="sourceLineNo">533</span> this.result = result;<a name="line.533"></a> |
| <span class="sourceLineNo">534</span> this.flushSequenceId = flushSequenceId;<a name="line.534"></a> |
| <span class="sourceLineNo">535</span> this.failureReason = failureReason;<a name="line.535"></a> |
| <span class="sourceLineNo">536</span> this.wroteFlushWalMarker = wroteFlushMarker;<a name="line.536"></a> |
| <span class="sourceLineNo">537</span> }<a name="line.537"></a> |
| <span class="sourceLineNo">538</span><a name="line.538"></a> |
| <span class="sourceLineNo">539</span> /**<a name="line.539"></a> |
| <span class="sourceLineNo">540</span> * Convenience method, the equivalent of checking if result is<a name="line.540"></a> |
| <span class="sourceLineNo">541</span> * FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_NO_COMPACTION_NEEDED.<a name="line.541"></a> |
| <span class="sourceLineNo">542</span> * @return true if the memstores were flushed, else false.<a name="line.542"></a> |
| <span class="sourceLineNo">543</span> */<a name="line.543"></a> |
| <span class="sourceLineNo">544</span> @Override<a name="line.544"></a> |
| <span class="sourceLineNo">545</span> public boolean isFlushSucceeded() {<a name="line.545"></a> |
| <span class="sourceLineNo">546</span> return result == Result.FLUSHED_NO_COMPACTION_NEEDED || result == Result<a name="line.546"></a> |
| <span class="sourceLineNo">547</span> .FLUSHED_COMPACTION_NEEDED;<a name="line.547"></a> |
| <span class="sourceLineNo">548</span> }<a name="line.548"></a> |
| <span class="sourceLineNo">549</span><a name="line.549"></a> |
| <span class="sourceLineNo">550</span> /**<a name="line.550"></a> |
| <span class="sourceLineNo">551</span> * Convenience method, the equivalent of checking if result is FLUSHED_COMPACTION_NEEDED.<a name="line.551"></a> |
| <span class="sourceLineNo">552</span> * @return True if the flush requested a compaction, else false (doesn't even mean it flushed).<a name="line.552"></a> |
| <span class="sourceLineNo">553</span> */<a name="line.553"></a> |
| <span class="sourceLineNo">554</span> @Override<a name="line.554"></a> |
| <span class="sourceLineNo">555</span> public boolean isCompactionNeeded() {<a name="line.555"></a> |
| <span class="sourceLineNo">556</span> return result == Result.FLUSHED_COMPACTION_NEEDED;<a name="line.556"></a> |
| <span class="sourceLineNo">557</span> }<a name="line.557"></a> |
| <span class="sourceLineNo">558</span><a name="line.558"></a> |
| <span class="sourceLineNo">559</span> @Override<a name="line.559"></a> |
| <span class="sourceLineNo">560</span> public String toString() {<a name="line.560"></a> |
| <span class="sourceLineNo">561</span> return new StringBuilder()<a name="line.561"></a> |
| <span class="sourceLineNo">562</span> .append("flush result:").append(result).append(", ")<a name="line.562"></a> |
| <span class="sourceLineNo">563</span> .append("failureReason:").append(failureReason).append(",")<a name="line.563"></a> |
| <span class="sourceLineNo">564</span> .append("flush seq id").append(flushSequenceId).toString();<a name="line.564"></a> |
| <span class="sourceLineNo">565</span> }<a name="line.565"></a> |
| <span class="sourceLineNo">566</span><a name="line.566"></a> |
| <span class="sourceLineNo">567</span> @Override<a name="line.567"></a> |
| <span class="sourceLineNo">568</span> public Result getResult() {<a name="line.568"></a> |
| <span class="sourceLineNo">569</span> return result;<a name="line.569"></a> |
| <span class="sourceLineNo">570</span> }<a name="line.570"></a> |
| <span class="sourceLineNo">571</span> }<a name="line.571"></a> |
| <span class="sourceLineNo">572</span><a name="line.572"></a> |
| <span class="sourceLineNo">573</span> /** A result object from prepare flush cache stage */<a name="line.573"></a> |
| <span class="sourceLineNo">574</span> static class PrepareFlushResult {<a name="line.574"></a> |
| <span class="sourceLineNo">575</span> final FlushResultImpl result; // indicating a failure result from prepare<a name="line.575"></a> |
| <span class="sourceLineNo">576</span> final TreeMap<byte[], StoreFlushContext> storeFlushCtxs;<a name="line.576"></a> |
| <span class="sourceLineNo">577</span> final TreeMap<byte[], List<Path>> committedFiles;<a name="line.577"></a> |
| <span class="sourceLineNo">578</span> final TreeMap<byte[], MemStoreSize> storeFlushableSize;<a name="line.578"></a> |
| <span class="sourceLineNo">579</span> final long startTime;<a name="line.579"></a> |
| <span class="sourceLineNo">580</span> final long flushOpSeqId;<a name="line.580"></a> |
| <span class="sourceLineNo">581</span> final long flushedSeqId;<a name="line.581"></a> |
| <span class="sourceLineNo">582</span> final MemStoreSizing totalFlushableSize;<a name="line.582"></a> |
| <span class="sourceLineNo">583</span><a name="line.583"></a> |
| <span class="sourceLineNo">584</span> /** Constructs an early exit case */<a name="line.584"></a> |
| <span class="sourceLineNo">585</span> PrepareFlushResult(FlushResultImpl result, long flushSeqId) {<a name="line.585"></a> |
| <span class="sourceLineNo">586</span> this(result, null, null, null, Math.max(0, flushSeqId), 0, 0, MemStoreSizing.DUD);<a name="line.586"></a> |
| <span class="sourceLineNo">587</span> }<a name="line.587"></a> |
| <span class="sourceLineNo">588</span><a name="line.588"></a> |
| <span class="sourceLineNo">589</span> /** Constructs a successful prepare flush result */<a name="line.589"></a> |
| <span class="sourceLineNo">590</span> PrepareFlushResult(<a name="line.590"></a> |
| <span class="sourceLineNo">591</span> TreeMap<byte[], StoreFlushContext> storeFlushCtxs,<a name="line.591"></a> |
| <span class="sourceLineNo">592</span> TreeMap<byte[], List<Path>> committedFiles,<a name="line.592"></a> |
| <span class="sourceLineNo">593</span> TreeMap<byte[], MemStoreSize> storeFlushableSize, long startTime, long flushSeqId,<a name="line.593"></a> |
| <span class="sourceLineNo">594</span> long flushedSeqId, MemStoreSizing totalFlushableSize) {<a name="line.594"></a> |
| <span class="sourceLineNo">595</span> this(null, storeFlushCtxs, committedFiles, storeFlushableSize, startTime,<a name="line.595"></a> |
| <span class="sourceLineNo">596</span> flushSeqId, flushedSeqId, totalFlushableSize);<a name="line.596"></a> |
| <span class="sourceLineNo">597</span> }<a name="line.597"></a> |
| <span class="sourceLineNo">598</span><a name="line.598"></a> |
| <span class="sourceLineNo">599</span> private PrepareFlushResult(<a name="line.599"></a> |
| <span class="sourceLineNo">600</span> FlushResultImpl result,<a name="line.600"></a> |
| <span class="sourceLineNo">601</span> TreeMap<byte[], StoreFlushContext> storeFlushCtxs,<a name="line.601"></a> |
| <span class="sourceLineNo">602</span> TreeMap<byte[], List<Path>> committedFiles,<a name="line.602"></a> |
| <span class="sourceLineNo">603</span> TreeMap<byte[], MemStoreSize> storeFlushableSize, long startTime, long flushSeqId,<a name="line.603"></a> |
| <span class="sourceLineNo">604</span> long flushedSeqId, MemStoreSizing totalFlushableSize) {<a name="line.604"></a> |
| <span class="sourceLineNo">605</span> this.result = result;<a name="line.605"></a> |
| <span class="sourceLineNo">606</span> this.storeFlushCtxs = storeFlushCtxs;<a name="line.606"></a> |
| <span class="sourceLineNo">607</span> this.committedFiles = committedFiles;<a name="line.607"></a> |
| <span class="sourceLineNo">608</span> this.storeFlushableSize = storeFlushableSize;<a name="line.608"></a> |
| <span class="sourceLineNo">609</span> this.startTime = startTime;<a name="line.609"></a> |
| <span class="sourceLineNo">610</span> this.flushOpSeqId = flushSeqId;<a name="line.610"></a> |
| <span class="sourceLineNo">611</span> this.flushedSeqId = flushedSeqId;<a name="line.611"></a> |
| <span class="sourceLineNo">612</span> this.totalFlushableSize = totalFlushableSize;<a name="line.612"></a> |
| <span class="sourceLineNo">613</span> }<a name="line.613"></a> |
| <span class="sourceLineNo">614</span><a name="line.614"></a> |
| <span class="sourceLineNo">615</span> public FlushResult getResult() {<a name="line.615"></a> |
| <span class="sourceLineNo">616</span> return this.result;<a name="line.616"></a> |
| <span class="sourceLineNo">617</span> }<a name="line.617"></a> |
| <span class="sourceLineNo">618</span> }<a name="line.618"></a> |
| <span class="sourceLineNo">619</span><a name="line.619"></a> |
| <span class="sourceLineNo">620</span> /**<a name="line.620"></a> |
| <span class="sourceLineNo">621</span> * A class that tracks exceptions that have been observed in one batch. Not thread safe.<a name="line.621"></a> |
| <span class="sourceLineNo">622</span> */<a name="line.622"></a> |
| <span class="sourceLineNo">623</span> static class ObservedExceptionsInBatch {<a name="line.623"></a> |
| <span class="sourceLineNo">624</span> private boolean wrongRegion = false;<a name="line.624"></a> |
| <span class="sourceLineNo">625</span> private boolean failedSanityCheck = false;<a name="line.625"></a> |
| <span class="sourceLineNo">626</span> private boolean wrongFamily = false;<a name="line.626"></a> |
| <span class="sourceLineNo">627</span><a name="line.627"></a> |
| <span class="sourceLineNo">628</span> /**<a name="line.628"></a> |
| <span class="sourceLineNo">629</span> * @return If a {@link WrongRegionException} has been observed.<a name="line.629"></a> |
| <span class="sourceLineNo">630</span> */<a name="line.630"></a> |
| <span class="sourceLineNo">631</span> boolean hasSeenWrongRegion() {<a name="line.631"></a> |
| <span class="sourceLineNo">632</span> return wrongRegion;<a name="line.632"></a> |
| <span class="sourceLineNo">633</span> }<a name="line.633"></a> |
| <span class="sourceLineNo">634</span><a name="line.634"></a> |
| <span class="sourceLineNo">635</span> /**<a name="line.635"></a> |
| <span class="sourceLineNo">636</span> * Records that a {@link WrongRegionException} has been observed.<a name="line.636"></a> |
| <span class="sourceLineNo">637</span> */<a name="line.637"></a> |
| <span class="sourceLineNo">638</span> void sawWrongRegion() {<a name="line.638"></a> |
| <span class="sourceLineNo">639</span> wrongRegion = true;<a name="line.639"></a> |
| <span class="sourceLineNo">640</span> }<a name="line.640"></a> |
| <span class="sourceLineNo">641</span><a name="line.641"></a> |
| <span class="sourceLineNo">642</span> /**<a name="line.642"></a> |
| <span class="sourceLineNo">643</span> * @return If a {@link FailedSanityCheckException} has been observed.<a name="line.643"></a> |
| <span class="sourceLineNo">644</span> */<a name="line.644"></a> |
| <span class="sourceLineNo">645</span> boolean hasSeenFailedSanityCheck() {<a name="line.645"></a> |
| <span class="sourceLineNo">646</span> return failedSanityCheck;<a name="line.646"></a> |
| <span class="sourceLineNo">647</span> }<a name="line.647"></a> |
| <span class="sourceLineNo">648</span><a name="line.648"></a> |
| <span class="sourceLineNo">649</span> /**<a name="line.649"></a> |
| <span class="sourceLineNo">650</span> * Records that a {@link FailedSanityCheckException} has been observed.<a name="line.650"></a> |
| <span class="sourceLineNo">651</span> */<a name="line.651"></a> |
| <span class="sourceLineNo">652</span> void sawFailedSanityCheck() {<a name="line.652"></a> |
| <span class="sourceLineNo">653</span> failedSanityCheck = true;<a name="line.653"></a> |
| <span class="sourceLineNo">654</span> }<a name="line.654"></a> |
| <span class="sourceLineNo">655</span><a name="line.655"></a> |
| <span class="sourceLineNo">656</span> /**<a name="line.656"></a> |
| <span class="sourceLineNo">657</span> * @return If a {@link NoSuchColumnFamilyException} has been observed.<a name="line.657"></a> |
| <span class="sourceLineNo">658</span> */<a name="line.658"></a> |
| <span class="sourceLineNo">659</span> boolean hasSeenNoSuchFamily() {<a name="line.659"></a> |
| <span class="sourceLineNo">660</span> return wrongFamily;<a name="line.660"></a> |
| <span class="sourceLineNo">661</span> }<a name="line.661"></a> |
| <span class="sourceLineNo">662</span><a name="line.662"></a> |
| <span class="sourceLineNo">663</span> /**<a name="line.663"></a> |
| <span class="sourceLineNo">664</span> * Records that a {@link NoSuchColumnFamilyException} has been observed.<a name="line.664"></a> |
| <span class="sourceLineNo">665</span> */<a name="line.665"></a> |
| <span class="sourceLineNo">666</span> void sawNoSuchFamily() {<a name="line.666"></a> |
| <span class="sourceLineNo">667</span> wrongFamily = true;<a name="line.667"></a> |
| <span class="sourceLineNo">668</span> }<a name="line.668"></a> |
| <span class="sourceLineNo">669</span> }<a name="line.669"></a> |
| <span class="sourceLineNo">670</span><a name="line.670"></a> |
| <span class="sourceLineNo">671</span> final WriteState writestate = new WriteState();<a name="line.671"></a> |
| <span class="sourceLineNo">672</span><a name="line.672"></a> |
| <span class="sourceLineNo">673</span> long memstoreFlushSize;<a name="line.673"></a> |
| <span class="sourceLineNo">674</span> final long timestampSlop;<a name="line.674"></a> |
| <span class="sourceLineNo">675</span> final long rowProcessorTimeout;<a name="line.675"></a> |
| <span class="sourceLineNo">676</span><a name="line.676"></a> |
| <span class="sourceLineNo">677</span> // Last flush time for each Store. Useful when we are flushing for each column<a name="line.677"></a> |
| <span class="sourceLineNo">678</span> private final ConcurrentMap<HStore, Long> lastStoreFlushTimeMap = new ConcurrentHashMap<>();<a name="line.678"></a> |
| <span class="sourceLineNo">679</span><a name="line.679"></a> |
| <span class="sourceLineNo">680</span> protected RegionServerServices rsServices;<a name="line.680"></a> |
| <span class="sourceLineNo">681</span> private RegionServerAccounting rsAccounting;<a name="line.681"></a> |
| <span class="sourceLineNo">682</span> private long flushCheckInterval;<a name="line.682"></a> |
| <span class="sourceLineNo">683</span> // flushPerChanges is to prevent too many changes in memstore<a name="line.683"></a> |
| <span class="sourceLineNo">684</span> private long flushPerChanges;<a name="line.684"></a> |
| <span class="sourceLineNo">685</span> private long blockingMemStoreSize;<a name="line.685"></a> |
| <span class="sourceLineNo">686</span> // Used to guard closes<a name="line.686"></a> |
| <span class="sourceLineNo">687</span> final ReentrantReadWriteLock lock;<a name="line.687"></a> |
| <span class="sourceLineNo">688</span> // Used to track interruptible holders of the region lock. Currently that is only RPC handler<a name="line.688"></a> |
| <span class="sourceLineNo">689</span> // threads. Boolean value in map determines if lock holder can be interrupted, normally true,<a name="line.689"></a> |
| <span class="sourceLineNo">690</span> // but may be false when thread is transiting a critical section.<a name="line.690"></a> |
| <span class="sourceLineNo">691</span> final ConcurrentHashMap<Thread, Boolean> regionLockHolders;<a name="line.691"></a> |
| <span class="sourceLineNo">692</span><a name="line.692"></a> |
| <span class="sourceLineNo">693</span> // Stop updates lock<a name="line.693"></a> |
| <span class="sourceLineNo">694</span> private final ReentrantReadWriteLock updatesLock = new ReentrantReadWriteLock();<a name="line.694"></a> |
| <span class="sourceLineNo">695</span><a name="line.695"></a> |
| <span class="sourceLineNo">696</span> private final MultiVersionConcurrencyControl mvcc;<a name="line.696"></a> |
| <span class="sourceLineNo">697</span><a name="line.697"></a> |
| <span class="sourceLineNo">698</span> // Coprocessor host<a name="line.698"></a> |
| <span class="sourceLineNo">699</span> private RegionCoprocessorHost coprocessorHost;<a name="line.699"></a> |
| <span class="sourceLineNo">700</span><a name="line.700"></a> |
| <span class="sourceLineNo">701</span> private TableDescriptor htableDescriptor = null;<a name="line.701"></a> |
| <span class="sourceLineNo">702</span> private RegionSplitPolicy splitPolicy;<a name="line.702"></a> |
| <span class="sourceLineNo">703</span> private FlushPolicy flushPolicy;<a name="line.703"></a> |
| <span class="sourceLineNo">704</span><a name="line.704"></a> |
| <span class="sourceLineNo">705</span> private final MetricsRegion metricsRegion;<a name="line.705"></a> |
| <span class="sourceLineNo">706</span> private final MetricsRegionWrapperImpl metricsRegionWrapper;<a name="line.706"></a> |
| <span class="sourceLineNo">707</span> private final Durability regionDurability;<a name="line.707"></a> |
| <span class="sourceLineNo">708</span> private final boolean regionStatsEnabled;<a name="line.708"></a> |
| <span class="sourceLineNo">709</span> // Stores the replication scope of the various column families of the table<a name="line.709"></a> |
| <span class="sourceLineNo">710</span> // that has non-default scope<a name="line.710"></a> |
| <span class="sourceLineNo">711</span> private final NavigableMap<byte[], Integer> replicationScope = new TreeMap<>(<a name="line.711"></a> |
| <span class="sourceLineNo">712</span> Bytes.BYTES_COMPARATOR);<a name="line.712"></a> |
| <span class="sourceLineNo">713</span><a name="line.713"></a> |
| <span class="sourceLineNo">714</span> private final StoreHotnessProtector storeHotnessProtector;<a name="line.714"></a> |
| <span class="sourceLineNo">715</span><a name="line.715"></a> |
| <span class="sourceLineNo">716</span> /**<a name="line.716"></a> |
| <span class="sourceLineNo">717</span> * HRegion constructor. This constructor should only be used for testing and<a name="line.717"></a> |
| <span class="sourceLineNo">718</span> * extensions. Instances of HRegion should be instantiated with the<a name="line.718"></a> |
| <span class="sourceLineNo">719</span> * {@link HRegion#createHRegion} or {@link HRegion#openHRegion} method.<a name="line.719"></a> |
| <span class="sourceLineNo">720</span> *<a name="line.720"></a> |
| <span class="sourceLineNo">721</span> * @param tableDir qualified path of directory where region should be located,<a name="line.721"></a> |
| <span class="sourceLineNo">722</span> * usually the table directory.<a name="line.722"></a> |
| <span class="sourceLineNo">723</span> * @param wal The WAL is the outbound log for any updates to the HRegion<a name="line.723"></a> |
| <span class="sourceLineNo">724</span> * The wal file is a logfile from the previous execution that's<a name="line.724"></a> |
| <span class="sourceLineNo">725</span> * custom-computed for this HRegion. The HRegionServer computes and sorts the<a name="line.725"></a> |
| <span class="sourceLineNo">726</span> * appropriate wal info for this HRegion. If there is a previous wal file<a name="line.726"></a> |
| <span class="sourceLineNo">727</span> * (implying that the HRegion has been written-to before), then read it from<a name="line.727"></a> |
| <span class="sourceLineNo">728</span> * the supplied path.<a name="line.728"></a> |
| <span class="sourceLineNo">729</span> * @param fs is the filesystem.<a name="line.729"></a> |
| <span class="sourceLineNo">730</span> * @param confParam is global configuration settings.<a name="line.730"></a> |
| <span class="sourceLineNo">731</span> * @param regionInfo - RegionInfo that describes the region<a name="line.731"></a> |
| <span class="sourceLineNo">732</span> * is new), then read them from the supplied path.<a name="line.732"></a> |
| <span class="sourceLineNo">733</span> * @param htd the table descriptor<a name="line.733"></a> |
| <span class="sourceLineNo">734</span> * @param rsServices reference to {@link RegionServerServices} or null<a name="line.734"></a> |
| <span class="sourceLineNo">735</span> * @deprecated Use other constructors.<a name="line.735"></a> |
| <span class="sourceLineNo">736</span> */<a name="line.736"></a> |
| <span class="sourceLineNo">737</span> @Deprecated<a name="line.737"></a> |
| <span class="sourceLineNo">738</span> public HRegion(final Path tableDir, final WAL wal, final FileSystem fs,<a name="line.738"></a> |
| <span class="sourceLineNo">739</span> final Configuration confParam, final RegionInfo regionInfo,<a name="line.739"></a> |
| <span class="sourceLineNo">740</span> final TableDescriptor htd, final RegionServerServices rsServices) {<a name="line.740"></a> |
| <span class="sourceLineNo">741</span> this(new HRegionFileSystem(confParam, fs, tableDir, regionInfo),<a name="line.741"></a> |
| <span class="sourceLineNo">742</span> wal, confParam, htd, rsServices);<a name="line.742"></a> |
| <span class="sourceLineNo">743</span> }<a name="line.743"></a> |
| <span class="sourceLineNo">744</span><a name="line.744"></a> |
| <span class="sourceLineNo">745</span> /**<a name="line.745"></a> |
| <span class="sourceLineNo">746</span> * HRegion constructor. This constructor should only be used for testing and<a name="line.746"></a> |
| <span class="sourceLineNo">747</span> * extensions. Instances of HRegion should be instantiated with the<a name="line.747"></a> |
| <span class="sourceLineNo">748</span> * {@link HRegion#createHRegion} or {@link HRegion#openHRegion} method.<a name="line.748"></a> |
| <span class="sourceLineNo">749</span> *<a name="line.749"></a> |
| <span class="sourceLineNo">750</span> * @param fs is the filesystem.<a name="line.750"></a> |
| <span class="sourceLineNo">751</span> * @param wal The WAL is the outbound log for any updates to the HRegion<a name="line.751"></a> |
| <span class="sourceLineNo">752</span> * The wal file is a logfile from the previous execution that's<a name="line.752"></a> |
| <span class="sourceLineNo">753</span> * custom-computed for this HRegion. The HRegionServer computes and sorts the<a name="line.753"></a> |
| <span class="sourceLineNo">754</span> * appropriate wal info for this HRegion. If there is a previous wal file<a name="line.754"></a> |
| <span class="sourceLineNo">755</span> * (implying that the HRegion has been written-to before), then read it from<a name="line.755"></a> |
| <span class="sourceLineNo">756</span> * the supplied path.<a name="line.756"></a> |
| <span class="sourceLineNo">757</span> * @param confParam is global configuration settings.<a name="line.757"></a> |
| <span class="sourceLineNo">758</span> * @param htd the table descriptor<a name="line.758"></a> |
| <span class="sourceLineNo">759</span> * @param rsServices reference to {@link RegionServerServices} or null<a name="line.759"></a> |
| <span class="sourceLineNo">760</span> */<a name="line.760"></a> |
| <span class="sourceLineNo">761</span> public HRegion(final HRegionFileSystem fs, final WAL wal, final Configuration confParam,<a name="line.761"></a> |
| <span class="sourceLineNo">762</span> final TableDescriptor htd, final RegionServerServices rsServices) {<a name="line.762"></a> |
| <span class="sourceLineNo">763</span> if (htd == null) {<a name="line.763"></a> |
| <span class="sourceLineNo">764</span> throw new IllegalArgumentException("Need table descriptor");<a name="line.764"></a> |
| <span class="sourceLineNo">765</span> }<a name="line.765"></a> |
| <span class="sourceLineNo">766</span><a name="line.766"></a> |
| <span class="sourceLineNo">767</span> if (confParam instanceof CompoundConfiguration) {<a name="line.767"></a> |
| <span class="sourceLineNo">768</span> throw new IllegalArgumentException("Need original base configuration");<a name="line.768"></a> |
| <span class="sourceLineNo">769</span> }<a name="line.769"></a> |
| <span class="sourceLineNo">770</span><a name="line.770"></a> |
| <span class="sourceLineNo">771</span> this.wal = wal;<a name="line.771"></a> |
| <span class="sourceLineNo">772</span> this.fs = fs;<a name="line.772"></a> |
| <span class="sourceLineNo">773</span> this.mvcc = new MultiVersionConcurrencyControl(getRegionInfo().getShortNameToLog());<a name="line.773"></a> |
| <span class="sourceLineNo">774</span><a name="line.774"></a> |
| <span class="sourceLineNo">775</span> // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor<a name="line.775"></a> |
| <span class="sourceLineNo">776</span> this.baseConf = confParam;<a name="line.776"></a> |
| <span class="sourceLineNo">777</span> this.conf = new CompoundConfiguration().add(confParam).addBytesMap(htd.getValues());<a name="line.777"></a> |
| <span class="sourceLineNo">778</span> this.cellComparator = htd.isMetaTable() ||<a name="line.778"></a> |
| <span class="sourceLineNo">779</span> conf.getBoolean(USE_META_CELL_COMPARATOR, DEFAULT_USE_META_CELL_COMPARATOR) ?<a name="line.779"></a> |
| <span class="sourceLineNo">780</span> MetaCellComparator.META_COMPARATOR : CellComparatorImpl.COMPARATOR;<a name="line.780"></a> |
| <span class="sourceLineNo">781</span> this.lock = new ReentrantReadWriteLock(conf.getBoolean(FAIR_REENTRANT_CLOSE_LOCK,<a name="line.781"></a> |
| <span class="sourceLineNo">782</span> DEFAULT_FAIR_REENTRANT_CLOSE_LOCK));<a name="line.782"></a> |
| <span class="sourceLineNo">783</span> this.regionLockHolders = new ConcurrentHashMap<>();<a name="line.783"></a> |
| <span class="sourceLineNo">784</span> this.flushCheckInterval = conf.getInt(MEMSTORE_PERIODIC_FLUSH_INTERVAL,<a name="line.784"></a> |
| <span class="sourceLineNo">785</span> DEFAULT_CACHE_FLUSH_INTERVAL);<a name="line.785"></a> |
| <span class="sourceLineNo">786</span> this.flushPerChanges = conf.getLong(MEMSTORE_FLUSH_PER_CHANGES, DEFAULT_FLUSH_PER_CHANGES);<a name="line.786"></a> |
| <span class="sourceLineNo">787</span> if (this.flushPerChanges > MAX_FLUSH_PER_CHANGES) {<a name="line.787"></a> |
| <span class="sourceLineNo">788</span> throw new IllegalArgumentException(MEMSTORE_FLUSH_PER_CHANGES + " can not exceed "<a name="line.788"></a> |
| <span class="sourceLineNo">789</span> + MAX_FLUSH_PER_CHANGES);<a name="line.789"></a> |
| <span class="sourceLineNo">790</span> }<a name="line.790"></a> |
| <span class="sourceLineNo">791</span> int tmpRowLockDuration = conf.getInt("hbase.rowlock.wait.duration",<a name="line.791"></a> |
| <span class="sourceLineNo">792</span> DEFAULT_ROWLOCK_WAIT_DURATION);<a name="line.792"></a> |
| <span class="sourceLineNo">793</span> if (tmpRowLockDuration <= 0) {<a name="line.793"></a> |
| <span class="sourceLineNo">794</span> LOG.info("Found hbase.rowlock.wait.duration set to {}. values <= 0 will cause all row " +<a name="line.794"></a> |
| <span class="sourceLineNo">795</span> "locking to fail. Treating it as 1ms to avoid region failure.", tmpRowLockDuration);<a name="line.795"></a> |
| <span class="sourceLineNo">796</span> tmpRowLockDuration = 1;<a name="line.796"></a> |
| <span class="sourceLineNo">797</span> }<a name="line.797"></a> |
| <span class="sourceLineNo">798</span> this.rowLockWaitDuration = tmpRowLockDuration;<a name="line.798"></a> |
| <span class="sourceLineNo">799</span><a name="line.799"></a> |
| <span class="sourceLineNo">800</span> this.isLoadingCfsOnDemandDefault = conf.getBoolean(LOAD_CFS_ON_DEMAND_CONFIG_KEY, true);<a name="line.800"></a> |
| <span class="sourceLineNo">801</span> this.htableDescriptor = htd;<a name="line.801"></a> |
| <span class="sourceLineNo">802</span> Set<byte[]> families = this.htableDescriptor.getColumnFamilyNames();<a name="line.802"></a> |
| <span class="sourceLineNo">803</span> for (byte[] family : families) {<a name="line.803"></a> |
| <span class="sourceLineNo">804</span> if (!replicationScope.containsKey(family)) {<a name="line.804"></a> |
| <span class="sourceLineNo">805</span> int scope = htd.getColumnFamily(family).getScope();<a name="line.805"></a> |
| <span class="sourceLineNo">806</span> // Only store those families that has NON-DEFAULT scope<a name="line.806"></a> |
| <span class="sourceLineNo">807</span> if (scope != REPLICATION_SCOPE_LOCAL) {<a name="line.807"></a> |
| <span class="sourceLineNo">808</span> // Do a copy before storing it here.<a name="line.808"></a> |
| <span class="sourceLineNo">809</span> replicationScope.put(Bytes.copy(family), scope);<a name="line.809"></a> |
| <span class="sourceLineNo">810</span> }<a name="line.810"></a> |
| <span class="sourceLineNo">811</span> }<a name="line.811"></a> |
| <span class="sourceLineNo">812</span> }<a name="line.812"></a> |
| <span class="sourceLineNo">813</span><a name="line.813"></a> |
| <span class="sourceLineNo">814</span> this.rsServices = rsServices;<a name="line.814"></a> |
| <span class="sourceLineNo">815</span> if (this.rsServices != null) {<a name="line.815"></a> |
| <span class="sourceLineNo">816</span> this.blockCache = rsServices.getBlockCache().orElse(null);<a name="line.816"></a> |
| <span class="sourceLineNo">817</span> this.mobFileCache = rsServices.getMobFileCache().orElse(null);<a name="line.817"></a> |
| <span class="sourceLineNo">818</span> }<a name="line.818"></a> |
| <span class="sourceLineNo">819</span> this.regionServicesForStores = new RegionServicesForStores(this, rsServices);<a name="line.819"></a> |
| <span class="sourceLineNo">820</span><a name="line.820"></a> |
| <span class="sourceLineNo">821</span> setHTableSpecificConf();<a name="line.821"></a> |
| <span class="sourceLineNo">822</span> this.scannerReadPoints = new ConcurrentHashMap<>();<a name="line.822"></a> |
| <span class="sourceLineNo">823</span><a name="line.823"></a> |
| <span class="sourceLineNo">824</span> this.busyWaitDuration = conf.getLong(<a name="line.824"></a> |
| <span class="sourceLineNo">825</span> "hbase.busy.wait.duration", DEFAULT_BUSY_WAIT_DURATION);<a name="line.825"></a> |
| <span class="sourceLineNo">826</span> this.maxBusyWaitMultiplier = conf.getInt("hbase.busy.wait.multiplier.max", 2);<a name="line.826"></a> |
| <span class="sourceLineNo">827</span> if (busyWaitDuration * maxBusyWaitMultiplier <= 0L) {<a name="line.827"></a> |
| <span class="sourceLineNo">828</span> throw new IllegalArgumentException("Invalid hbase.busy.wait.duration ("<a name="line.828"></a> |
| <span class="sourceLineNo">829</span> + busyWaitDuration + ") or hbase.busy.wait.multiplier.max ("<a name="line.829"></a> |
| <span class="sourceLineNo">830</span> + maxBusyWaitMultiplier + "). Their product should be positive");<a name="line.830"></a> |
| <span class="sourceLineNo">831</span> }<a name="line.831"></a> |
| <span class="sourceLineNo">832</span> this.maxBusyWaitDuration = conf.getLong("hbase.ipc.client.call.purge.timeout",<a name="line.832"></a> |
| <span class="sourceLineNo">833</span> 2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);<a name="line.833"></a> |
| <span class="sourceLineNo">834</span><a name="line.834"></a> |
| <span class="sourceLineNo">835</span> /*<a name="line.835"></a> |
| <span class="sourceLineNo">836</span> * timestamp.slop provides a server-side constraint on the timestamp. This<a name="line.836"></a> |
| <span class="sourceLineNo">837</span> * assumes that you base your TS around currentTimeMillis(). In this case,<a name="line.837"></a> |
| <span class="sourceLineNo">838</span> * throw an error to the user if the user-specified TS is newer than now +<a name="line.838"></a> |
| <span class="sourceLineNo">839</span> * slop. LATEST_TIMESTAMP == don't use this functionality<a name="line.839"></a> |
| <span class="sourceLineNo">840</span> */<a name="line.840"></a> |
| <span class="sourceLineNo">841</span> this.timestampSlop = conf.getLong(<a name="line.841"></a> |
| <span class="sourceLineNo">842</span> "hbase.hregion.keyvalue.timestamp.slop.millisecs",<a name="line.842"></a> |
| <span class="sourceLineNo">843</span> HConstants.LATEST_TIMESTAMP);<a name="line.843"></a> |
| <span class="sourceLineNo">844</span><a name="line.844"></a> |
| <span class="sourceLineNo">845</span> /**<a name="line.845"></a> |
| <span class="sourceLineNo">846</span> * Timeout for the process time in processRowsWithLocks().<a name="line.846"></a> |
| <span class="sourceLineNo">847</span> * Use -1 to switch off time bound.<a name="line.847"></a> |
| <span class="sourceLineNo">848</span> */<a name="line.848"></a> |
| <span class="sourceLineNo">849</span> this.rowProcessorTimeout = conf.getLong(<a name="line.849"></a> |
| <span class="sourceLineNo">850</span> "hbase.hregion.row.processor.timeout", DEFAULT_ROW_PROCESSOR_TIMEOUT);<a name="line.850"></a> |
| <span class="sourceLineNo">851</span><a name="line.851"></a> |
| <span class="sourceLineNo">852</span> this.storeHotnessProtector = new StoreHotnessProtector(this, conf);<a name="line.852"></a> |
| <span class="sourceLineNo">853</span><a name="line.853"></a> |
| <span class="sourceLineNo">854</span> boolean forceSync = conf.getBoolean(WAL_HSYNC_CONF_KEY, DEFAULT_WAL_HSYNC);<a name="line.854"></a> |
| <span class="sourceLineNo">855</span> /**<a name="line.855"></a> |
| <span class="sourceLineNo">856</span> * This is the global default value for durability. All tables/mutations not defining a<a name="line.856"></a> |
| <span class="sourceLineNo">857</span> * durability or using USE_DEFAULT will default to this value.<a name="line.857"></a> |
| <span class="sourceLineNo">858</span> */<a name="line.858"></a> |
| <span class="sourceLineNo">859</span> Durability defaultDurability = forceSync ? Durability.FSYNC_WAL : Durability.SYNC_WAL;<a name="line.859"></a> |
| <span class="sourceLineNo">860</span> this.regionDurability =<a name="line.860"></a> |
| <span class="sourceLineNo">861</span> this.htableDescriptor.getDurability() == Durability.USE_DEFAULT ? defaultDurability :<a name="line.861"></a> |
| <span class="sourceLineNo">862</span> this.htableDescriptor.getDurability();<a name="line.862"></a> |
| <span class="sourceLineNo">863</span><a name="line.863"></a> |
| <span class="sourceLineNo">864</span> decorateRegionConfiguration(conf);<a name="line.864"></a> |
| <span class="sourceLineNo">865</span> if (rsServices != null) {<a name="line.865"></a> |
| <span class="sourceLineNo">866</span> this.rsAccounting = this.rsServices.getRegionServerAccounting();<a name="line.866"></a> |
| <span class="sourceLineNo">867</span> // don't initialize coprocessors if not running within a regionserver<a name="line.867"></a> |
| <span class="sourceLineNo">868</span> // TODO: revisit if coprocessors should load in other cases<a name="line.868"></a> |
| <span class="sourceLineNo">869</span> this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);<a name="line.869"></a> |
| <span class="sourceLineNo">870</span> this.metricsRegionWrapper = new MetricsRegionWrapperImpl(this);<a name="line.870"></a> |
| <span class="sourceLineNo">871</span> this.metricsRegion = new MetricsRegion(this.metricsRegionWrapper, conf);<a name="line.871"></a> |
| <span class="sourceLineNo">872</span> } else {<a name="line.872"></a> |
| <span class="sourceLineNo">873</span> this.metricsRegionWrapper = null;<a name="line.873"></a> |
| <span class="sourceLineNo">874</span> this.metricsRegion = null;<a name="line.874"></a> |
| <span class="sourceLineNo">875</span> }<a name="line.875"></a> |
| <span class="sourceLineNo">876</span> if (LOG.isDebugEnabled()) {<a name="line.876"></a> |
| <span class="sourceLineNo">877</span> // Write out region name, its encoded name and storeHotnessProtector as string.<a name="line.877"></a> |
| <span class="sourceLineNo">878</span> LOG.debug("Instantiated " + this +"; "+ storeHotnessProtector.toString());<a name="line.878"></a> |
| <span class="sourceLineNo">879</span> }<a name="line.879"></a> |
| <span class="sourceLineNo">880</span><a name="line.880"></a> |
| <span class="sourceLineNo">881</span> configurationManager = null;<a name="line.881"></a> |
| <span class="sourceLineNo">882</span><a name="line.882"></a> |
| <span class="sourceLineNo">883</span> // disable stats tracking system tables, but check the config for everything else<a name="line.883"></a> |
| <span class="sourceLineNo">884</span> this.regionStatsEnabled = htd.getTableName().getNamespaceAsString().equals(<a name="line.884"></a> |
| <span class="sourceLineNo">885</span> NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR) ?<a name="line.885"></a> |
| <span class="sourceLineNo">886</span> false :<a name="line.886"></a> |
| <span class="sourceLineNo">887</span> conf.getBoolean(HConstants.ENABLE_CLIENT_BACKPRESSURE,<a name="line.887"></a> |
| <span class="sourceLineNo">888</span> HConstants.DEFAULT_ENABLE_CLIENT_BACKPRESSURE);<a name="line.888"></a> |
| <span class="sourceLineNo">889</span><a name="line.889"></a> |
| <span class="sourceLineNo">890</span> this.maxCellSize = conf.getLong(HBASE_MAX_CELL_SIZE_KEY, DEFAULT_MAX_CELL_SIZE);<a name="line.890"></a> |
| <span class="sourceLineNo">891</span> this.miniBatchSize = conf.getInt(HBASE_REGIONSERVER_MINIBATCH_SIZE,<a name="line.891"></a> |
| <span class="sourceLineNo">892</span> DEFAULT_HBASE_REGIONSERVER_MINIBATCH_SIZE);<a name="line.892"></a> |
| <span class="sourceLineNo">893</span><a name="line.893"></a> |
| <span class="sourceLineNo">894</span> // recover the metrics of read and write requests count if they were retained<a name="line.894"></a> |
| <span class="sourceLineNo">895</span> if (rsServices != null && rsServices.getRegionServerAccounting() != null) {<a name="line.895"></a> |
| <span class="sourceLineNo">896</span> Pair<Long, Long> retainedRWRequestsCnt = rsServices.getRegionServerAccounting()<a name="line.896"></a> |
| <span class="sourceLineNo">897</span> .getRetainedRegionRWRequestsCnt().get(getRegionInfo().getEncodedName());<a name="line.897"></a> |
| <span class="sourceLineNo">898</span> if (retainedRWRequestsCnt != null) {<a name="line.898"></a> |
| <span class="sourceLineNo">899</span> this.addReadRequestsCount(retainedRWRequestsCnt.getFirst());<a name="line.899"></a> |
| <span class="sourceLineNo">900</span> this.addWriteRequestsCount(retainedRWRequestsCnt.getSecond());<a name="line.900"></a> |
| <span class="sourceLineNo">901</span> // remove them since won't use again<a name="line.901"></a> |
| <span class="sourceLineNo">902</span> rsServices.getRegionServerAccounting().getRetainedRegionRWRequestsCnt()<a name="line.902"></a> |
| <span class="sourceLineNo">903</span> .remove(getRegionInfo().getEncodedName());<a name="line.903"></a> |
| <span class="sourceLineNo">904</span> }<a name="line.904"></a> |
| <span class="sourceLineNo">905</span> }<a name="line.905"></a> |
| <span class="sourceLineNo">906</span> }<a name="line.906"></a> |
| <span class="sourceLineNo">907</span><a name="line.907"></a> |
| <span class="sourceLineNo">908</span> private void setHTableSpecificConf() {<a name="line.908"></a> |
| <span class="sourceLineNo">909</span> if (this.htableDescriptor == null) {<a name="line.909"></a> |
| <span class="sourceLineNo">910</span> return;<a name="line.910"></a> |
| <span class="sourceLineNo">911</span> }<a name="line.911"></a> |
| <span class="sourceLineNo">912</span> long flushSize = this.htableDescriptor.getMemStoreFlushSize();<a name="line.912"></a> |
| <span class="sourceLineNo">913</span><a name="line.913"></a> |
| <span class="sourceLineNo">914</span> if (flushSize <= 0) {<a name="line.914"></a> |
| <span class="sourceLineNo">915</span> flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,<a name="line.915"></a> |
| <span class="sourceLineNo">916</span> TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE);<a name="line.916"></a> |
| <span class="sourceLineNo">917</span> }<a name="line.917"></a> |
| <span class="sourceLineNo">918</span> this.memstoreFlushSize = flushSize;<a name="line.918"></a> |
| <span class="sourceLineNo">919</span> long mult = conf.getLong(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER,<a name="line.919"></a> |
| <span class="sourceLineNo">920</span> HConstants.DEFAULT_HREGION_MEMSTORE_BLOCK_MULTIPLIER);<a name="line.920"></a> |
| <span class="sourceLineNo">921</span> this.blockingMemStoreSize = this.memstoreFlushSize * mult;<a name="line.921"></a> |
| <span class="sourceLineNo">922</span> }<a name="line.922"></a> |
| <span class="sourceLineNo">923</span><a name="line.923"></a> |
| <span class="sourceLineNo">924</span> /**<a name="line.924"></a> |
| <span class="sourceLineNo">925</span> * Initialize this region.<a name="line.925"></a> |
| <span class="sourceLineNo">926</span> * Used only by tests and SplitTransaction to reopen the region.<a name="line.926"></a> |
| <span class="sourceLineNo">927</span> * You should use createHRegion() or openHRegion()<a name="line.927"></a> |
| <span class="sourceLineNo">928</span> * @return What the next sequence (edit) id should be.<a name="line.928"></a> |
| <span class="sourceLineNo">929</span> * @throws IOException e<a name="line.929"></a> |
| <span class="sourceLineNo">930</span> * @deprecated use HRegion.createHRegion() or HRegion.openHRegion()<a name="line.930"></a> |
| <span class="sourceLineNo">931</span> */<a name="line.931"></a> |
| <span class="sourceLineNo">932</span> @Deprecated<a name="line.932"></a> |
| <span class="sourceLineNo">933</span> public long initialize() throws IOException {<a name="line.933"></a> |
| <span class="sourceLineNo">934</span> return initialize(null);<a name="line.934"></a> |
| <span class="sourceLineNo">935</span> }<a name="line.935"></a> |
| <span class="sourceLineNo">936</span><a name="line.936"></a> |
| <span class="sourceLineNo">937</span> /**<a name="line.937"></a> |
| <span class="sourceLineNo">938</span> * Initialize this region.<a name="line.938"></a> |
| <span class="sourceLineNo">939</span> *<a name="line.939"></a> |
| <span class="sourceLineNo">940</span> * @param reporter Tickle every so often if initialize is taking a while.<a name="line.940"></a> |
| <span class="sourceLineNo">941</span> * @return What the next sequence (edit) id should be.<a name="line.941"></a> |
| <span class="sourceLineNo">942</span> * @throws IOException e<a name="line.942"></a> |
| <span class="sourceLineNo">943</span> */<a name="line.943"></a> |
| <span class="sourceLineNo">944</span> long initialize(final CancelableProgressable reporter) throws IOException {<a name="line.944"></a> |
| <span class="sourceLineNo">945</span><a name="line.945"></a> |
| <span class="sourceLineNo">946</span> //Refuse to open the region if there is no column family in the table<a name="line.946"></a> |
| <span class="sourceLineNo">947</span> if (htableDescriptor.getColumnFamilyCount() == 0) {<a name="line.947"></a> |
| <span class="sourceLineNo">948</span> throw new DoNotRetryIOException("Table " + htableDescriptor.getTableName().getNameAsString()+<a name="line.948"></a> |
| <span class="sourceLineNo">949</span> " should have at least one column family.");<a name="line.949"></a> |
| <span class="sourceLineNo">950</span> }<a name="line.950"></a> |
| <span class="sourceLineNo">951</span><a name="line.951"></a> |
| <span class="sourceLineNo">952</span> MonitoredTask status = TaskMonitor.get().createStatus("Initializing region " + this);<a name="line.952"></a> |
| <span class="sourceLineNo">953</span> status.enableStatusJournal(true);<a name="line.953"></a> |
| <span class="sourceLineNo">954</span> long nextSeqId = -1;<a name="line.954"></a> |
| <span class="sourceLineNo">955</span> try {<a name="line.955"></a> |
| <span class="sourceLineNo">956</span> nextSeqId = initializeRegionInternals(reporter, status);<a name="line.956"></a> |
| <span class="sourceLineNo">957</span> return nextSeqId;<a name="line.957"></a> |
| <span class="sourceLineNo">958</span> } catch (IOException e) {<a name="line.958"></a> |
| <span class="sourceLineNo">959</span> LOG.warn("Failed initialize of region= {}, starting to roll back memstore",<a name="line.959"></a> |
| <span class="sourceLineNo">960</span> getRegionInfo().getRegionNameAsString(), e);<a name="line.960"></a> |
| <span class="sourceLineNo">961</span> // global memstore size will be decreased when dropping memstore<a name="line.961"></a> |
| <span class="sourceLineNo">962</span> try {<a name="line.962"></a> |
| <span class="sourceLineNo">963</span> //drop the memory used by memstore if open region fails<a name="line.963"></a> |
| <span class="sourceLineNo">964</span> dropMemStoreContents();<a name="line.964"></a> |
| <span class="sourceLineNo">965</span> } catch (IOException ioE) {<a name="line.965"></a> |
| <span class="sourceLineNo">966</span> if (conf.getBoolean(MemStoreLAB.USEMSLAB_KEY, MemStoreLAB.USEMSLAB_DEFAULT)) {<a name="line.966"></a> |
| <span class="sourceLineNo">967</span> LOG.warn("Failed drop memstore of region= {}, "<a name="line.967"></a> |
| <span class="sourceLineNo">968</span> + "some chunks may not released forever since MSLAB is enabled",<a name="line.968"></a> |
| <span class="sourceLineNo">969</span> getRegionInfo().getRegionNameAsString());<a name="line.969"></a> |
| <span class="sourceLineNo">970</span> }<a name="line.970"></a> |
| <span class="sourceLineNo">971</span><a name="line.971"></a> |
| <span class="sourceLineNo">972</span> }<a name="line.972"></a> |
| <span class="sourceLineNo">973</span> throw e;<a name="line.973"></a> |
| <span class="sourceLineNo">974</span> } finally {<a name="line.974"></a> |
| <span class="sourceLineNo">975</span> // nextSeqid will be -1 if the initialization fails.<a name="line.975"></a> |
| <span class="sourceLineNo">976</span> // At least it will be 0 otherwise.<a name="line.976"></a> |
| <span class="sourceLineNo">977</span> if (nextSeqId == -1) {<a name="line.977"></a> |
| <span class="sourceLineNo">978</span> status.abort("Exception during region " + getRegionInfo().getRegionNameAsString() +<a name="line.978"></a> |
| <span class="sourceLineNo">979</span> " initialization.");<a name="line.979"></a> |
| <span class="sourceLineNo">980</span> }<a name="line.980"></a> |
| <span class="sourceLineNo">981</span> if (LOG.isDebugEnabled()) {<a name="line.981"></a> |
| <span class="sourceLineNo">982</span> LOG.debug("Region open journal for {}:\n{}", this.getRegionInfo().getEncodedName(),<a name="line.982"></a> |
| <span class="sourceLineNo">983</span> status.prettyPrintJournal());<a name="line.983"></a> |
| <span class="sourceLineNo">984</span> }<a name="line.984"></a> |
| <span class="sourceLineNo">985</span> status.cleanup();<a name="line.985"></a> |
| <span class="sourceLineNo">986</span> }<a name="line.986"></a> |
| <span class="sourceLineNo">987</span> }<a name="line.987"></a> |
| <span class="sourceLineNo">988</span><a name="line.988"></a> |
| <span class="sourceLineNo">989</span> private long initializeRegionInternals(final CancelableProgressable reporter,<a name="line.989"></a> |
| <span class="sourceLineNo">990</span> final MonitoredTask status) throws IOException {<a name="line.990"></a> |
| <span class="sourceLineNo">991</span> if (coprocessorHost != null) {<a name="line.991"></a> |
| <span class="sourceLineNo">992</span> status.setStatus("Running coprocessor pre-open hook");<a name="line.992"></a> |
| <span class="sourceLineNo">993</span> coprocessorHost.preOpen();<a name="line.993"></a> |
| <span class="sourceLineNo">994</span> }<a name="line.994"></a> |
| <span class="sourceLineNo">995</span><a name="line.995"></a> |
| <span class="sourceLineNo">996</span> // Write HRI to a file in case we need to recover hbase:meta<a name="line.996"></a> |
| <span class="sourceLineNo">997</span> // Only the primary replica should write .regioninfo<a name="line.997"></a> |
| <span class="sourceLineNo">998</span> if (this.getRegionInfo().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {<a name="line.998"></a> |
| <span class="sourceLineNo">999</span> status.setStatus("Writing region info on filesystem");<a name="line.999"></a> |
| <span class="sourceLineNo">1000</span> fs.checkRegionInfoOnFilesystem();<a name="line.1000"></a> |
| <span class="sourceLineNo">1001</span> }<a name="line.1001"></a> |
| <span class="sourceLineNo">1002</span><a name="line.1002"></a> |
| <span class="sourceLineNo">1003</span> // Initialize all the HStores<a name="line.1003"></a> |
| <span class="sourceLineNo">1004</span> status.setStatus("Initializing all the Stores");<a name="line.1004"></a> |
| <span class="sourceLineNo">1005</span> long maxSeqId = initializeStores(reporter, status);<a name="line.1005"></a> |
| <span class="sourceLineNo">1006</span> this.mvcc.advanceTo(maxSeqId);<a name="line.1006"></a> |
| <span class="sourceLineNo">1007</span> if (!isRestoredRegion && ServerRegionReplicaUtil.shouldReplayRecoveredEdits(this)) {<a name="line.1007"></a> |
| <span class="sourceLineNo">1008</span> Collection<HStore> stores = this.stores.values();<a name="line.1008"></a> |
| <span class="sourceLineNo">1009</span> try {<a name="line.1009"></a> |
| <span class="sourceLineNo">1010</span> // update the stores that we are replaying<a name="line.1010"></a> |
| <span class="sourceLineNo">1011</span> LOG.debug("replaying wal for " + this.getRegionInfo().getEncodedName());<a name="line.1011"></a> |
| <span class="sourceLineNo">1012</span> stores.forEach(HStore::startReplayingFromWAL);<a name="line.1012"></a> |
| <span class="sourceLineNo">1013</span> // Recover any edits if available.<a name="line.1013"></a> |
| <span class="sourceLineNo">1014</span> maxSeqId = Math.max(maxSeqId,<a name="line.1014"></a> |
| <span class="sourceLineNo">1015</span> replayRecoveredEditsIfAny(maxSeqIdInStores, reporter, status));<a name="line.1015"></a> |
| <span class="sourceLineNo">1016</span> // Recover any hfiles if available<a name="line.1016"></a> |
| <span class="sourceLineNo">1017</span> maxSeqId = Math.max(maxSeqId, loadRecoveredHFilesIfAny(stores));<a name="line.1017"></a> |
| <span class="sourceLineNo">1018</span> // Make sure mvcc is up to max.<a name="line.1018"></a> |
| <span class="sourceLineNo">1019</span> this.mvcc.advanceTo(maxSeqId);<a name="line.1019"></a> |
| <span class="sourceLineNo">1020</span> } finally {<a name="line.1020"></a> |
| <span class="sourceLineNo">1021</span> LOG.debug("stopping wal replay for " + this.getRegionInfo().getEncodedName());<a name="line.1021"></a> |
| <span class="sourceLineNo">1022</span> // update the stores that we are done replaying<a name="line.1022"></a> |
| <span class="sourceLineNo">1023</span> stores.forEach(HStore::stopReplayingFromWAL);<a name="line.1023"></a> |
| <span class="sourceLineNo">1024</span> }<a name="line.1024"></a> |
| <span class="sourceLineNo">1025</span> }<a name="line.1025"></a> |
| <span class="sourceLineNo">1026</span> this.lastReplayedOpenRegionSeqId = maxSeqId;<a name="line.1026"></a> |
| <span class="sourceLineNo">1027</span><a name="line.1027"></a> |
| <span class="sourceLineNo">1028</span> this.writestate.setReadOnly(ServerRegionReplicaUtil.isReadOnly(this));<a name="line.1028"></a> |
| <span class="sourceLineNo">1029</span> this.writestate.flushRequested = false;<a name="line.1029"></a> |
| <span class="sourceLineNo">1030</span> this.writestate.compacting.set(0);<a name="line.1030"></a> |
| <span class="sourceLineNo">1031</span><a name="line.1031"></a> |
| <span class="sourceLineNo">1032</span> if (this.writestate.writesEnabled) {<a name="line.1032"></a> |
| <span class="sourceLineNo">1033</span> LOG.debug("Cleaning up temporary data for " + this.getRegionInfo().getEncodedName());<a name="line.1033"></a> |
| <span class="sourceLineNo">1034</span> // Remove temporary data left over from old regions<a name="line.1034"></a> |
| <span class="sourceLineNo">1035</span> status.setStatus("Cleaning up temporary data from old regions");<a name="line.1035"></a> |
| <span class="sourceLineNo">1036</span> fs.cleanupTempDir();<a name="line.1036"></a> |
| <span class="sourceLineNo">1037</span> }<a name="line.1037"></a> |
| <span class="sourceLineNo">1038</span><a name="line.1038"></a> |
| <span class="sourceLineNo">1039</span> if (this.writestate.writesEnabled) {<a name="line.1039"></a> |
| <span class="sourceLineNo">1040</span> status.setStatus("Cleaning up detritus from prior splits");<a name="line.1040"></a> |
| <span class="sourceLineNo">1041</span> // Get rid of any splits or merges that were lost in-progress. Clean out<a name="line.1041"></a> |
| <span class="sourceLineNo">1042</span> // these directories here on open. We may be opening a region that was<a name="line.1042"></a> |
| <span class="sourceLineNo">1043</span> // being split but we crashed in the middle of it all.<a name="line.1043"></a> |
| <span class="sourceLineNo">1044</span> LOG.debug("Cleaning up detritus for " + this.getRegionInfo().getEncodedName());<a name="line.1044"></a> |
| <span class="sourceLineNo">1045</span> fs.cleanupAnySplitDetritus();<a name="line.1045"></a> |
| <span class="sourceLineNo">1046</span> fs.cleanupMergesDir();<a name="line.1046"></a> |
| <span class="sourceLineNo">1047</span> }<a name="line.1047"></a> |
| <span class="sourceLineNo">1048</span><a name="line.1048"></a> |
| <span class="sourceLineNo">1049</span> // Initialize split policy<a name="line.1049"></a> |
| <span class="sourceLineNo">1050</span> this.splitPolicy = RegionSplitPolicy.create(this, conf);<a name="line.1050"></a> |
| <span class="sourceLineNo">1051</span><a name="line.1051"></a> |
| <span class="sourceLineNo">1052</span> // Initialize flush policy<a name="line.1052"></a> |
| <span class="sourceLineNo">1053</span> this.flushPolicy = FlushPolicyFactory.create(this, conf);<a name="line.1053"></a> |
| <span class="sourceLineNo">1054</span><a name="line.1054"></a> |
| <span class="sourceLineNo">1055</span> long lastFlushTime = EnvironmentEdgeManager.currentTime();<a name="line.1055"></a> |
| <span class="sourceLineNo">1056</span> for (HStore store: stores.values()) {<a name="line.1056"></a> |
| <span class="sourceLineNo">1057</span> this.lastStoreFlushTimeMap.put(store, lastFlushTime);<a name="line.1057"></a> |
| <span class="sourceLineNo">1058</span> }<a name="line.1058"></a> |
| <span class="sourceLineNo">1059</span><a name="line.1059"></a> |
| <span class="sourceLineNo">1060</span> // Use maximum of log sequenceid or that which was found in stores<a name="line.1060"></a> |
| <span class="sourceLineNo">1061</span> // (particularly if no recovered edits, seqid will be -1).<a name="line.1061"></a> |
| <span class="sourceLineNo">1062</span> long nextSeqId = maxSeqId + 1;<a name="line.1062"></a> |
| <span class="sourceLineNo">1063</span> if (!isRestoredRegion) {<a name="line.1063"></a> |
| <span class="sourceLineNo">1064</span> // always get openSeqNum from the default replica, even if we are secondary replicas<a name="line.1064"></a> |
| <span class="sourceLineNo">1065</span> long maxSeqIdFromFile = WALSplitUtil.getMaxRegionSequenceId(conf,<a name="line.1065"></a> |
| <span class="sourceLineNo">1066</span> RegionReplicaUtil.getRegionInfoForDefaultReplica(getRegionInfo()), this::getFilesystem,<a name="line.1066"></a> |
| <span class="sourceLineNo">1067</span> this::getWalFileSystem);<a name="line.1067"></a> |
| <span class="sourceLineNo">1068</span> nextSeqId = Math.max(maxSeqId, maxSeqIdFromFile) + 1;<a name="line.1068"></a> |
| <span class="sourceLineNo">1069</span> // The openSeqNum will always be increase even for read only region, as we rely on it to<a name="line.1069"></a> |
| <span class="sourceLineNo">1070</span> // determine whether a region has been successfully reopened, so here we always need to update<a name="line.1070"></a> |
| <span class="sourceLineNo">1071</span> // the max sequence id file.<a name="line.1071"></a> |
| <span class="sourceLineNo">1072</span> if (RegionReplicaUtil.isDefaultReplica(getRegionInfo())) {<a name="line.1072"></a> |
| <span class="sourceLineNo">1073</span> LOG.debug("writing seq id for {}", this.getRegionInfo().getEncodedName());<a name="line.1073"></a> |
| <span class="sourceLineNo">1074</span> WALSplitUtil.writeRegionSequenceIdFile(getWalFileSystem(), getWALRegionDir(),<a name="line.1074"></a> |
| <span class="sourceLineNo">1075</span> nextSeqId - 1);<a name="line.1075"></a> |
| <span class="sourceLineNo">1076</span> // This means we have replayed all the recovered edits and also written out the max sequence<a name="line.1076"></a> |
| <span class="sourceLineNo">1077</span> // id file, let's delete the wrong directories introduced in HBASE-20734, see HBASE-22617<a name="line.1077"></a> |
| <span class="sourceLineNo">1078</span> // for more details.<a name="line.1078"></a> |
| <span class="sourceLineNo">1079</span> Path wrongRegionWALDir = CommonFSUtils.getWrongWALRegionDir(conf,<a name="line.1079"></a> |
| <span class="sourceLineNo">1080</span> getRegionInfo().getTable(), getRegionInfo().getEncodedName());<a name="line.1080"></a> |
| <span class="sourceLineNo">1081</span> FileSystem walFs = getWalFileSystem();<a name="line.1081"></a> |
| <span class="sourceLineNo">1082</span> if (walFs.exists(wrongRegionWALDir)) {<a name="line.1082"></a> |
| <span class="sourceLineNo">1083</span> if (!walFs.delete(wrongRegionWALDir, true)) {<a name="line.1083"></a> |
| <span class="sourceLineNo">1084</span> LOG.debug("Failed to clean up wrong region WAL directory {}", wrongRegionWALDir);<a name="line.1084"></a> |
| <span class="sourceLineNo">1085</span> }<a name="line.1085"></a> |
| <span class="sourceLineNo">1086</span> }<a name="line.1086"></a> |
| <span class="sourceLineNo">1087</span> }<a name="line.1087"></a> |
| <span class="sourceLineNo">1088</span> }<a name="line.1088"></a> |
| <span class="sourceLineNo">1089</span><a name="line.1089"></a> |
| <span class="sourceLineNo">1090</span> LOG.info("Opened {}; next sequenceid={}", this.getRegionInfo().getShortNameToLog(), nextSeqId);<a name="line.1090"></a> |
| <span class="sourceLineNo">1091</span><a name="line.1091"></a> |
| <span class="sourceLineNo">1092</span> // A region can be reopened if failed a split; reset flags<a name="line.1092"></a> |
| <span class="sourceLineNo">1093</span> this.closing.set(false);<a name="line.1093"></a> |
| <span class="sourceLineNo">1094</span> this.closed.set(false);<a name="line.1094"></a> |
| <span class="sourceLineNo">1095</span><a name="line.1095"></a> |
| <span class="sourceLineNo">1096</span> if (coprocessorHost != null) {<a name="line.1096"></a> |
| <span class="sourceLineNo">1097</span> LOG.debug("Running coprocessor post-open hooks for " + this.getRegionInfo().getEncodedName());<a name="line.1097"></a> |
| <span class="sourceLineNo">1098</span> status.setStatus("Running coprocessor post-open hooks");<a name="line.1098"></a> |
| <span class="sourceLineNo">1099</span> coprocessorHost.postOpen();<a name="line.1099"></a> |
| <span class="sourceLineNo">1100</span> }<a name="line.1100"></a> |
| <span class="sourceLineNo">1101</span><a name="line.1101"></a> |
| <span class="sourceLineNo">1102</span> status.markComplete("Region opened successfully");<a name="line.1102"></a> |
| <span class="sourceLineNo">1103</span> return nextSeqId;<a name="line.1103"></a> |
| <span class="sourceLineNo">1104</span> }<a name="line.1104"></a> |
| <span class="sourceLineNo">1105</span><a name="line.1105"></a> |
| <span class="sourceLineNo">1106</span> /**<a name="line.1106"></a> |
| <span class="sourceLineNo">1107</span> * Open all Stores.<a name="line.1107"></a> |
| <span class="sourceLineNo">1108</span> * @param reporter<a name="line.1108"></a> |
| <span class="sourceLineNo">1109</span> * @param status<a name="line.1109"></a> |
| <span class="sourceLineNo">1110</span> * @return Highest sequenceId found out in a Store.<a name="line.1110"></a> |
| <span class="sourceLineNo">1111</span> * @throws IOException<a name="line.1111"></a> |
| <span class="sourceLineNo">1112</span> */<a name="line.1112"></a> |
| <span class="sourceLineNo">1113</span> private long initializeStores(CancelableProgressable reporter, MonitoredTask status)<a name="line.1113"></a> |
| <span class="sourceLineNo">1114</span> throws IOException {<a name="line.1114"></a> |
| <span class="sourceLineNo">1115</span> return initializeStores(reporter, status, false);<a name="line.1115"></a> |
| <span class="sourceLineNo">1116</span> }<a name="line.1116"></a> |
| <span class="sourceLineNo">1117</span><a name="line.1117"></a> |
| <span class="sourceLineNo">1118</span> private long initializeStores(CancelableProgressable reporter, MonitoredTask status,<a name="line.1118"></a> |
| <span class="sourceLineNo">1119</span> boolean warmup) throws IOException {<a name="line.1119"></a> |
| <span class="sourceLineNo">1120</span> // Load in all the HStores.<a name="line.1120"></a> |
| <span class="sourceLineNo">1121</span> long maxSeqId = -1;<a name="line.1121"></a> |
| <span class="sourceLineNo">1122</span> // initialized to -1 so that we pick up MemstoreTS from column families<a name="line.1122"></a> |
| <span class="sourceLineNo">1123</span> long maxMemstoreTS = -1;<a name="line.1123"></a> |
| <span class="sourceLineNo">1124</span><a name="line.1124"></a> |
| <span class="sourceLineNo">1125</span> if (htableDescriptor.getColumnFamilyCount() != 0) {<a name="line.1125"></a> |
| <span class="sourceLineNo">1126</span> // initialize the thread pool for opening stores in parallel.<a name="line.1126"></a> |
| <span class="sourceLineNo">1127</span> ThreadPoolExecutor storeOpenerThreadPool =<a name="line.1127"></a> |
| <span class="sourceLineNo">1128</span> getStoreOpenAndCloseThreadPool("StoreOpener-" + this.getRegionInfo().getShortNameToLog());<a name="line.1128"></a> |
| <span class="sourceLineNo">1129</span> CompletionService<HStore> completionService = new ExecutorCompletionService<>(storeOpenerThreadPool);<a name="line.1129"></a> |
| <span class="sourceLineNo">1130</span><a name="line.1130"></a> |
| <span class="sourceLineNo">1131</span> // initialize each store in parallel<a name="line.1131"></a> |
| <span class="sourceLineNo">1132</span> for (final ColumnFamilyDescriptor family : htableDescriptor.getColumnFamilies()) {<a name="line.1132"></a> |
| <span class="sourceLineNo">1133</span> status.setStatus("Instantiating store for column family " + family);<a name="line.1133"></a> |
| <span class="sourceLineNo">1134</span> completionService.submit(new Callable<HStore>() {<a name="line.1134"></a> |
| <span class="sourceLineNo">1135</span> @Override<a name="line.1135"></a> |
| <span class="sourceLineNo">1136</span> public HStore call() throws IOException {<a name="line.1136"></a> |
| <span class="sourceLineNo">1137</span> return instantiateHStore(family, warmup);<a name="line.1137"></a> |
| <span class="sourceLineNo">1138</span> }<a name="line.1138"></a> |
| <span class="sourceLineNo">1139</span> });<a name="line.1139"></a> |
| <span class="sourceLineNo">1140</span> }<a name="line.1140"></a> |
| <span class="sourceLineNo">1141</span> boolean allStoresOpened = false;<a name="line.1141"></a> |
| <span class="sourceLineNo">1142</span> boolean hasSloppyStores = false;<a name="line.1142"></a> |
| <span class="sourceLineNo">1143</span> try {<a name="line.1143"></a> |
| <span class="sourceLineNo">1144</span> for (int i = 0; i < htableDescriptor.getColumnFamilyCount(); i++) {<a name="line.1144"></a> |
| <span class="sourceLineNo">1145</span> Future<HStore> future = completionService.take();<a name="line.1145"></a> |
| <span class="sourceLineNo">1146</span> HStore store = future.get();<a name="line.1146"></a> |
| <span class="sourceLineNo">1147</span> this.stores.put(store.getColumnFamilyDescriptor().getName(), store);<a name="line.1147"></a> |
| <span class="sourceLineNo">1148</span> if (store.isSloppyMemStore()) {<a name="line.1148"></a> |
| <span class="sourceLineNo">1149</span> hasSloppyStores = true;<a name="line.1149"></a> |
| <span class="sourceLineNo">1150</span> }<a name="line.1150"></a> |
| <span class="sourceLineNo">1151</span><a name="line.1151"></a> |
| <span class="sourceLineNo">1152</span> long storeMaxSequenceId = store.getMaxSequenceId().orElse(0L);<a name="line.1152"></a> |
| <span class="sourceLineNo">1153</span> maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()),<a name="line.1153"></a> |
| <span class="sourceLineNo">1154</span> storeMaxSequenceId);<a name="line.1154"></a> |
| <span class="sourceLineNo">1155</span> if (maxSeqId == -1 || storeMaxSequenceId > maxSeqId) {<a name="line.1155"></a> |
| <span class="sourceLineNo">1156</span> maxSeqId = storeMaxSequenceId;<a name="line.1156"></a> |
| <span class="sourceLineNo">1157</span> }<a name="line.1157"></a> |
| <span class="sourceLineNo">1158</span> long maxStoreMemstoreTS = store.getMaxMemStoreTS().orElse(0L);<a name="line.1158"></a> |
| <span class="sourceLineNo">1159</span> if (maxStoreMemstoreTS > maxMemstoreTS) {<a name="line.1159"></a> |
| <span class="sourceLineNo">1160</span> maxMemstoreTS = maxStoreMemstoreTS;<a name="line.1160"></a> |
| <span class="sourceLineNo">1161</span> }<a name="line.1161"></a> |
| <span class="sourceLineNo">1162</span> }<a name="line.1162"></a> |
| <span class="sourceLineNo">1163</span> allStoresOpened = true;<a name="line.1163"></a> |
| <span class="sourceLineNo">1164</span> if(hasSloppyStores) {<a name="line.1164"></a> |
| <span class="sourceLineNo">1165</span> htableDescriptor = TableDescriptorBuilder.newBuilder(htableDescriptor)<a name="line.1165"></a> |
| <span class="sourceLineNo">1166</span> .setFlushPolicyClassName(FlushNonSloppyStoresFirstPolicy.class.getName())<a name="line.1166"></a> |
| <span class="sourceLineNo">1167</span> .build();<a name="line.1167"></a> |
| <span class="sourceLineNo">1168</span> LOG.info("Setting FlushNonSloppyStoresFirstPolicy for the region=" + this);<a name="line.1168"></a> |
| <span class="sourceLineNo">1169</span> }<a name="line.1169"></a> |
| <span class="sourceLineNo">1170</span> } catch (InterruptedException e) {<a name="line.1170"></a> |
| <span class="sourceLineNo">1171</span> throw throwOnInterrupt(e);<a name="line.1171"></a> |
| <span class="sourceLineNo">1172</span> } catch (ExecutionException e) {<a name="line.1172"></a> |
| <span class="sourceLineNo">1173</span> throw new IOException(e.getCause());<a name="line.1173"></a> |
| <span class="sourceLineNo">1174</span> } finally {<a name="line.1174"></a> |
| <span class="sourceLineNo">1175</span> storeOpenerThreadPool.shutdownNow();<a name="line.1175"></a> |
| <span class="sourceLineNo">1176</span> if (!allStoresOpened) {<a name="line.1176"></a> |
| <span class="sourceLineNo">1177</span> // something went wrong, close all opened stores<a name="line.1177"></a> |
| <span class="sourceLineNo">1178</span> LOG.error("Could not initialize all stores for the region=" + this);<a name="line.1178"></a> |
| <span class="sourceLineNo">1179</span> for (HStore store : this.stores.values()) {<a name="line.1179"></a> |
| <span class="sourceLineNo">1180</span> try {<a name="line.1180"></a> |
| <span class="sourceLineNo">1181</span> store.close();<a name="line.1181"></a> |
| <span class="sourceLineNo">1182</span> } catch (IOException e) {<a name="line.1182"></a> |
| <span class="sourceLineNo">1183</span> LOG.warn("close store {} failed in region {}", store.toString(), this, e);<a name="line.1183"></a> |
| <span class="sourceLineNo">1184</span> }<a name="line.1184"></a> |
| <span class="sourceLineNo">1185</span> }<a name="line.1185"></a> |
| <span class="sourceLineNo">1186</span> }<a name="line.1186"></a> |
| <span class="sourceLineNo">1187</span> }<a name="line.1187"></a> |
| <span class="sourceLineNo">1188</span> }<a name="line.1188"></a> |
| <span class="sourceLineNo">1189</span> return Math.max(maxSeqId, maxMemstoreTS + 1);<a name="line.1189"></a> |
| <span class="sourceLineNo">1190</span> }<a name="line.1190"></a> |
| <span class="sourceLineNo">1191</span><a name="line.1191"></a> |
| <span class="sourceLineNo">1192</span> private void initializeWarmup(final CancelableProgressable reporter) throws IOException {<a name="line.1192"></a> |
| <span class="sourceLineNo">1193</span> MonitoredTask status = TaskMonitor.get().createStatus("Initializing region " + this);<a name="line.1193"></a> |
| <span class="sourceLineNo">1194</span> // Initialize all the HStores<a name="line.1194"></a> |
| <span class="sourceLineNo">1195</span> status.setStatus("Warmup all stores of " + this.getRegionInfo().getRegionNameAsString());<a name="line.1195"></a> |
| <span class="sourceLineNo">1196</span> try {<a name="line.1196"></a> |
| <span class="sourceLineNo">1197</span> initializeStores(reporter, status, true);<a name="line.1197"></a> |
| <span class="sourceLineNo">1198</span> } finally {<a name="line.1198"></a> |
| <span class="sourceLineNo">1199</span> status.markComplete("Warmed up " + this.getRegionInfo().getRegionNameAsString());<a name="line.1199"></a> |
| <span class="sourceLineNo">1200</span> }<a name="line.1200"></a> |
| <span class="sourceLineNo">1201</span> }<a name="line.1201"></a> |
| <span class="sourceLineNo">1202</span><a name="line.1202"></a> |
| <span class="sourceLineNo">1203</span> /**<a name="line.1203"></a> |
| <span class="sourceLineNo">1204</span> * @return Map of StoreFiles by column family<a name="line.1204"></a> |
| <span class="sourceLineNo">1205</span> */<a name="line.1205"></a> |
| <span class="sourceLineNo">1206</span> private NavigableMap<byte[], List<Path>> getStoreFiles() {<a name="line.1206"></a> |
| <span class="sourceLineNo">1207</span> NavigableMap<byte[], List<Path>> allStoreFiles = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.1207"></a> |
| <span class="sourceLineNo">1208</span> for (HStore store : stores.values()) {<a name="line.1208"></a> |
| <span class="sourceLineNo">1209</span> Collection<HStoreFile> storeFiles = store.getStorefiles();<a name="line.1209"></a> |
| <span class="sourceLineNo">1210</span> if (storeFiles == null) {<a name="line.1210"></a> |
| <span class="sourceLineNo">1211</span> continue;<a name="line.1211"></a> |
| <span class="sourceLineNo">1212</span> }<a name="line.1212"></a> |
| <span class="sourceLineNo">1213</span> List<Path> storeFileNames = new ArrayList<>();<a name="line.1213"></a> |
| <span class="sourceLineNo">1214</span> for (HStoreFile storeFile : storeFiles) {<a name="line.1214"></a> |
| <span class="sourceLineNo">1215</span> storeFileNames.add(storeFile.getPath());<a name="line.1215"></a> |
| <span class="sourceLineNo">1216</span> }<a name="line.1216"></a> |
| <span class="sourceLineNo">1217</span> allStoreFiles.put(store.getColumnFamilyDescriptor().getName(), storeFileNames);<a name="line.1217"></a> |
| <span class="sourceLineNo">1218</span> }<a name="line.1218"></a> |
| <span class="sourceLineNo">1219</span> return allStoreFiles;<a name="line.1219"></a> |
| <span class="sourceLineNo">1220</span> }<a name="line.1220"></a> |
| <span class="sourceLineNo">1221</span><a name="line.1221"></a> |
| <span class="sourceLineNo">1222</span> protected void writeRegionOpenMarker(WAL wal, long openSeqId) throws IOException {<a name="line.1222"></a> |
| <span class="sourceLineNo">1223</span> Map<byte[], List<Path>> storeFiles = getStoreFiles();<a name="line.1223"></a> |
| <span class="sourceLineNo">1224</span> RegionEventDescriptor regionOpenDesc = ProtobufUtil.toRegionEventDescriptor(<a name="line.1224"></a> |
| <span class="sourceLineNo">1225</span> RegionEventDescriptor.EventType.REGION_OPEN, getRegionInfo(), openSeqId,<a name="line.1225"></a> |
| <span class="sourceLineNo">1226</span> getRegionServerServices().getServerName(), storeFiles);<a name="line.1226"></a> |
| <span class="sourceLineNo">1227</span> WALUtil.writeRegionEventMarker(wal, getReplicationScope(), getRegionInfo(), regionOpenDesc,<a name="line.1227"></a> |
| <span class="sourceLineNo">1228</span> mvcc);<a name="line.1228"></a> |
| <span class="sourceLineNo">1229</span> }<a name="line.1229"></a> |
| <span class="sourceLineNo">1230</span><a name="line.1230"></a> |
| <span class="sourceLineNo">1231</span> private void writeRegionCloseMarker(WAL wal) throws IOException {<a name="line.1231"></a> |
| <span class="sourceLineNo">1232</span> Map<byte[], List<Path>> storeFiles = getStoreFiles();<a name="line.1232"></a> |
| <span class="sourceLineNo">1233</span> RegionEventDescriptor regionEventDesc = ProtobufUtil.toRegionEventDescriptor(<a name="line.1233"></a> |
| <span class="sourceLineNo">1234</span> RegionEventDescriptor.EventType.REGION_CLOSE, getRegionInfo(), mvcc.getReadPoint(),<a name="line.1234"></a> |
| <span class="sourceLineNo">1235</span> getRegionServerServices().getServerName(), storeFiles);<a name="line.1235"></a> |
| <span class="sourceLineNo">1236</span> WALUtil.writeRegionEventMarker(wal, getReplicationScope(), getRegionInfo(), regionEventDesc,<a name="line.1236"></a> |
| <span class="sourceLineNo">1237</span> mvcc);<a name="line.1237"></a> |
| <span class="sourceLineNo">1238</span><a name="line.1238"></a> |
| <span class="sourceLineNo">1239</span> // Store SeqId in WAL FileSystem when a region closes<a name="line.1239"></a> |
| <span class="sourceLineNo">1240</span> // checking region folder exists is due to many tests which delete the table folder while a<a name="line.1240"></a> |
| <span class="sourceLineNo">1241</span> // table is still online<a name="line.1241"></a> |
| <span class="sourceLineNo">1242</span> if (getWalFileSystem().exists(getWALRegionDir())) {<a name="line.1242"></a> |
| <span class="sourceLineNo">1243</span> WALSplitUtil.writeRegionSequenceIdFile(getWalFileSystem(), getWALRegionDir(),<a name="line.1243"></a> |
| <span class="sourceLineNo">1244</span> mvcc.getReadPoint());<a name="line.1244"></a> |
| <span class="sourceLineNo">1245</span> }<a name="line.1245"></a> |
| <span class="sourceLineNo">1246</span> }<a name="line.1246"></a> |
| <span class="sourceLineNo">1247</span><a name="line.1247"></a> |
| <span class="sourceLineNo">1248</span> /**<a name="line.1248"></a> |
| <span class="sourceLineNo">1249</span> * @return True if this region has references.<a name="line.1249"></a> |
| <span class="sourceLineNo">1250</span> */<a name="line.1250"></a> |
| <span class="sourceLineNo">1251</span> public boolean hasReferences() {<a name="line.1251"></a> |
| <span class="sourceLineNo">1252</span> return stores.values().stream().anyMatch(HStore::hasReferences);<a name="line.1252"></a> |
| <span class="sourceLineNo">1253</span> }<a name="line.1253"></a> |
| <span class="sourceLineNo">1254</span><a name="line.1254"></a> |
| <span class="sourceLineNo">1255</span> public void blockUpdates() {<a name="line.1255"></a> |
| <span class="sourceLineNo">1256</span> this.updatesLock.writeLock().lock();<a name="line.1256"></a> |
| <span class="sourceLineNo">1257</span> }<a name="line.1257"></a> |
| <span class="sourceLineNo">1258</span><a name="line.1258"></a> |
| <span class="sourceLineNo">1259</span> public void unblockUpdates() {<a name="line.1259"></a> |
| <span class="sourceLineNo">1260</span> this.updatesLock.writeLock().unlock();<a name="line.1260"></a> |
| <span class="sourceLineNo">1261</span> }<a name="line.1261"></a> |
| <span class="sourceLineNo">1262</span><a name="line.1262"></a> |
| <span class="sourceLineNo">1263</span> public HDFSBlocksDistribution getHDFSBlocksDistribution() {<a name="line.1263"></a> |
| <span class="sourceLineNo">1264</span> HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();<a name="line.1264"></a> |
| <span class="sourceLineNo">1265</span> stores.values().stream().filter(s -> s.getStorefiles() != null)<a name="line.1265"></a> |
| <span class="sourceLineNo">1266</span> .flatMap(s -> s.getStorefiles().stream()).map(HStoreFile::getHDFSBlockDistribution)<a name="line.1266"></a> |
| <span class="sourceLineNo">1267</span> .forEachOrdered(hdfsBlocksDistribution::add);<a name="line.1267"></a> |
| <span class="sourceLineNo">1268</span> return hdfsBlocksDistribution;<a name="line.1268"></a> |
| <span class="sourceLineNo">1269</span> }<a name="line.1269"></a> |
| <span class="sourceLineNo">1270</span><a name="line.1270"></a> |
| <span class="sourceLineNo">1271</span> /**<a name="line.1271"></a> |
| <span class="sourceLineNo">1272</span> * This is a helper function to compute HDFS block distribution on demand<a name="line.1272"></a> |
| <span class="sourceLineNo">1273</span> * @param conf configuration<a name="line.1273"></a> |
| <span class="sourceLineNo">1274</span> * @param tableDescriptor TableDescriptor of the table<a name="line.1274"></a> |
| <span class="sourceLineNo">1275</span> * @param regionInfo encoded name of the region<a name="line.1275"></a> |
| <span class="sourceLineNo">1276</span> * @return The HDFS blocks distribution for the given region.<a name="line.1276"></a> |
| <span class="sourceLineNo">1277</span> */<a name="line.1277"></a> |
| <span class="sourceLineNo">1278</span> public static HDFSBlocksDistribution computeHDFSBlocksDistribution(Configuration conf,<a name="line.1278"></a> |
| <span class="sourceLineNo">1279</span> TableDescriptor tableDescriptor, RegionInfo regionInfo) throws IOException {<a name="line.1279"></a> |
| <span class="sourceLineNo">1280</span> Path tablePath =<a name="line.1280"></a> |
| <span class="sourceLineNo">1281</span> CommonFSUtils.getTableDir(CommonFSUtils.getRootDir(conf), tableDescriptor.getTableName());<a name="line.1281"></a> |
| <span class="sourceLineNo">1282</span> return computeHDFSBlocksDistribution(conf, tableDescriptor, regionInfo, tablePath);<a name="line.1282"></a> |
| <span class="sourceLineNo">1283</span> }<a name="line.1283"></a> |
| <span class="sourceLineNo">1284</span><a name="line.1284"></a> |
| <span class="sourceLineNo">1285</span> /**<a name="line.1285"></a> |
| <span class="sourceLineNo">1286</span> * This is a helper function to compute HDFS block distribution on demand<a name="line.1286"></a> |
| <span class="sourceLineNo">1287</span> * @param conf configuration<a name="line.1287"></a> |
| <span class="sourceLineNo">1288</span> * @param tableDescriptor TableDescriptor of the table<a name="line.1288"></a> |
| <span class="sourceLineNo">1289</span> * @param regionInfo encoded name of the region<a name="line.1289"></a> |
| <span class="sourceLineNo">1290</span> * @param tablePath the table directory<a name="line.1290"></a> |
| <span class="sourceLineNo">1291</span> * @return The HDFS blocks distribution for the given region.<a name="line.1291"></a> |
| <span class="sourceLineNo">1292</span> * @throws IOException<a name="line.1292"></a> |
| <span class="sourceLineNo">1293</span> */<a name="line.1293"></a> |
| <span class="sourceLineNo">1294</span> public static HDFSBlocksDistribution computeHDFSBlocksDistribution(Configuration conf,<a name="line.1294"></a> |
| <span class="sourceLineNo">1295</span> TableDescriptor tableDescriptor, RegionInfo regionInfo, Path tablePath) throws IOException {<a name="line.1295"></a> |
| <span class="sourceLineNo">1296</span> HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();<a name="line.1296"></a> |
| <span class="sourceLineNo">1297</span> FileSystem fs = tablePath.getFileSystem(conf);<a name="line.1297"></a> |
| <span class="sourceLineNo">1298</span><a name="line.1298"></a> |
| <span class="sourceLineNo">1299</span> HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tablePath, regionInfo);<a name="line.1299"></a> |
| <span class="sourceLineNo">1300</span> for (ColumnFamilyDescriptor family : tableDescriptor.getColumnFamilies()) {<a name="line.1300"></a> |
| <span class="sourceLineNo">1301</span> List<LocatedFileStatus> locatedFileStatusList = HRegionFileSystem<a name="line.1301"></a> |
| <span class="sourceLineNo">1302</span> .getStoreFilesLocatedStatus(regionFs, family.getNameAsString(), true);<a name="line.1302"></a> |
| <span class="sourceLineNo">1303</span> if (locatedFileStatusList == null) {<a name="line.1303"></a> |
| <span class="sourceLineNo">1304</span> continue;<a name="line.1304"></a> |
| <span class="sourceLineNo">1305</span> }<a name="line.1305"></a> |
| <span class="sourceLineNo">1306</span><a name="line.1306"></a> |
| <span class="sourceLineNo">1307</span> for (LocatedFileStatus status : locatedFileStatusList) {<a name="line.1307"></a> |
| <span class="sourceLineNo">1308</span> Path p = status.getPath();<a name="line.1308"></a> |
| <span class="sourceLineNo">1309</span> if (StoreFileInfo.isReference(p) || HFileLink.isHFileLink(p)) {<a name="line.1309"></a> |
| <span class="sourceLineNo">1310</span> // Only construct StoreFileInfo object if its not a hfile, save obj<a name="line.1310"></a> |
| <span class="sourceLineNo">1311</span> // creation<a name="line.1311"></a> |
| <span class="sourceLineNo">1312</span> StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, status);<a name="line.1312"></a> |
| <span class="sourceLineNo">1313</span> hdfsBlocksDistribution.add(storeFileInfo<a name="line.1313"></a> |
| <span class="sourceLineNo">1314</span> .computeHDFSBlocksDistribution(fs));<a name="line.1314"></a> |
| <span class="sourceLineNo">1315</span> } else if (StoreFileInfo.isHFile(p)) {<a name="line.1315"></a> |
| <span class="sourceLineNo">1316</span> // If its a HFile, then lets just add to the block distribution<a name="line.1316"></a> |
| <span class="sourceLineNo">1317</span> // lets not create more objects here, not even another HDFSBlocksDistribution<a name="line.1317"></a> |
| <span class="sourceLineNo">1318</span> FSUtils.addToHDFSBlocksDistribution(hdfsBlocksDistribution,<a name="line.1318"></a> |
| <span class="sourceLineNo">1319</span> status.getBlockLocations());<a name="line.1319"></a> |
| <span class="sourceLineNo">1320</span> } else {<a name="line.1320"></a> |
| <span class="sourceLineNo">1321</span> throw new IOException("path=" + p<a name="line.1321"></a> |
| <span class="sourceLineNo">1322</span> + " doesn't look like a valid StoreFile");<a name="line.1322"></a> |
| <span class="sourceLineNo">1323</span> }<a name="line.1323"></a> |
| <span class="sourceLineNo">1324</span> }<a name="line.1324"></a> |
| <span class="sourceLineNo">1325</span> }<a name="line.1325"></a> |
| <span class="sourceLineNo">1326</span> return hdfsBlocksDistribution;<a name="line.1326"></a> |
| <span class="sourceLineNo">1327</span> }<a name="line.1327"></a> |
| <span class="sourceLineNo">1328</span><a name="line.1328"></a> |
| <span class="sourceLineNo">1329</span> /**<a name="line.1329"></a> |
| <span class="sourceLineNo">1330</span> * Increase the size of mem store in this region and the size of global mem<a name="line.1330"></a> |
| <span class="sourceLineNo">1331</span> * store<a name="line.1331"></a> |
| <span class="sourceLineNo">1332</span> */<a name="line.1332"></a> |
| <span class="sourceLineNo">1333</span> private void incMemStoreSize(MemStoreSize mss) {<a name="line.1333"></a> |
| <span class="sourceLineNo">1334</span> incMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(),<a name="line.1334"></a> |
| <span class="sourceLineNo">1335</span> mss.getCellsCount());<a name="line.1335"></a> |
| <span class="sourceLineNo">1336</span> }<a name="line.1336"></a> |
| <span class="sourceLineNo">1337</span><a name="line.1337"></a> |
| <span class="sourceLineNo">1338</span> void incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta,<a name="line.1338"></a> |
| <span class="sourceLineNo">1339</span> int cellsCountDelta) {<a name="line.1339"></a> |
| <span class="sourceLineNo">1340</span> if (this.rsAccounting != null) {<a name="line.1340"></a> |
| <span class="sourceLineNo">1341</span> rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta);<a name="line.1341"></a> |
| <span class="sourceLineNo">1342</span> }<a name="line.1342"></a> |
| <span class="sourceLineNo">1343</span> long dataSize = this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta,<a name="line.1343"></a> |
| <span class="sourceLineNo">1344</span> offHeapSizeDelta, cellsCountDelta);<a name="line.1344"></a> |
| <span class="sourceLineNo">1345</span> checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);<a name="line.1345"></a> |
| <span class="sourceLineNo">1346</span> }<a name="line.1346"></a> |
| <span class="sourceLineNo">1347</span><a name="line.1347"></a> |
| <span class="sourceLineNo">1348</span> void decrMemStoreSize(MemStoreSize mss) {<a name="line.1348"></a> |
| <span class="sourceLineNo">1349</span> decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(),<a name="line.1349"></a> |
| <span class="sourceLineNo">1350</span> mss.getCellsCount());<a name="line.1350"></a> |
| <span class="sourceLineNo">1351</span> }<a name="line.1351"></a> |
| <span class="sourceLineNo">1352</span><a name="line.1352"></a> |
| <span class="sourceLineNo">1353</span> private void decrMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta,<a name="line.1353"></a> |
| <span class="sourceLineNo">1354</span> int cellsCountDelta) {<a name="line.1354"></a> |
| <span class="sourceLineNo">1355</span> if (this.rsAccounting != null) {<a name="line.1355"></a> |
| <span class="sourceLineNo">1356</span> rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta);<a name="line.1356"></a> |
| <span class="sourceLineNo">1357</span> }<a name="line.1357"></a> |
| <span class="sourceLineNo">1358</span> long dataSize = this.memStoreSizing.decMemStoreSize(dataSizeDelta, heapSizeDelta,<a name="line.1358"></a> |
| <span class="sourceLineNo">1359</span> offHeapSizeDelta, cellsCountDelta);<a name="line.1359"></a> |
| <span class="sourceLineNo">1360</span> checkNegativeMemStoreDataSize(dataSize, -dataSizeDelta);<a name="line.1360"></a> |
| <span class="sourceLineNo">1361</span> }<a name="line.1361"></a> |
| <span class="sourceLineNo">1362</span><a name="line.1362"></a> |
| <span class="sourceLineNo">1363</span> private void checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {<a name="line.1363"></a> |
| <span class="sourceLineNo">1364</span> // This is extremely bad if we make memStoreSizing negative. Log as much info on the offending<a name="line.1364"></a> |
| <span class="sourceLineNo">1365</span> // caller as possible. (memStoreSizing might be a negative value already -- freeing memory)<a name="line.1365"></a> |
| <span class="sourceLineNo">1366</span> if (memStoreDataSize < 0) {<a name="line.1366"></a> |
| <span class="sourceLineNo">1367</span> LOG.error("Asked to modify this region's (" + this.toString()<a name="line.1367"></a> |
| <span class="sourceLineNo">1368</span> + ") memStoreSizing to a negative value which is incorrect. Current memStoreSizing="<a name="line.1368"></a> |
| <span class="sourceLineNo">1369</span> + (memStoreDataSize - delta) + ", delta=" + delta, new Exception());<a name="line.1369"></a> |
| <span class="sourceLineNo">1370</span> }<a name="line.1370"></a> |
| <span class="sourceLineNo">1371</span> }<a name="line.1371"></a> |
| <span class="sourceLineNo">1372</span><a name="line.1372"></a> |
| <span class="sourceLineNo">1373</span> @Override<a name="line.1373"></a> |
| <span class="sourceLineNo">1374</span> public RegionInfo getRegionInfo() {<a name="line.1374"></a> |
| <span class="sourceLineNo">1375</span> return this.fs.getRegionInfo();<a name="line.1375"></a> |
| <span class="sourceLineNo">1376</span> }<a name="line.1376"></a> |
| <span class="sourceLineNo">1377</span><a name="line.1377"></a> |
| <span class="sourceLineNo">1378</span> /**<a name="line.1378"></a> |
| <span class="sourceLineNo">1379</span> * @return Instance of {@link RegionServerServices} used by this HRegion.<a name="line.1379"></a> |
| <span class="sourceLineNo">1380</span> * Can be null.<a name="line.1380"></a> |
| <span class="sourceLineNo">1381</span> */<a name="line.1381"></a> |
| <span class="sourceLineNo">1382</span> RegionServerServices getRegionServerServices() {<a name="line.1382"></a> |
| <span class="sourceLineNo">1383</span> return this.rsServices;<a name="line.1383"></a> |
| <span class="sourceLineNo">1384</span> }<a name="line.1384"></a> |
| <span class="sourceLineNo">1385</span><a name="line.1385"></a> |
| <span class="sourceLineNo">1386</span> @Override<a name="line.1386"></a> |
| <span class="sourceLineNo">1387</span> public long getReadRequestsCount() {<a name="line.1387"></a> |
| <span class="sourceLineNo">1388</span> return readRequestsCount.sum();<a name="line.1388"></a> |
| <span class="sourceLineNo">1389</span> }<a name="line.1389"></a> |
| <span class="sourceLineNo">1390</span><a name="line.1390"></a> |
| <span class="sourceLineNo">1391</span> @Override<a name="line.1391"></a> |
| <span class="sourceLineNo">1392</span> public long getCpRequestsCount() {<a name="line.1392"></a> |
| <span class="sourceLineNo">1393</span> return cpRequestsCount.sum();<a name="line.1393"></a> |
| <span class="sourceLineNo">1394</span> }<a name="line.1394"></a> |
| <span class="sourceLineNo">1395</span><a name="line.1395"></a> |
| <span class="sourceLineNo">1396</span> @Override<a name="line.1396"></a> |
| <span class="sourceLineNo">1397</span> public long getFilteredReadRequestsCount() {<a name="line.1397"></a> |
| <span class="sourceLineNo">1398</span> return filteredReadRequestsCount.sum();<a name="line.1398"></a> |
| <span class="sourceLineNo">1399</span> }<a name="line.1399"></a> |
| <span class="sourceLineNo">1400</span><a name="line.1400"></a> |
| <span class="sourceLineNo">1401</span> @Override<a name="line.1401"></a> |
| <span class="sourceLineNo">1402</span> public long getWriteRequestsCount() {<a name="line.1402"></a> |
| <span class="sourceLineNo">1403</span> return writeRequestsCount.sum();<a name="line.1403"></a> |
| <span class="sourceLineNo">1404</span> }<a name="line.1404"></a> |
| <span class="sourceLineNo">1405</span><a name="line.1405"></a> |
| <span class="sourceLineNo">1406</span> @Override<a name="line.1406"></a> |
| <span class="sourceLineNo">1407</span> public long getMemStoreDataSize() {<a name="line.1407"></a> |
| <span class="sourceLineNo">1408</span> return memStoreSizing.getDataSize();<a name="line.1408"></a> |
| <span class="sourceLineNo">1409</span> }<a name="line.1409"></a> |
| <span class="sourceLineNo">1410</span><a name="line.1410"></a> |
| <span class="sourceLineNo">1411</span> @Override<a name="line.1411"></a> |
| <span class="sourceLineNo">1412</span> public long getMemStoreHeapSize() {<a name="line.1412"></a> |
| <span class="sourceLineNo">1413</span> return memStoreSizing.getHeapSize();<a name="line.1413"></a> |
| <span class="sourceLineNo">1414</span> }<a name="line.1414"></a> |
| <span class="sourceLineNo">1415</span><a name="line.1415"></a> |
| <span class="sourceLineNo">1416</span> @Override<a name="line.1416"></a> |
| <span class="sourceLineNo">1417</span> public long getMemStoreOffHeapSize() {<a name="line.1417"></a> |
| <span class="sourceLineNo">1418</span> return memStoreSizing.getOffHeapSize();<a name="line.1418"></a> |
| <span class="sourceLineNo">1419</span> }<a name="line.1419"></a> |
| <span class="sourceLineNo">1420</span><a name="line.1420"></a> |
| <span class="sourceLineNo">1421</span> /** @return store services for this region, to access services required by store level needs */<a name="line.1421"></a> |
| <span class="sourceLineNo">1422</span> public RegionServicesForStores getRegionServicesForStores() {<a name="line.1422"></a> |
| <span class="sourceLineNo">1423</span> return regionServicesForStores;<a name="line.1423"></a> |
| <span class="sourceLineNo">1424</span> }<a name="line.1424"></a> |
| <span class="sourceLineNo">1425</span><a name="line.1425"></a> |
| <span class="sourceLineNo">1426</span> @Override<a name="line.1426"></a> |
| <span class="sourceLineNo">1427</span> public long getNumMutationsWithoutWAL() {<a name="line.1427"></a> |
| <span class="sourceLineNo">1428</span> return numMutationsWithoutWAL.sum();<a name="line.1428"></a> |
| <span class="sourceLineNo">1429</span> }<a name="line.1429"></a> |
| <span class="sourceLineNo">1430</span><a name="line.1430"></a> |
| <span class="sourceLineNo">1431</span> @Override<a name="line.1431"></a> |
| <span class="sourceLineNo">1432</span> public long getDataInMemoryWithoutWAL() {<a name="line.1432"></a> |
| <span class="sourceLineNo">1433</span> return dataInMemoryWithoutWAL.sum();<a name="line.1433"></a> |
| <span class="sourceLineNo">1434</span> }<a name="line.1434"></a> |
| <span class="sourceLineNo">1435</span><a name="line.1435"></a> |
| <span class="sourceLineNo">1436</span> @Override<a name="line.1436"></a> |
| <span class="sourceLineNo">1437</span> public long getBlockedRequestsCount() {<a name="line.1437"></a> |
| <span class="sourceLineNo">1438</span> return blockedRequestsCount.sum();<a name="line.1438"></a> |
| <span class="sourceLineNo">1439</span> }<a name="line.1439"></a> |
| <span class="sourceLineNo">1440</span><a name="line.1440"></a> |
| <span class="sourceLineNo">1441</span> @Override<a name="line.1441"></a> |
| <span class="sourceLineNo">1442</span> public long getCheckAndMutateChecksPassed() {<a name="line.1442"></a> |
| <span class="sourceLineNo">1443</span> return checkAndMutateChecksPassed.sum();<a name="line.1443"></a> |
| <span class="sourceLineNo">1444</span> }<a name="line.1444"></a> |
| <span class="sourceLineNo">1445</span><a name="line.1445"></a> |
| <span class="sourceLineNo">1446</span> @Override<a name="line.1446"></a> |
| <span class="sourceLineNo">1447</span> public long getCheckAndMutateChecksFailed() {<a name="line.1447"></a> |
| <span class="sourceLineNo">1448</span> return checkAndMutateChecksFailed.sum();<a name="line.1448"></a> |
| <span class="sourceLineNo">1449</span> }<a name="line.1449"></a> |
| <span class="sourceLineNo">1450</span><a name="line.1450"></a> |
| <span class="sourceLineNo">1451</span> // TODO Needs to check whether we should expose our metrics system to CPs. If CPs themselves doing<a name="line.1451"></a> |
| <span class="sourceLineNo">1452</span> // the op and bypassing the core, this might be needed? Should be stop supporting the bypass<a name="line.1452"></a> |
| <span class="sourceLineNo">1453</span> // feature?<a name="line.1453"></a> |
| <span class="sourceLineNo">1454</span> public MetricsRegion getMetrics() {<a name="line.1454"></a> |
| <span class="sourceLineNo">1455</span> return metricsRegion;<a name="line.1455"></a> |
| <span class="sourceLineNo">1456</span> }<a name="line.1456"></a> |
| <span class="sourceLineNo">1457</span><a name="line.1457"></a> |
| <span class="sourceLineNo">1458</span> @Override<a name="line.1458"></a> |
| <span class="sourceLineNo">1459</span> public boolean isClosed() {<a name="line.1459"></a> |
| <span class="sourceLineNo">1460</span> return this.closed.get();<a name="line.1460"></a> |
| <span class="sourceLineNo">1461</span> }<a name="line.1461"></a> |
| <span class="sourceLineNo">1462</span><a name="line.1462"></a> |
| <span class="sourceLineNo">1463</span> @Override<a name="line.1463"></a> |
| <span class="sourceLineNo">1464</span> public boolean isClosing() {<a name="line.1464"></a> |
| <span class="sourceLineNo">1465</span> return this.closing.get();<a name="line.1465"></a> |
| <span class="sourceLineNo">1466</span> }<a name="line.1466"></a> |
| <span class="sourceLineNo">1467</span><a name="line.1467"></a> |
| <span class="sourceLineNo">1468</span> @Override<a name="line.1468"></a> |
| <span class="sourceLineNo">1469</span> public boolean isReadOnly() {<a name="line.1469"></a> |
| <span class="sourceLineNo">1470</span> return this.writestate.isReadOnly();<a name="line.1470"></a> |
| <span class="sourceLineNo">1471</span> }<a name="line.1471"></a> |
| <span class="sourceLineNo">1472</span><a name="line.1472"></a> |
| <span class="sourceLineNo">1473</span> @Override<a name="line.1473"></a> |
| <span class="sourceLineNo">1474</span> public boolean isAvailable() {<a name="line.1474"></a> |
| <span class="sourceLineNo">1475</span> return !isClosed() && !isClosing();<a name="line.1475"></a> |
| <span class="sourceLineNo">1476</span> }<a name="line.1476"></a> |
| <span class="sourceLineNo">1477</span><a name="line.1477"></a> |
| <span class="sourceLineNo">1478</span> @Override<a name="line.1478"></a> |
| <span class="sourceLineNo">1479</span> public boolean isSplittable() {<a name="line.1479"></a> |
| <span class="sourceLineNo">1480</span> return splitPolicy.canSplit();<a name="line.1480"></a> |
| <span class="sourceLineNo">1481</span> }<a name="line.1481"></a> |
| <span class="sourceLineNo">1482</span><a name="line.1482"></a> |
| <span class="sourceLineNo">1483</span> @Override<a name="line.1483"></a> |
| <span class="sourceLineNo">1484</span> public boolean isMergeable() {<a name="line.1484"></a> |
| <span class="sourceLineNo">1485</span> if (!isAvailable()) {<a name="line.1485"></a> |
| <span class="sourceLineNo">1486</span> LOG.debug("Region " + this<a name="line.1486"></a> |
| <span class="sourceLineNo">1487</span> + " is not mergeable because it is closing or closed");<a name="line.1487"></a> |
| <span class="sourceLineNo">1488</span> return false;<a name="line.1488"></a> |
| <span class="sourceLineNo">1489</span> }<a name="line.1489"></a> |
| <span class="sourceLineNo">1490</span> if (hasReferences()) {<a name="line.1490"></a> |
| <span class="sourceLineNo">1491</span> LOG.debug("Region " + this<a name="line.1491"></a> |
| <span class="sourceLineNo">1492</span> + " is not mergeable because it has references");<a name="line.1492"></a> |
| <span class="sourceLineNo">1493</span> return false;<a name="line.1493"></a> |
| <span class="sourceLineNo">1494</span> }<a name="line.1494"></a> |
| <span class="sourceLineNo">1495</span><a name="line.1495"></a> |
| <span class="sourceLineNo">1496</span> return true;<a name="line.1496"></a> |
| <span class="sourceLineNo">1497</span> }<a name="line.1497"></a> |
| <span class="sourceLineNo">1498</span><a name="line.1498"></a> |
| <span class="sourceLineNo">1499</span> public boolean areWritesEnabled() {<a name="line.1499"></a> |
| <span class="sourceLineNo">1500</span> synchronized(this.writestate) {<a name="line.1500"></a> |
| <span class="sourceLineNo">1501</span> return this.writestate.writesEnabled;<a name="line.1501"></a> |
| <span class="sourceLineNo">1502</span> }<a name="line.1502"></a> |
| <span class="sourceLineNo">1503</span> }<a name="line.1503"></a> |
| <span class="sourceLineNo">1504</span><a name="line.1504"></a> |
| <span class="sourceLineNo">1505</span> public MultiVersionConcurrencyControl getMVCC() {<a name="line.1505"></a> |
| <span class="sourceLineNo">1506</span> return mvcc;<a name="line.1506"></a> |
| <span class="sourceLineNo">1507</span> }<a name="line.1507"></a> |
| <span class="sourceLineNo">1508</span><a name="line.1508"></a> |
| <span class="sourceLineNo">1509</span> @Override<a name="line.1509"></a> |
| <span class="sourceLineNo">1510</span> public long getMaxFlushedSeqId() {<a name="line.1510"></a> |
| <span class="sourceLineNo">1511</span> return maxFlushedSeqId;<a name="line.1511"></a> |
| <span class="sourceLineNo">1512</span> }<a name="line.1512"></a> |
| <span class="sourceLineNo">1513</span><a name="line.1513"></a> |
| <span class="sourceLineNo">1514</span> /**<a name="line.1514"></a> |
| <span class="sourceLineNo">1515</span> * @return readpoint considering given IsolationLevel. Pass {@code null} for default<a name="line.1515"></a> |
| <span class="sourceLineNo">1516</span> */<a name="line.1516"></a> |
| <span class="sourceLineNo">1517</span> public long getReadPoint(IsolationLevel isolationLevel) {<a name="line.1517"></a> |
| <span class="sourceLineNo">1518</span> if (isolationLevel != null && isolationLevel == IsolationLevel.READ_UNCOMMITTED) {<a name="line.1518"></a> |
| <span class="sourceLineNo">1519</span> // This scan can read even uncommitted transactions<a name="line.1519"></a> |
| <span class="sourceLineNo">1520</span> return Long.MAX_VALUE;<a name="line.1520"></a> |
| <span class="sourceLineNo">1521</span> }<a name="line.1521"></a> |
| <span class="sourceLineNo">1522</span> return mvcc.getReadPoint();<a name="line.1522"></a> |
| <span class="sourceLineNo">1523</span> }<a name="line.1523"></a> |
| <span class="sourceLineNo">1524</span><a name="line.1524"></a> |
| <span class="sourceLineNo">1525</span> public boolean isLoadingCfsOnDemandDefault() {<a name="line.1525"></a> |
| <span class="sourceLineNo">1526</span> return this.isLoadingCfsOnDemandDefault;<a name="line.1526"></a> |
| <span class="sourceLineNo">1527</span> }<a name="line.1527"></a> |
| <span class="sourceLineNo">1528</span><a name="line.1528"></a> |
| <span class="sourceLineNo">1529</span> /**<a name="line.1529"></a> |
| <span class="sourceLineNo">1530</span> * Close down this HRegion. Flush the cache, shut down each HStore, don't<a name="line.1530"></a> |
| <span class="sourceLineNo">1531</span> * service any more calls.<a name="line.1531"></a> |
| <span class="sourceLineNo">1532</span> *<a name="line.1532"></a> |
| <span class="sourceLineNo">1533</span> * <p>This method could take some time to execute, so don't call it from a<a name="line.1533"></a> |
| <span class="sourceLineNo">1534</span> * time-sensitive thread.<a name="line.1534"></a> |
| <span class="sourceLineNo">1535</span> *<a name="line.1535"></a> |
| <span class="sourceLineNo">1536</span> * @return Vector of all the storage files that the HRegion's component<a name="line.1536"></a> |
| <span class="sourceLineNo">1537</span> * HStores make use of. It's a list of all StoreFile objects. Returns empty<a name="line.1537"></a> |
| <span class="sourceLineNo">1538</span> * vector if already closed and null if judged that it should not close.<a name="line.1538"></a> |
| <span class="sourceLineNo">1539</span> *<a name="line.1539"></a> |
| <span class="sourceLineNo">1540</span> * @throws IOException e<a name="line.1540"></a> |
| <span class="sourceLineNo">1541</span> * @throws DroppedSnapshotException Thrown when replay of wal is required<a name="line.1541"></a> |
| <span class="sourceLineNo">1542</span> * because a Snapshot was not properly persisted. The region is put in closing mode, and the<a name="line.1542"></a> |
| <span class="sourceLineNo">1543</span> * caller MUST abort after this.<a name="line.1543"></a> |
| <span class="sourceLineNo">1544</span> */<a name="line.1544"></a> |
| <span class="sourceLineNo">1545</span> public Map<byte[], List<HStoreFile>> close() throws IOException {<a name="line.1545"></a> |
| <span class="sourceLineNo">1546</span> return close(false);<a name="line.1546"></a> |
| <span class="sourceLineNo">1547</span> }<a name="line.1547"></a> |
| <span class="sourceLineNo">1548</span><a name="line.1548"></a> |
| <span class="sourceLineNo">1549</span> private final Object closeLock = new Object();<a name="line.1549"></a> |
| <span class="sourceLineNo">1550</span><a name="line.1550"></a> |
| <span class="sourceLineNo">1551</span> /** Conf key for fair locking policy */<a name="line.1551"></a> |
| <span class="sourceLineNo">1552</span> public static final String FAIR_REENTRANT_CLOSE_LOCK =<a name="line.1552"></a> |
| <span class="sourceLineNo">1553</span> "hbase.regionserver.fair.region.close.lock";<a name="line.1553"></a> |
| <span class="sourceLineNo">1554</span> public static final boolean DEFAULT_FAIR_REENTRANT_CLOSE_LOCK = true;<a name="line.1554"></a> |
| <span class="sourceLineNo">1555</span> /** Conf key for the periodic flush interval */<a name="line.1555"></a> |
| <span class="sourceLineNo">1556</span> public static final String MEMSTORE_PERIODIC_FLUSH_INTERVAL =<a name="line.1556"></a> |
| <span class="sourceLineNo">1557</span> "hbase.regionserver.optionalcacheflushinterval";<a name="line.1557"></a> |
| <span class="sourceLineNo">1558</span> /** Default interval for the memstore flush */<a name="line.1558"></a> |
| <span class="sourceLineNo">1559</span> public static final int DEFAULT_CACHE_FLUSH_INTERVAL = 3600000;<a name="line.1559"></a> |
| <span class="sourceLineNo">1560</span> /** Default interval for System tables memstore flush */<a name="line.1560"></a> |
| <span class="sourceLineNo">1561</span> public static final int SYSTEM_CACHE_FLUSH_INTERVAL = 300000; // 5 minutes<a name="line.1561"></a> |
| <span class="sourceLineNo">1562</span><a name="line.1562"></a> |
| <span class="sourceLineNo">1563</span> /** Conf key to force a flush if there are already enough changes for one region in memstore */<a name="line.1563"></a> |
| <span class="sourceLineNo">1564</span> public static final String MEMSTORE_FLUSH_PER_CHANGES =<a name="line.1564"></a> |
| <span class="sourceLineNo">1565</span> "hbase.regionserver.flush.per.changes";<a name="line.1565"></a> |
| <span class="sourceLineNo">1566</span> public static final long DEFAULT_FLUSH_PER_CHANGES = 30000000; // 30 millions<a name="line.1566"></a> |
| <span class="sourceLineNo">1567</span> /**<a name="line.1567"></a> |
| <span class="sourceLineNo">1568</span> * The following MAX_FLUSH_PER_CHANGES is large enough because each KeyValue has 20+ bytes<a name="line.1568"></a> |
| <span class="sourceLineNo">1569</span> * overhead. Therefore, even 1G empty KVs occupy at least 20GB memstore size for a single region<a name="line.1569"></a> |
| <span class="sourceLineNo">1570</span> */<a name="line.1570"></a> |
| <span class="sourceLineNo">1571</span> public static final long MAX_FLUSH_PER_CHANGES = 1000000000; // 1G<a name="line.1571"></a> |
| <span class="sourceLineNo">1572</span><a name="line.1572"></a> |
| <span class="sourceLineNo">1573</span> public static final String CLOSE_WAIT_ABORT = "hbase.regionserver.close.wait.abort";<a name="line.1573"></a> |
| <span class="sourceLineNo">1574</span> public static final boolean DEFAULT_CLOSE_WAIT_ABORT = true;<a name="line.1574"></a> |
| <span class="sourceLineNo">1575</span> public static final String CLOSE_WAIT_TIME = "hbase.regionserver.close.wait.time.ms";<a name="line.1575"></a> |
| <span class="sourceLineNo">1576</span> public static final long DEFAULT_CLOSE_WAIT_TIME = 60000; // 1 minute<a name="line.1576"></a> |
| <span class="sourceLineNo">1577</span> public static final String CLOSE_WAIT_INTERVAL = "hbase.regionserver.close.wait.interval.ms";<a name="line.1577"></a> |
| <span class="sourceLineNo">1578</span> public static final long DEFAULT_CLOSE_WAIT_INTERVAL = 10000; // 10 seconds<a name="line.1578"></a> |
| <span class="sourceLineNo">1579</span><a name="line.1579"></a> |
| <span class="sourceLineNo">1580</span> public Map<byte[], List<HStoreFile>> close(boolean abort) throws IOException {<a name="line.1580"></a> |
| <span class="sourceLineNo">1581</span> return close(abort, false);<a name="line.1581"></a> |
| <span class="sourceLineNo">1582</span> }<a name="line.1582"></a> |
| <span class="sourceLineNo">1583</span><a name="line.1583"></a> |
| <span class="sourceLineNo">1584</span> /**<a name="line.1584"></a> |
| <span class="sourceLineNo">1585</span> * Close down this HRegion. Flush the cache unless abort parameter is true,<a name="line.1585"></a> |
| <span class="sourceLineNo">1586</span> * Shut down each HStore, don't service any more calls.<a name="line.1586"></a> |
| <span class="sourceLineNo">1587</span> *<a name="line.1587"></a> |
| <span class="sourceLineNo">1588</span> * This method could take some time to execute, so don't call it from a<a name="line.1588"></a> |
| <span class="sourceLineNo">1589</span> * time-sensitive thread.<a name="line.1589"></a> |
| <span class="sourceLineNo">1590</span> *<a name="line.1590"></a> |
| <span class="sourceLineNo">1591</span> * @param abort true if server is aborting (only during testing)<a name="line.1591"></a> |
| <span class="sourceLineNo">1592</span> * @param ignoreStatus true if ignore the status (wont be showed on task list)<a name="line.1592"></a> |
| <span class="sourceLineNo">1593</span> * @return Vector of all the storage files that the HRegion's component<a name="line.1593"></a> |
| <span class="sourceLineNo">1594</span> * HStores make use of. It's a list of StoreFile objects. Can be null if<a name="line.1594"></a> |
| <span class="sourceLineNo">1595</span> * we are not to close at this time or we are already closed.<a name="line.1595"></a> |
| <span class="sourceLineNo">1596</span> *<a name="line.1596"></a> |
| <span class="sourceLineNo">1597</span> * @throws IOException e<a name="line.1597"></a> |
| <span class="sourceLineNo">1598</span> * @throws DroppedSnapshotException Thrown when replay of wal is required<a name="line.1598"></a> |
| <span class="sourceLineNo">1599</span> * because a Snapshot was not properly persisted. The region is put in closing mode, and the<a name="line.1599"></a> |
| <span class="sourceLineNo">1600</span> * caller MUST abort after this.<a name="line.1600"></a> |
| <span class="sourceLineNo">1601</span> */<a name="line.1601"></a> |
| <span class="sourceLineNo">1602</span> public Map<byte[], List<HStoreFile>> close(boolean abort, boolean ignoreStatus)<a name="line.1602"></a> |
| <span class="sourceLineNo">1603</span> throws IOException {<a name="line.1603"></a> |
| <span class="sourceLineNo">1604</span> // Only allow one thread to close at a time. Serialize them so dual<a name="line.1604"></a> |
| <span class="sourceLineNo">1605</span> // threads attempting to close will run up against each other.<a name="line.1605"></a> |
| <span class="sourceLineNo">1606</span> MonitoredTask status = TaskMonitor.get().createStatus(<a name="line.1606"></a> |
| <span class="sourceLineNo">1607</span> "Closing region " + this.getRegionInfo().getEncodedName() +<a name="line.1607"></a> |
| <span class="sourceLineNo">1608</span> (abort ? " due to abort" : ""), ignoreStatus);<a name="line.1608"></a> |
| <span class="sourceLineNo">1609</span> status.enableStatusJournal(true);<a name="line.1609"></a> |
| <span class="sourceLineNo">1610</span> status.setStatus("Waiting for close lock");<a name="line.1610"></a> |
| <span class="sourceLineNo">1611</span> try {<a name="line.1611"></a> |
| <span class="sourceLineNo">1612</span> synchronized (closeLock) {<a name="line.1612"></a> |
| <span class="sourceLineNo">1613</span> return doClose(abort, status);<a name="line.1613"></a> |
| <span class="sourceLineNo">1614</span> }<a name="line.1614"></a> |
| <span class="sourceLineNo">1615</span> } finally {<a name="line.1615"></a> |
| <span class="sourceLineNo">1616</span> if (LOG.isDebugEnabled()) {<a name="line.1616"></a> |
| <span class="sourceLineNo">1617</span> LOG.debug("Region close journal for {}:\n{}", this.getRegionInfo().getEncodedName(),<a name="line.1617"></a> |
| <span class="sourceLineNo">1618</span> status.prettyPrintJournal());<a name="line.1618"></a> |
| <span class="sourceLineNo">1619</span> }<a name="line.1619"></a> |
| <span class="sourceLineNo">1620</span> status.cleanup();<a name="line.1620"></a> |
| <span class="sourceLineNo">1621</span> }<a name="line.1621"></a> |
| <span class="sourceLineNo">1622</span> }<a name="line.1622"></a> |
| <span class="sourceLineNo">1623</span><a name="line.1623"></a> |
| <span class="sourceLineNo">1624</span> /**<a name="line.1624"></a> |
| <span class="sourceLineNo">1625</span> * Exposed for some very specific unit tests.<a name="line.1625"></a> |
| <span class="sourceLineNo">1626</span> */<a name="line.1626"></a> |
| <span class="sourceLineNo">1627</span> public void setClosing(boolean closing) {<a name="line.1627"></a> |
| <span class="sourceLineNo">1628</span> this.closing.set(closing);<a name="line.1628"></a> |
| <span class="sourceLineNo">1629</span> }<a name="line.1629"></a> |
| <span class="sourceLineNo">1630</span><a name="line.1630"></a> |
| <span class="sourceLineNo">1631</span> /**<a name="line.1631"></a> |
| <span class="sourceLineNo">1632</span> * The {@link HRegion#doClose} will block forever if someone tries proving the dead lock via the unit test.<a name="line.1632"></a> |
| <span class="sourceLineNo">1633</span> * Instead of blocking, the {@link HRegion#doClose} will throw exception if you set the timeout.<a name="line.1633"></a> |
| <span class="sourceLineNo">1634</span> * @param timeoutForWriteLock the second time to wait for the write lock in {@link HRegion#doClose}<a name="line.1634"></a> |
| <span class="sourceLineNo">1635</span> */<a name="line.1635"></a> |
| <span class="sourceLineNo">1636</span> public void setTimeoutForWriteLock(long timeoutForWriteLock) {<a name="line.1636"></a> |
| <span class="sourceLineNo">1637</span> assert timeoutForWriteLock >= 0;<a name="line.1637"></a> |
| <span class="sourceLineNo">1638</span> this.timeoutForWriteLock = timeoutForWriteLock;<a name="line.1638"></a> |
| <span class="sourceLineNo">1639</span> }<a name="line.1639"></a> |
| <span class="sourceLineNo">1640</span><a name="line.1640"></a> |
| <span class="sourceLineNo">1641</span> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UL_UNRELEASED_LOCK_EXCEPTION_PATH",<a name="line.1641"></a> |
| <span class="sourceLineNo">1642</span> justification="I think FindBugs is confused")<a name="line.1642"></a> |
| <span class="sourceLineNo">1643</span> private Map<byte[], List<HStoreFile>> doClose(boolean abort, MonitoredTask status)<a name="line.1643"></a> |
| <span class="sourceLineNo">1644</span> throws IOException {<a name="line.1644"></a> |
| <span class="sourceLineNo">1645</span> if (isClosed()) {<a name="line.1645"></a> |
| <span class="sourceLineNo">1646</span> LOG.warn("Region " + this + " already closed");<a name="line.1646"></a> |
| <span class="sourceLineNo">1647</span> return null;<a name="line.1647"></a> |
| <span class="sourceLineNo">1648</span> }<a name="line.1648"></a> |
| <span class="sourceLineNo">1649</span><a name="line.1649"></a> |
| <span class="sourceLineNo">1650</span> if (coprocessorHost != null) {<a name="line.1650"></a> |
| <span class="sourceLineNo">1651</span> status.setStatus("Running coprocessor pre-close hooks");<a name="line.1651"></a> |
| <span class="sourceLineNo">1652</span> this.coprocessorHost.preClose(abort);<a name="line.1652"></a> |
| <span class="sourceLineNo">1653</span> }<a name="line.1653"></a> |
| <span class="sourceLineNo">1654</span> status.setStatus("Disabling compacts and flushes for region");<a name="line.1654"></a> |
| <span class="sourceLineNo">1655</span> boolean canFlush = true;<a name="line.1655"></a> |
| <span class="sourceLineNo">1656</span> synchronized (writestate) {<a name="line.1656"></a> |
| <span class="sourceLineNo">1657</span> // Disable compacting and flushing by background threads for this<a name="line.1657"></a> |
| <span class="sourceLineNo">1658</span> // region.<a name="line.1658"></a> |
| <span class="sourceLineNo">1659</span> canFlush = !writestate.readOnly;<a name="line.1659"></a> |
| <span class="sourceLineNo">1660</span> writestate.writesEnabled = false;<a name="line.1660"></a> |
| <span class="sourceLineNo">1661</span> LOG.debug("Closing {}, disabling compactions & flushes",<a name="line.1661"></a> |
| <span class="sourceLineNo">1662</span> this.getRegionInfo().getEncodedName());<a name="line.1662"></a> |
| <span class="sourceLineNo">1663</span> waitForFlushesAndCompactions();<a name="line.1663"></a> |
| <span class="sourceLineNo">1664</span> }<a name="line.1664"></a> |
| <span class="sourceLineNo">1665</span> // If we were not just flushing, is it worth doing a preflush...one<a name="line.1665"></a> |
| <span class="sourceLineNo">1666</span> // that will clear out of the bulk of the memstore before we put up<a name="line.1666"></a> |
| <span class="sourceLineNo">1667</span> // the close flag?<a name="line.1667"></a> |
| <span class="sourceLineNo">1668</span> if (!abort && worthPreFlushing() && canFlush) {<a name="line.1668"></a> |
| <span class="sourceLineNo">1669</span> status.setStatus("Pre-flushing region before close");<a name="line.1669"></a> |
| <span class="sourceLineNo">1670</span> LOG.info("Running close preflush of {}", this.getRegionInfo().getEncodedName());<a name="line.1670"></a> |
| <span class="sourceLineNo">1671</span> try {<a name="line.1671"></a> |
| <span class="sourceLineNo">1672</span> internalFlushcache(status);<a name="line.1672"></a> |
| <span class="sourceLineNo">1673</span> } catch (IOException ioe) {<a name="line.1673"></a> |
| <span class="sourceLineNo">1674</span> // Failed to flush the region. Keep going.<a name="line.1674"></a> |
| <span class="sourceLineNo">1675</span> status.setStatus("Failed pre-flush " + this + "; " + ioe.getMessage());<a name="line.1675"></a> |
| <span class="sourceLineNo">1676</span> }<a name="line.1676"></a> |
| <span class="sourceLineNo">1677</span> }<a name="line.1677"></a> |
| <span class="sourceLineNo">1678</span><a name="line.1678"></a> |
| <span class="sourceLineNo">1679</span> // Set the closing flag<a name="line.1679"></a> |
| <span class="sourceLineNo">1680</span> // From this point new arrivals at the region lock will get NSRE.<a name="line.1680"></a> |
| <span class="sourceLineNo">1681</span><a name="line.1681"></a> |
| <span class="sourceLineNo">1682</span> this.closing.set(true);<a name="line.1682"></a> |
| <span class="sourceLineNo">1683</span> LOG.info("Closing region {}", this);<a name="line.1683"></a> |
| <span class="sourceLineNo">1684</span><a name="line.1684"></a> |
| <span class="sourceLineNo">1685</span> // Acquire the close lock<a name="line.1685"></a> |
| <span class="sourceLineNo">1686</span><a name="line.1686"></a> |
| <span class="sourceLineNo">1687</span> // The configuration parameter CLOSE_WAIT_ABORT is overloaded to enable both<a name="line.1687"></a> |
| <span class="sourceLineNo">1688</span> // the new regionserver abort condition and interrupts for running requests.<a name="line.1688"></a> |
| <span class="sourceLineNo">1689</span> // If CLOSE_WAIT_ABORT is not enabled there is no change from earlier behavior,<a name="line.1689"></a> |
| <span class="sourceLineNo">1690</span> // we will not attempt to interrupt threads servicing requests nor crash out<a name="line.1690"></a> |
| <span class="sourceLineNo">1691</span> // the regionserver if something remains stubborn.<a name="line.1691"></a> |
| <span class="sourceLineNo">1692</span><a name="line.1692"></a> |
| <span class="sourceLineNo">1693</span> final boolean canAbort = conf.getBoolean(CLOSE_WAIT_ABORT, DEFAULT_CLOSE_WAIT_ABORT);<a name="line.1693"></a> |
| <span class="sourceLineNo">1694</span> boolean useTimedWait = false;<a name="line.1694"></a> |
| <span class="sourceLineNo">1695</span> if (timeoutForWriteLock != null && timeoutForWriteLock != Long.MAX_VALUE) {<a name="line.1695"></a> |
| <span class="sourceLineNo">1696</span> // convert legacy use of timeoutForWriteLock in seconds to new use in millis<a name="line.1696"></a> |
| <span class="sourceLineNo">1697</span> timeoutForWriteLock = TimeUnit.SECONDS.toMillis(timeoutForWriteLock);<a name="line.1697"></a> |
| <span class="sourceLineNo">1698</span> useTimedWait = true;<a name="line.1698"></a> |
| <span class="sourceLineNo">1699</span> } else if (canAbort) {<a name="line.1699"></a> |
| <span class="sourceLineNo">1700</span> timeoutForWriteLock = conf.getLong(CLOSE_WAIT_TIME, DEFAULT_CLOSE_WAIT_TIME);<a name="line.1700"></a> |
| <span class="sourceLineNo">1701</span> useTimedWait = true;<a name="line.1701"></a> |
| <span class="sourceLineNo">1702</span> }<a name="line.1702"></a> |
| <span class="sourceLineNo">1703</span> if (LOG.isDebugEnabled()) {<a name="line.1703"></a> |
| <span class="sourceLineNo">1704</span> LOG.debug((useTimedWait ? "Time limited wait" : "Waiting without time limit") +<a name="line.1704"></a> |
| <span class="sourceLineNo">1705</span> " for close lock on " + this);<a name="line.1705"></a> |
| <span class="sourceLineNo">1706</span> }<a name="line.1706"></a> |
| <span class="sourceLineNo">1707</span> final long closeWaitInterval = conf.getLong(CLOSE_WAIT_INTERVAL, DEFAULT_CLOSE_WAIT_INTERVAL);<a name="line.1707"></a> |
| <span class="sourceLineNo">1708</span> long elapsedWaitTime = 0;<a name="line.1708"></a> |
| <span class="sourceLineNo">1709</span> if (useTimedWait) {<a name="line.1709"></a> |
| <span class="sourceLineNo">1710</span> // Sanity check configuration<a name="line.1710"></a> |
| <span class="sourceLineNo">1711</span> long remainingWaitTime = timeoutForWriteLock;<a name="line.1711"></a> |
| <span class="sourceLineNo">1712</span> if (remainingWaitTime < closeWaitInterval) {<a name="line.1712"></a> |
| <span class="sourceLineNo">1713</span> LOG.warn("Time limit for close wait of " + timeoutForWriteLock +<a name="line.1713"></a> |
| <span class="sourceLineNo">1714</span> " ms is less than the configured lock acquisition wait interval " +<a name="line.1714"></a> |
| <span class="sourceLineNo">1715</span> closeWaitInterval + " ms, using wait interval as time limit");<a name="line.1715"></a> |
| <span class="sourceLineNo">1716</span> remainingWaitTime = closeWaitInterval;<a name="line.1716"></a> |
| <span class="sourceLineNo">1717</span> }<a name="line.1717"></a> |
| <span class="sourceLineNo">1718</span> boolean acquired = false;<a name="line.1718"></a> |
| <span class="sourceLineNo">1719</span> do {<a name="line.1719"></a> |
| <span class="sourceLineNo">1720</span> long start = EnvironmentEdgeManager.currentTime();<a name="line.1720"></a> |
| <span class="sourceLineNo">1721</span> try {<a name="line.1721"></a> |
| <span class="sourceLineNo">1722</span> acquired = lock.writeLock().tryLock(Math.min(remainingWaitTime, closeWaitInterval),<a name="line.1722"></a> |
| <span class="sourceLineNo">1723</span> TimeUnit.MILLISECONDS);<a name="line.1723"></a> |
| <span class="sourceLineNo">1724</span> } catch (InterruptedException e) {<a name="line.1724"></a> |
| <span class="sourceLineNo">1725</span> // Interrupted waiting for close lock. More likely the server is shutting down, not<a name="line.1725"></a> |
| <span class="sourceLineNo">1726</span> // normal operation, so aborting upon interrupt while waiting on this lock would not<a name="line.1726"></a> |
| <span class="sourceLineNo">1727</span> // provide much value. Throw an IOE (as IIOE) like we would in the case where we<a name="line.1727"></a> |
| <span class="sourceLineNo">1728</span> // fail to acquire the lock.<a name="line.1728"></a> |
| <span class="sourceLineNo">1729</span> String msg = "Interrupted while waiting for close lock on " + this;<a name="line.1729"></a> |
| <span class="sourceLineNo">1730</span> LOG.warn(msg, e);<a name="line.1730"></a> |
| <span class="sourceLineNo">1731</span> throw (InterruptedIOException) new InterruptedIOException(msg).initCause(e);<a name="line.1731"></a> |
| <span class="sourceLineNo">1732</span> }<a name="line.1732"></a> |
| <span class="sourceLineNo">1733</span> long elapsed = EnvironmentEdgeManager.currentTime() - start;<a name="line.1733"></a> |
| <span class="sourceLineNo">1734</span> elapsedWaitTime += elapsed;<a name="line.1734"></a> |
| <span class="sourceLineNo">1735</span> remainingWaitTime -= elapsed;<a name="line.1735"></a> |
| <span class="sourceLineNo">1736</span> if (canAbort && !acquired && remainingWaitTime > 0) {<a name="line.1736"></a> |
| <span class="sourceLineNo">1737</span> // Before we loop to wait again, interrupt all region operations that might<a name="line.1737"></a> |
| <span class="sourceLineNo">1738</span> // still be in progress, to encourage them to break out of waiting states or<a name="line.1738"></a> |
| <span class="sourceLineNo">1739</span> // inner loops, throw an exception to clients, and release the read lock via<a name="line.1739"></a> |
| <span class="sourceLineNo">1740</span> // endRegionOperation.<a name="line.1740"></a> |
| <span class="sourceLineNo">1741</span> if (LOG.isDebugEnabled()) {<a name="line.1741"></a> |
| <span class="sourceLineNo">1742</span> LOG.debug("Interrupting region operations after waiting for close lock for " +<a name="line.1742"></a> |
| <span class="sourceLineNo">1743</span> elapsedWaitTime + " ms on " + this + ", " + remainingWaitTime +<a name="line.1743"></a> |
| <span class="sourceLineNo">1744</span> " ms remaining");<a name="line.1744"></a> |
| <span class="sourceLineNo">1745</span> }<a name="line.1745"></a> |
| <span class="sourceLineNo">1746</span> interruptRegionOperations();<a name="line.1746"></a> |
| <span class="sourceLineNo">1747</span> }<a name="line.1747"></a> |
| <span class="sourceLineNo">1748</span> } while (!acquired && remainingWaitTime > 0);<a name="line.1748"></a> |
| <span class="sourceLineNo">1749</span><a name="line.1749"></a> |
| <span class="sourceLineNo">1750</span> // If we fail to acquire the lock, trigger an abort if we can; otherwise throw an IOE<a name="line.1750"></a> |
| <span class="sourceLineNo">1751</span> // to let the caller know we could not proceed with the close.<a name="line.1751"></a> |
| <span class="sourceLineNo">1752</span> if (!acquired) {<a name="line.1752"></a> |
| <span class="sourceLineNo">1753</span> String msg = "Failed to acquire close lock on " + this + " after waiting " +<a name="line.1753"></a> |
| <span class="sourceLineNo">1754</span> elapsedWaitTime + " ms";<a name="line.1754"></a> |
| <span class="sourceLineNo">1755</span> LOG.error(msg);<a name="line.1755"></a> |
| <span class="sourceLineNo">1756</span> if (canAbort) {<a name="line.1756"></a> |
| <span class="sourceLineNo">1757</span> // If we failed to acquire the write lock, abort the server<a name="line.1757"></a> |
| <span class="sourceLineNo">1758</span> rsServices.abort(msg, null);<a name="line.1758"></a> |
| <span class="sourceLineNo">1759</span> }<a name="line.1759"></a> |
| <span class="sourceLineNo">1760</span> throw new IOException(msg);<a name="line.1760"></a> |
| <span class="sourceLineNo">1761</span> }<a name="line.1761"></a> |
| <span class="sourceLineNo">1762</span><a name="line.1762"></a> |
| <span class="sourceLineNo">1763</span> } else {<a name="line.1763"></a> |
| <span class="sourceLineNo">1764</span><a name="line.1764"></a> |
| <span class="sourceLineNo">1765</span> long start = EnvironmentEdgeManager.currentTime();<a name="line.1765"></a> |
| <span class="sourceLineNo">1766</span> lock.writeLock().lock();<a name="line.1766"></a> |
| <span class="sourceLineNo">1767</span> elapsedWaitTime = EnvironmentEdgeManager.currentTime() - start;<a name="line.1767"></a> |
| <span class="sourceLineNo">1768</span><a name="line.1768"></a> |
| <span class="sourceLineNo">1769</span> }<a name="line.1769"></a> |
| <span class="sourceLineNo">1770</span><a name="line.1770"></a> |
| <span class="sourceLineNo">1771</span> if (LOG.isDebugEnabled()) {<a name="line.1771"></a> |
| <span class="sourceLineNo">1772</span> LOG.debug("Acquired close lock on " + this + " after waiting " +<a name="line.1772"></a> |
| <span class="sourceLineNo">1773</span> elapsedWaitTime + " ms");<a name="line.1773"></a> |
| <span class="sourceLineNo">1774</span> }<a name="line.1774"></a> |
| <span class="sourceLineNo">1775</span><a name="line.1775"></a> |
| <span class="sourceLineNo">1776</span> status.setStatus("Disabling writes for close");<a name="line.1776"></a> |
| <span class="sourceLineNo">1777</span> try {<a name="line.1777"></a> |
| <span class="sourceLineNo">1778</span> if (this.isClosed()) {<a name="line.1778"></a> |
| <span class="sourceLineNo">1779</span> status.abort("Already got closed by another process");<a name="line.1779"></a> |
| <span class="sourceLineNo">1780</span> // SplitTransaction handles the null<a name="line.1780"></a> |
| <span class="sourceLineNo">1781</span> return null;<a name="line.1781"></a> |
| <span class="sourceLineNo">1782</span> }<a name="line.1782"></a> |
| <span class="sourceLineNo">1783</span> LOG.debug("Updates disabled for region " + this);<a name="line.1783"></a> |
| <span class="sourceLineNo">1784</span> // Don't flush the cache if we are aborting<a name="line.1784"></a> |
| <span class="sourceLineNo">1785</span> if (!abort && canFlush) {<a name="line.1785"></a> |
| <span class="sourceLineNo">1786</span> int failedfFlushCount = 0;<a name="line.1786"></a> |
| <span class="sourceLineNo">1787</span> int flushCount = 0;<a name="line.1787"></a> |
| <span class="sourceLineNo">1788</span> long tmp = 0;<a name="line.1788"></a> |
| <span class="sourceLineNo">1789</span> long remainingSize = this.memStoreSizing.getDataSize();<a name="line.1789"></a> |
| <span class="sourceLineNo">1790</span> while (remainingSize > 0) {<a name="line.1790"></a> |
| <span class="sourceLineNo">1791</span> try {<a name="line.1791"></a> |
| <span class="sourceLineNo">1792</span> internalFlushcache(status);<a name="line.1792"></a> |
| <span class="sourceLineNo">1793</span> if(flushCount >0) {<a name="line.1793"></a> |
| <span class="sourceLineNo">1794</span> LOG.info("Running extra flush, " + flushCount +<a name="line.1794"></a> |
| <span class="sourceLineNo">1795</span> " (carrying snapshot?) " + this);<a name="line.1795"></a> |
| <span class="sourceLineNo">1796</span> }<a name="line.1796"></a> |
| <span class="sourceLineNo">1797</span> flushCount++;<a name="line.1797"></a> |
| <span class="sourceLineNo">1798</span> tmp = this.memStoreSizing.getDataSize();<a name="line.1798"></a> |
| <span class="sourceLineNo">1799</span> if (tmp >= remainingSize) {<a name="line.1799"></a> |
| <span class="sourceLineNo">1800</span> failedfFlushCount++;<a name="line.1800"></a> |
| <span class="sourceLineNo">1801</span> }<a name="line.1801"></a> |
| <span class="sourceLineNo">1802</span> remainingSize = tmp;<a name="line.1802"></a> |
| <span class="sourceLineNo">1803</span> if (failedfFlushCount > 5) {<a name="line.1803"></a> |
| <span class="sourceLineNo">1804</span> // If we failed 5 times and are unable to clear memory, abort<a name="line.1804"></a> |
| <span class="sourceLineNo">1805</span> // so we do not lose data<a name="line.1805"></a> |
| <span class="sourceLineNo">1806</span> throw new DroppedSnapshotException("Failed clearing memory after " +<a name="line.1806"></a> |
| <span class="sourceLineNo">1807</span> flushCount + " attempts on region: " +<a name="line.1807"></a> |
| <span class="sourceLineNo">1808</span> Bytes.toStringBinary(getRegionInfo().getRegionName()));<a name="line.1808"></a> |
| <span class="sourceLineNo">1809</span> }<a name="line.1809"></a> |
| <span class="sourceLineNo">1810</span> } catch (IOException ioe) {<a name="line.1810"></a> |
| <span class="sourceLineNo">1811</span> status.setStatus("Failed flush " + this + ", putting online again");<a name="line.1811"></a> |
| <span class="sourceLineNo">1812</span> synchronized (writestate) {<a name="line.1812"></a> |
| <span class="sourceLineNo">1813</span> writestate.writesEnabled = true;<a name="line.1813"></a> |
| <span class="sourceLineNo">1814</span> }<a name="line.1814"></a> |
| <span class="sourceLineNo">1815</span> // Have to throw to upper layers. I can't abort server from here.<a name="line.1815"></a> |
| <span class="sourceLineNo">1816</span> throw ioe;<a name="line.1816"></a> |
| <span class="sourceLineNo">1817</span> }<a name="line.1817"></a> |
| <span class="sourceLineNo">1818</span> }<a name="line.1818"></a> |
| <span class="sourceLineNo">1819</span> }<a name="line.1819"></a> |
| <span class="sourceLineNo">1820</span><a name="line.1820"></a> |
| <span class="sourceLineNo">1821</span> Map<byte[], List<HStoreFile>> result = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.1821"></a> |
| <span class="sourceLineNo">1822</span> if (!stores.isEmpty()) {<a name="line.1822"></a> |
| <span class="sourceLineNo">1823</span> // initialize the thread pool for closing stores in parallel.<a name="line.1823"></a> |
| <span class="sourceLineNo">1824</span> ThreadPoolExecutor storeCloserThreadPool =<a name="line.1824"></a> |
| <span class="sourceLineNo">1825</span> getStoreOpenAndCloseThreadPool("StoreCloser-" +<a name="line.1825"></a> |
| <span class="sourceLineNo">1826</span> getRegionInfo().getRegionNameAsString());<a name="line.1826"></a> |
| <span class="sourceLineNo">1827</span> CompletionService<Pair<byte[], Collection<HStoreFile>>> completionService =<a name="line.1827"></a> |
| <span class="sourceLineNo">1828</span> new ExecutorCompletionService<>(storeCloserThreadPool);<a name="line.1828"></a> |
| <span class="sourceLineNo">1829</span><a name="line.1829"></a> |
| <span class="sourceLineNo">1830</span> // close each store in parallel<a name="line.1830"></a> |
| <span class="sourceLineNo">1831</span> for (HStore store : stores.values()) {<a name="line.1831"></a> |
| <span class="sourceLineNo">1832</span> MemStoreSize mss = store.getFlushableSize();<a name="line.1832"></a> |
| <span class="sourceLineNo">1833</span> if (!(abort || mss.getDataSize() == 0 || writestate.readOnly)) {<a name="line.1833"></a> |
| <span class="sourceLineNo">1834</span> if (getRegionServerServices() != null) {<a name="line.1834"></a> |
| <span class="sourceLineNo">1835</span> getRegionServerServices().abort("Assertion failed while closing store "<a name="line.1835"></a> |
| <span class="sourceLineNo">1836</span> + getRegionInfo().getRegionNameAsString() + " " + store<a name="line.1836"></a> |
| <span class="sourceLineNo">1837</span> + ". flushableSize expected=0, actual={" + mss<a name="line.1837"></a> |
| <span class="sourceLineNo">1838</span> + "}. Current memStoreSize=" + this.memStoreSizing.getMemStoreSize() +<a name="line.1838"></a> |
| <span class="sourceLineNo">1839</span> ". Maybe a coprocessor "<a name="line.1839"></a> |
| <span class="sourceLineNo">1840</span> + "operation failed and left the memstore in a partially updated state.", null);<a name="line.1840"></a> |
| <span class="sourceLineNo">1841</span> }<a name="line.1841"></a> |
| <span class="sourceLineNo">1842</span> }<a name="line.1842"></a> |
| <span class="sourceLineNo">1843</span> completionService<a name="line.1843"></a> |
| <span class="sourceLineNo">1844</span> .submit(new Callable<Pair<byte[], Collection<HStoreFile>>>() {<a name="line.1844"></a> |
| <span class="sourceLineNo">1845</span> @Override<a name="line.1845"></a> |
| <span class="sourceLineNo">1846</span> public Pair<byte[], Collection<HStoreFile>> call() throws IOException {<a name="line.1846"></a> |
| <span class="sourceLineNo">1847</span> return new Pair<>(store.getColumnFamilyDescriptor().getName(), store.close());<a name="line.1847"></a> |
| <span class="sourceLineNo">1848</span> }<a name="line.1848"></a> |
| <span class="sourceLineNo">1849</span> });<a name="line.1849"></a> |
| <span class="sourceLineNo">1850</span> }<a name="line.1850"></a> |
| <span class="sourceLineNo">1851</span> try {<a name="line.1851"></a> |
| <span class="sourceLineNo">1852</span> for (int i = 0; i < stores.size(); i++) {<a name="line.1852"></a> |
| <span class="sourceLineNo">1853</span> Future<Pair<byte[], Collection<HStoreFile>>> future = completionService.take();<a name="line.1853"></a> |
| <span class="sourceLineNo">1854</span> Pair<byte[], Collection<HStoreFile>> storeFiles = future.get();<a name="line.1854"></a> |
| <span class="sourceLineNo">1855</span> List<HStoreFile> familyFiles = result.get(storeFiles.getFirst());<a name="line.1855"></a> |
| <span class="sourceLineNo">1856</span> if (familyFiles == null) {<a name="line.1856"></a> |
| <span class="sourceLineNo">1857</span> familyFiles = new ArrayList<>();<a name="line.1857"></a> |
| <span class="sourceLineNo">1858</span> result.put(storeFiles.getFirst(), familyFiles);<a name="line.1858"></a> |
| <span class="sourceLineNo">1859</span> }<a name="line.1859"></a> |
| <span class="sourceLineNo">1860</span> familyFiles.addAll(storeFiles.getSecond());<a name="line.1860"></a> |
| <span class="sourceLineNo">1861</span> }<a name="line.1861"></a> |
| <span class="sourceLineNo">1862</span> } catch (InterruptedException e) {<a name="line.1862"></a> |
| <span class="sourceLineNo">1863</span> throw throwOnInterrupt(e);<a name="line.1863"></a> |
| <span class="sourceLineNo">1864</span> } catch (ExecutionException e) {<a name="line.1864"></a> |
| <span class="sourceLineNo">1865</span> Throwable cause = e.getCause();<a name="line.1865"></a> |
| <span class="sourceLineNo">1866</span> if (cause instanceof IOException) {<a name="line.1866"></a> |
| <span class="sourceLineNo">1867</span> throw (IOException) cause;<a name="line.1867"></a> |
| <span class="sourceLineNo">1868</span> }<a name="line.1868"></a> |
| <span class="sourceLineNo">1869</span> throw new IOException(cause);<a name="line.1869"></a> |
| <span class="sourceLineNo">1870</span> } finally {<a name="line.1870"></a> |
| <span class="sourceLineNo">1871</span> storeCloserThreadPool.shutdownNow();<a name="line.1871"></a> |
| <span class="sourceLineNo">1872</span> }<a name="line.1872"></a> |
| <span class="sourceLineNo">1873</span> }<a name="line.1873"></a> |
| <span class="sourceLineNo">1874</span><a name="line.1874"></a> |
| <span class="sourceLineNo">1875</span> status.setStatus("Writing region close event to WAL");<a name="line.1875"></a> |
| <span class="sourceLineNo">1876</span> // Always write close marker to wal even for read only table. This is not a big problem as we<a name="line.1876"></a> |
| <span class="sourceLineNo">1877</span> // do not write any data into the region; it is just a meta edit in the WAL file.<a name="line.1877"></a> |
| <span class="sourceLineNo">1878</span> if (!abort && wal != null && getRegionServerServices() != null &&<a name="line.1878"></a> |
| <span class="sourceLineNo">1879</span> RegionReplicaUtil.isDefaultReplica(getRegionInfo())) {<a name="line.1879"></a> |
| <span class="sourceLineNo">1880</span> writeRegionCloseMarker(wal);<a name="line.1880"></a> |
| <span class="sourceLineNo">1881</span> }<a name="line.1881"></a> |
| <span class="sourceLineNo">1882</span><a name="line.1882"></a> |
| <span class="sourceLineNo">1883</span> this.closed.set(true);<a name="line.1883"></a> |
| <span class="sourceLineNo">1884</span> if (!canFlush) {<a name="line.1884"></a> |
| <span class="sourceLineNo">1885</span> decrMemStoreSize(this.memStoreSizing.getMemStoreSize());<a name="line.1885"></a> |
| <span class="sourceLineNo">1886</span> } else if (this.memStoreSizing.getDataSize() != 0) {<a name="line.1886"></a> |
| <span class="sourceLineNo">1887</span> LOG.error("Memstore data size is {} in region {}", this.memStoreSizing.getDataSize(), this);<a name="line.1887"></a> |
| <span class="sourceLineNo">1888</span> }<a name="line.1888"></a> |
| <span class="sourceLineNo">1889</span> if (coprocessorHost != null) {<a name="line.1889"></a> |
| <span class="sourceLineNo">1890</span> status.setStatus("Running coprocessor post-close hooks");<a name="line.1890"></a> |
| <span class="sourceLineNo">1891</span> this.coprocessorHost.postClose(abort);<a name="line.1891"></a> |
| <span class="sourceLineNo">1892</span> }<a name="line.1892"></a> |
| <span class="sourceLineNo">1893</span> if (this.metricsRegion != null) {<a name="line.1893"></a> |
| <span class="sourceLineNo">1894</span> this.metricsRegion.close();<a name="line.1894"></a> |
| <span class="sourceLineNo">1895</span> }<a name="line.1895"></a> |
| <span class="sourceLineNo">1896</span> if (this.metricsRegionWrapper != null) {<a name="line.1896"></a> |
| <span class="sourceLineNo">1897</span> Closeables.close(this.metricsRegionWrapper, true);<a name="line.1897"></a> |
| <span class="sourceLineNo">1898</span> }<a name="line.1898"></a> |
| <span class="sourceLineNo">1899</span> status.markComplete("Closed");<a name="line.1899"></a> |
| <span class="sourceLineNo">1900</span> LOG.info("Closed {}", this);<a name="line.1900"></a> |
| <span class="sourceLineNo">1901</span> return result;<a name="line.1901"></a> |
| <span class="sourceLineNo">1902</span> } finally {<a name="line.1902"></a> |
| <span class="sourceLineNo">1903</span> lock.writeLock().unlock();<a name="line.1903"></a> |
| <span class="sourceLineNo">1904</span> }<a name="line.1904"></a> |
| <span class="sourceLineNo">1905</span> }<a name="line.1905"></a> |
| <span class="sourceLineNo">1906</span><a name="line.1906"></a> |
| <span class="sourceLineNo">1907</span> /** Wait for all current flushes and compactions of the region to complete */<a name="line.1907"></a> |
| <span class="sourceLineNo">1908</span> // TODO HBASE-18906. Check the usage (if any) in Phoenix and expose this or give alternate way for<a name="line.1908"></a> |
| <span class="sourceLineNo">1909</span> // Phoenix needs.<a name="line.1909"></a> |
| <span class="sourceLineNo">1910</span> public void waitForFlushesAndCompactions() {<a name="line.1910"></a> |
| <span class="sourceLineNo">1911</span> synchronized (writestate) {<a name="line.1911"></a> |
| <span class="sourceLineNo">1912</span> if (this.writestate.readOnly) {<a name="line.1912"></a> |
| <span class="sourceLineNo">1913</span> // we should not wait for replayed flushed if we are read only (for example in case the<a name="line.1913"></a> |
| <span class="sourceLineNo">1914</span> // region is a secondary replica).<a name="line.1914"></a> |
| <span class="sourceLineNo">1915</span> return;<a name="line.1915"></a> |
| <span class="sourceLineNo">1916</span> }<a name="line.1916"></a> |
| <span class="sourceLineNo">1917</span> boolean interrupted = false;<a name="line.1917"></a> |
| <span class="sourceLineNo">1918</span> try {<a name="line.1918"></a> |
| <span class="sourceLineNo">1919</span> while (writestate.compacting.get() > 0 || writestate.flushing) {<a name="line.1919"></a> |
| <span class="sourceLineNo">1920</span> LOG.debug("waiting for " + writestate.compacting + " compactions"<a name="line.1920"></a> |
| <span class="sourceLineNo">1921</span> + (writestate.flushing ? " & cache flush" : "") + " to complete for region " + this);<a name="line.1921"></a> |
| <span class="sourceLineNo">1922</span> try {<a name="line.1922"></a> |
| <span class="sourceLineNo">1923</span> writestate.wait();<a name="line.1923"></a> |
| <span class="sourceLineNo">1924</span> } catch (InterruptedException iex) {<a name="line.1924"></a> |
| <span class="sourceLineNo">1925</span> // essentially ignore and propagate the interrupt back up<a name="line.1925"></a> |
| <span class="sourceLineNo">1926</span> LOG.warn("Interrupted while waiting in region {}", this);<a name="line.1926"></a> |
| <span class="sourceLineNo">1927</span> interrupted = true;<a name="line.1927"></a> |
| <span class="sourceLineNo">1928</span> break;<a name="line.1928"></a> |
| <span class="sourceLineNo">1929</span> }<a name="line.1929"></a> |
| <span class="sourceLineNo">1930</span> }<a name="line.1930"></a> |
| <span class="sourceLineNo">1931</span> } finally {<a name="line.1931"></a> |
| <span class="sourceLineNo">1932</span> if (interrupted) {<a name="line.1932"></a> |
| <span class="sourceLineNo">1933</span> Thread.currentThread().interrupt();<a name="line.1933"></a> |
| <span class="sourceLineNo">1934</span> }<a name="line.1934"></a> |
| <span class="sourceLineNo">1935</span> }<a name="line.1935"></a> |
| <span class="sourceLineNo">1936</span> }<a name="line.1936"></a> |
| <span class="sourceLineNo">1937</span> }<a name="line.1937"></a> |
| <span class="sourceLineNo">1938</span><a name="line.1938"></a> |
| <span class="sourceLineNo">1939</span> /**<a name="line.1939"></a> |
| <span class="sourceLineNo">1940</span> * Wait for all current flushes of the region to complete<a name="line.1940"></a> |
| <span class="sourceLineNo">1941</span> */<a name="line.1941"></a> |
| <span class="sourceLineNo">1942</span> public void waitForFlushes() {<a name="line.1942"></a> |
| <span class="sourceLineNo">1943</span> waitForFlushes(0);// Unbound wait<a name="line.1943"></a> |
| <span class="sourceLineNo">1944</span> }<a name="line.1944"></a> |
| <span class="sourceLineNo">1945</span><a name="line.1945"></a> |
| <span class="sourceLineNo">1946</span> @Override<a name="line.1946"></a> |
| <span class="sourceLineNo">1947</span> public boolean waitForFlushes(long timeout) {<a name="line.1947"></a> |
| <span class="sourceLineNo">1948</span> synchronized (writestate) {<a name="line.1948"></a> |
| <span class="sourceLineNo">1949</span> if (this.writestate.readOnly) {<a name="line.1949"></a> |
| <span class="sourceLineNo">1950</span> // we should not wait for replayed flushed if we are read only (for example in case the<a name="line.1950"></a> |
| <span class="sourceLineNo">1951</span> // region is a secondary replica).<a name="line.1951"></a> |
| <span class="sourceLineNo">1952</span> return true;<a name="line.1952"></a> |
| <span class="sourceLineNo">1953</span> }<a name="line.1953"></a> |
| <span class="sourceLineNo">1954</span> if (!writestate.flushing) return true;<a name="line.1954"></a> |
| <span class="sourceLineNo">1955</span> long start = System.currentTimeMillis();<a name="line.1955"></a> |
| <span class="sourceLineNo">1956</span> long duration = 0;<a name="line.1956"></a> |
| <span class="sourceLineNo">1957</span> boolean interrupted = false;<a name="line.1957"></a> |
| <span class="sourceLineNo">1958</span> LOG.debug("waiting for cache flush to complete for region " + this);<a name="line.1958"></a> |
| <span class="sourceLineNo">1959</span> try {<a name="line.1959"></a> |
| <span class="sourceLineNo">1960</span> while (writestate.flushing) {<a name="line.1960"></a> |
| <span class="sourceLineNo">1961</span> if (timeout > 0 && duration >= timeout) break;<a name="line.1961"></a> |
| <span class="sourceLineNo">1962</span> try {<a name="line.1962"></a> |
| <span class="sourceLineNo">1963</span> long toWait = timeout == 0 ? 0 : (timeout - duration);<a name="line.1963"></a> |
| <span class="sourceLineNo">1964</span> writestate.wait(toWait);<a name="line.1964"></a> |
| <span class="sourceLineNo">1965</span> } catch (InterruptedException iex) {<a name="line.1965"></a> |
| <span class="sourceLineNo">1966</span> // essentially ignore and propagate the interrupt back up<a name="line.1966"></a> |
| <span class="sourceLineNo">1967</span> LOG.warn("Interrupted while waiting in region {}", this);<a name="line.1967"></a> |
| <span class="sourceLineNo">1968</span> interrupted = true;<a name="line.1968"></a> |
| <span class="sourceLineNo">1969</span> break;<a name="line.1969"></a> |
| <span class="sourceLineNo">1970</span> } finally {<a name="line.1970"></a> |
| <span class="sourceLineNo">1971</span> duration = System.currentTimeMillis() - start;<a name="line.1971"></a> |
| <span class="sourceLineNo">1972</span> }<a name="line.1972"></a> |
| <span class="sourceLineNo">1973</span> }<a name="line.1973"></a> |
| <span class="sourceLineNo">1974</span> } finally {<a name="line.1974"></a> |
| <span class="sourceLineNo">1975</span> if (interrupted) {<a name="line.1975"></a> |
| <span class="sourceLineNo">1976</span> Thread.currentThread().interrupt();<a name="line.1976"></a> |
| <span class="sourceLineNo">1977</span> }<a name="line.1977"></a> |
| <span class="sourceLineNo">1978</span> }<a name="line.1978"></a> |
| <span class="sourceLineNo">1979</span> LOG.debug("Waited {} ms for region {} flush to complete", duration, this);<a name="line.1979"></a> |
| <span class="sourceLineNo">1980</span> return !(writestate.flushing);<a name="line.1980"></a> |
| <span class="sourceLineNo">1981</span> }<a name="line.1981"></a> |
| <span class="sourceLineNo">1982</span> }<a name="line.1982"></a> |
| <span class="sourceLineNo">1983</span><a name="line.1983"></a> |
| <span class="sourceLineNo">1984</span> private ThreadPoolExecutor getStoreOpenAndCloseThreadPool(<a name="line.1984"></a> |
| <span class="sourceLineNo">1985</span> final String threadNamePrefix) {<a name="line.1985"></a> |
| <span class="sourceLineNo">1986</span> int numStores = Math.max(1, this.htableDescriptor.getColumnFamilyCount());<a name="line.1986"></a> |
| <span class="sourceLineNo">1987</span> int maxThreads = Math.min(numStores,<a name="line.1987"></a> |
| <span class="sourceLineNo">1988</span> conf.getInt(HConstants.HSTORE_OPEN_AND_CLOSE_THREADS_MAX,<a name="line.1988"></a> |
| <span class="sourceLineNo">1989</span> HConstants.DEFAULT_HSTORE_OPEN_AND_CLOSE_THREADS_MAX));<a name="line.1989"></a> |
| <span class="sourceLineNo">1990</span> return getOpenAndCloseThreadPool(maxThreads, threadNamePrefix);<a name="line.1990"></a> |
| <span class="sourceLineNo">1991</span> }<a name="line.1991"></a> |
| <span class="sourceLineNo">1992</span><a name="line.1992"></a> |
| <span class="sourceLineNo">1993</span> ThreadPoolExecutor getStoreFileOpenAndCloseThreadPool(<a name="line.1993"></a> |
| <span class="sourceLineNo">1994</span> final String threadNamePrefix) {<a name="line.1994"></a> |
| <span class="sourceLineNo">1995</span> int numStores = Math.max(1, this.htableDescriptor.getColumnFamilyCount());<a name="line.1995"></a> |
| <span class="sourceLineNo">1996</span> int maxThreads = Math.max(1,<a name="line.1996"></a> |
| <span class="sourceLineNo">1997</span> conf.getInt(HConstants.HSTORE_OPEN_AND_CLOSE_THREADS_MAX,<a name="line.1997"></a> |
| <span class="sourceLineNo">1998</span> HConstants.DEFAULT_HSTORE_OPEN_AND_CLOSE_THREADS_MAX)<a name="line.1998"></a> |
| <span class="sourceLineNo">1999</span> / numStores);<a name="line.1999"></a> |
| <span class="sourceLineNo">2000</span> return getOpenAndCloseThreadPool(maxThreads, threadNamePrefix);<a name="line.2000"></a> |
| <span class="sourceLineNo">2001</span> }<a name="line.2001"></a> |
| <span class="sourceLineNo">2002</span><a name="line.2002"></a> |
| <span class="sourceLineNo">2003</span> private static ThreadPoolExecutor getOpenAndCloseThreadPool(int maxThreads,<a name="line.2003"></a> |
| <span class="sourceLineNo">2004</span> final String threadNamePrefix) {<a name="line.2004"></a> |
| <span class="sourceLineNo">2005</span> return Threads.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,<a name="line.2005"></a> |
| <span class="sourceLineNo">2006</span> new ThreadFactory() {<a name="line.2006"></a> |
| <span class="sourceLineNo">2007</span> private int count = 1;<a name="line.2007"></a> |
| <span class="sourceLineNo">2008</span><a name="line.2008"></a> |
| <span class="sourceLineNo">2009</span> @Override<a name="line.2009"></a> |
| <span class="sourceLineNo">2010</span> public Thread newThread(Runnable r) {<a name="line.2010"></a> |
| <span class="sourceLineNo">2011</span> return new Thread(r, threadNamePrefix + "-" + count++);<a name="line.2011"></a> |
| <span class="sourceLineNo">2012</span> }<a name="line.2012"></a> |
| <span class="sourceLineNo">2013</span> });<a name="line.2013"></a> |
| <span class="sourceLineNo">2014</span> }<a name="line.2014"></a> |
| <span class="sourceLineNo">2015</span><a name="line.2015"></a> |
| <span class="sourceLineNo">2016</span> /**<a name="line.2016"></a> |
| <span class="sourceLineNo">2017</span> * @return True if its worth doing a flush before we put up the close flag.<a name="line.2017"></a> |
| <span class="sourceLineNo">2018</span> */<a name="line.2018"></a> |
| <span class="sourceLineNo">2019</span> private boolean worthPreFlushing() {<a name="line.2019"></a> |
| <span class="sourceLineNo">2020</span> return this.memStoreSizing.getDataSize() ><a name="line.2020"></a> |
| <span class="sourceLineNo">2021</span> this.conf.getLong("hbase.hregion.preclose.flush.size", 1024 * 1024 * 5);<a name="line.2021"></a> |
| <span class="sourceLineNo">2022</span> }<a name="line.2022"></a> |
| <span class="sourceLineNo">2023</span><a name="line.2023"></a> |
| <span class="sourceLineNo">2024</span> //////////////////////////////////////////////////////////////////////////////<a name="line.2024"></a> |
| <span class="sourceLineNo">2025</span> // HRegion accessors<a name="line.2025"></a> |
| <span class="sourceLineNo">2026</span> //////////////////////////////////////////////////////////////////////////////<a name="line.2026"></a> |
| <span class="sourceLineNo">2027</span><a name="line.2027"></a> |
| <span class="sourceLineNo">2028</span> @Override<a name="line.2028"></a> |
| <span class="sourceLineNo">2029</span> public TableDescriptor getTableDescriptor() {<a name="line.2029"></a> |
| <span class="sourceLineNo">2030</span> return this.htableDescriptor;<a name="line.2030"></a> |
| <span class="sourceLineNo">2031</span> }<a name="line.2031"></a> |
| <span class="sourceLineNo">2032</span><a name="line.2032"></a> |
| <span class="sourceLineNo">2033</span> public void setTableDescriptor(TableDescriptor desc) {<a name="line.2033"></a> |
| <span class="sourceLineNo">2034</span> htableDescriptor = desc;<a name="line.2034"></a> |
| <span class="sourceLineNo">2035</span> }<a name="line.2035"></a> |
| <span class="sourceLineNo">2036</span><a name="line.2036"></a> |
| <span class="sourceLineNo">2037</span> /** @return WAL in use for this region */<a name="line.2037"></a> |
| <span class="sourceLineNo">2038</span> public WAL getWAL() {<a name="line.2038"></a> |
| <span class="sourceLineNo">2039</span> return this.wal;<a name="line.2039"></a> |
| <span class="sourceLineNo">2040</span> }<a name="line.2040"></a> |
| <span class="sourceLineNo">2041</span><a name="line.2041"></a> |
| <span class="sourceLineNo">2042</span> public BlockCache getBlockCache() {<a name="line.2042"></a> |
| <span class="sourceLineNo">2043</span> return this.blockCache;<a name="line.2043"></a> |
| <span class="sourceLineNo">2044</span> }<a name="line.2044"></a> |
| <span class="sourceLineNo">2045</span><a name="line.2045"></a> |
| <span class="sourceLineNo">2046</span> /**<a name="line.2046"></a> |
| <span class="sourceLineNo">2047</span> * Only used for unit test which doesn't start region server.<a name="line.2047"></a> |
| <span class="sourceLineNo">2048</span> */<a name="line.2048"></a> |
| <span class="sourceLineNo">2049</span> public void setBlockCache(BlockCache blockCache) {<a name="line.2049"></a> |
| <span class="sourceLineNo">2050</span> this.blockCache = blockCache;<a name="line.2050"></a> |
| <span class="sourceLineNo">2051</span> }<a name="line.2051"></a> |
| <span class="sourceLineNo">2052</span><a name="line.2052"></a> |
| <span class="sourceLineNo">2053</span> public MobFileCache getMobFileCache() {<a name="line.2053"></a> |
| <span class="sourceLineNo">2054</span> return this.mobFileCache;<a name="line.2054"></a> |
| <span class="sourceLineNo">2055</span> }<a name="line.2055"></a> |
| <span class="sourceLineNo">2056</span><a name="line.2056"></a> |
| <span class="sourceLineNo">2057</span> /**<a name="line.2057"></a> |
| <span class="sourceLineNo">2058</span> * Only used for unit test which doesn't start region server.<a name="line.2058"></a> |
| <span class="sourceLineNo">2059</span> */<a name="line.2059"></a> |
| <span class="sourceLineNo">2060</span> public void setMobFileCache(MobFileCache mobFileCache) {<a name="line.2060"></a> |
| <span class="sourceLineNo">2061</span> this.mobFileCache = mobFileCache;<a name="line.2061"></a> |
| <span class="sourceLineNo">2062</span> }<a name="line.2062"></a> |
| <span class="sourceLineNo">2063</span><a name="line.2063"></a> |
| <span class="sourceLineNo">2064</span> /**<a name="line.2064"></a> |
| <span class="sourceLineNo">2065</span> * @return split policy for this region.<a name="line.2065"></a> |
| <span class="sourceLineNo">2066</span> */<a name="line.2066"></a> |
| <span class="sourceLineNo">2067</span> RegionSplitPolicy getSplitPolicy() {<a name="line.2067"></a> |
| <span class="sourceLineNo">2068</span> return this.splitPolicy;<a name="line.2068"></a> |
| <span class="sourceLineNo">2069</span> }<a name="line.2069"></a> |
| <span class="sourceLineNo">2070</span><a name="line.2070"></a> |
| <span class="sourceLineNo">2071</span> /**<a name="line.2071"></a> |
| <span class="sourceLineNo">2072</span> * A split takes the config from the parent region & passes it to the daughter<a name="line.2072"></a> |
| <span class="sourceLineNo">2073</span> * region's constructor. If 'conf' was passed, you would end up using the HTD<a name="line.2073"></a> |
| <span class="sourceLineNo">2074</span> * of the parent region in addition to the new daughter HTD. Pass 'baseConf'<a name="line.2074"></a> |
| <span class="sourceLineNo">2075</span> * to the daughter regions to avoid this tricky dedupe problem.<a name="line.2075"></a> |
| <span class="sourceLineNo">2076</span> * @return Configuration object<a name="line.2076"></a> |
| <span class="sourceLineNo">2077</span> */<a name="line.2077"></a> |
| <span class="sourceLineNo">2078</span> Configuration getBaseConf() {<a name="line.2078"></a> |
| <span class="sourceLineNo">2079</span> return this.baseConf;<a name="line.2079"></a> |
| <span class="sourceLineNo">2080</span> }<a name="line.2080"></a> |
| <span class="sourceLineNo">2081</span><a name="line.2081"></a> |
| <span class="sourceLineNo">2082</span> /** @return {@link FileSystem} being used by this region */<a name="line.2082"></a> |
| <span class="sourceLineNo">2083</span> public FileSystem getFilesystem() {<a name="line.2083"></a> |
| <span class="sourceLineNo">2084</span> return fs.getFileSystem();<a name="line.2084"></a> |
| <span class="sourceLineNo">2085</span> }<a name="line.2085"></a> |
| <span class="sourceLineNo">2086</span><a name="line.2086"></a> |
| <span class="sourceLineNo">2087</span> /** @return the {@link HRegionFileSystem} used by this region */<a name="line.2087"></a> |
| <span class="sourceLineNo">2088</span> public HRegionFileSystem getRegionFileSystem() {<a name="line.2088"></a> |
| <span class="sourceLineNo">2089</span> return this.fs;<a name="line.2089"></a> |
| <span class="sourceLineNo">2090</span> }<a name="line.2090"></a> |
| <span class="sourceLineNo">2091</span><a name="line.2091"></a> |
| <span class="sourceLineNo">2092</span> /** @return the WAL {@link HRegionFileSystem} used by this region */<a name="line.2092"></a> |
| <span class="sourceLineNo">2093</span> HRegionWALFileSystem getRegionWALFileSystem() throws IOException {<a name="line.2093"></a> |
| <span class="sourceLineNo">2094</span> return new HRegionWALFileSystem(conf, getWalFileSystem(),<a name="line.2094"></a> |
| <span class="sourceLineNo">2095</span> CommonFSUtils.getWALTableDir(conf, htableDescriptor.getTableName()), fs.getRegionInfo());<a name="line.2095"></a> |
| <span class="sourceLineNo">2096</span> }<a name="line.2096"></a> |
| <span class="sourceLineNo">2097</span><a name="line.2097"></a> |
| <span class="sourceLineNo">2098</span> /** @return the WAL {@link FileSystem} being used by this region */<a name="line.2098"></a> |
| <span class="sourceLineNo">2099</span> FileSystem getWalFileSystem() throws IOException {<a name="line.2099"></a> |
| <span class="sourceLineNo">2100</span> if (walFS == null) {<a name="line.2100"></a> |
| <span class="sourceLineNo">2101</span> walFS = CommonFSUtils.getWALFileSystem(conf);<a name="line.2101"></a> |
| <span class="sourceLineNo">2102</span> }<a name="line.2102"></a> |
| <span class="sourceLineNo">2103</span> return walFS;<a name="line.2103"></a> |
| <span class="sourceLineNo">2104</span> }<a name="line.2104"></a> |
| <span class="sourceLineNo">2105</span><a name="line.2105"></a> |
| <span class="sourceLineNo">2106</span> /**<a name="line.2106"></a> |
| <span class="sourceLineNo">2107</span> * @return the Region directory under WALRootDirectory<a name="line.2107"></a> |
| <span class="sourceLineNo">2108</span> * @throws IOException if there is an error getting WALRootDir<a name="line.2108"></a> |
| <span class="sourceLineNo">2109</span> */<a name="line.2109"></a> |
| <span class="sourceLineNo">2110</span> public Path getWALRegionDir() throws IOException {<a name="line.2110"></a> |
| <span class="sourceLineNo">2111</span> if (regionDir == null) {<a name="line.2111"></a> |
| <span class="sourceLineNo">2112</span> regionDir = CommonFSUtils.getWALRegionDir(conf, getRegionInfo().getTable(),<a name="line.2112"></a> |
| <span class="sourceLineNo">2113</span> getRegionInfo().getEncodedName());<a name="line.2113"></a> |
| <span class="sourceLineNo">2114</span> }<a name="line.2114"></a> |
| <span class="sourceLineNo">2115</span> return regionDir;<a name="line.2115"></a> |
| <span class="sourceLineNo">2116</span> }<a name="line.2116"></a> |
| <span class="sourceLineNo">2117</span><a name="line.2117"></a> |
| <span class="sourceLineNo">2118</span> @Override<a name="line.2118"></a> |
| <span class="sourceLineNo">2119</span> public long getEarliestFlushTimeForAllStores() {<a name="line.2119"></a> |
| <span class="sourceLineNo">2120</span> return Collections.min(lastStoreFlushTimeMap.values());<a name="line.2120"></a> |
| <span class="sourceLineNo">2121</span> }<a name="line.2121"></a> |
| <span class="sourceLineNo">2122</span><a name="line.2122"></a> |
| <span class="sourceLineNo">2123</span> @Override<a name="line.2123"></a> |
| <span class="sourceLineNo">2124</span> public long getOldestHfileTs(boolean majorCompactionOnly) throws IOException {<a name="line.2124"></a> |
| <span class="sourceLineNo">2125</span> long result = Long.MAX_VALUE;<a name="line.2125"></a> |
| <span class="sourceLineNo">2126</span> for (HStore store : stores.values()) {<a name="line.2126"></a> |
| <span class="sourceLineNo">2127</span> Collection<HStoreFile> storeFiles = store.getStorefiles();<a name="line.2127"></a> |
| <span class="sourceLineNo">2128</span> if (storeFiles == null) {<a name="line.2128"></a> |
| <span class="sourceLineNo">2129</span> continue;<a name="line.2129"></a> |
| <span class="sourceLineNo">2130</span> }<a name="line.2130"></a> |
| <span class="sourceLineNo">2131</span> for (HStoreFile file : storeFiles) {<a name="line.2131"></a> |
| <span class="sourceLineNo">2132</span> StoreFileReader sfReader = file.getReader();<a name="line.2132"></a> |
| <span class="sourceLineNo">2133</span> if (sfReader == null) {<a name="line.2133"></a> |
| <span class="sourceLineNo">2134</span> continue;<a name="line.2134"></a> |
| <span class="sourceLineNo">2135</span> }<a name="line.2135"></a> |
| <span class="sourceLineNo">2136</span> HFile.Reader reader = sfReader.getHFileReader();<a name="line.2136"></a> |
| <span class="sourceLineNo">2137</span> if (reader == null) {<a name="line.2137"></a> |
| <span class="sourceLineNo">2138</span> continue;<a name="line.2138"></a> |
| <span class="sourceLineNo">2139</span> }<a name="line.2139"></a> |
| <span class="sourceLineNo">2140</span> if (majorCompactionOnly) {<a name="line.2140"></a> |
| <span class="sourceLineNo">2141</span> byte[] val = reader.getHFileInfo().get(MAJOR_COMPACTION_KEY);<a name="line.2141"></a> |
| <span class="sourceLineNo">2142</span> if (val == null || !Bytes.toBoolean(val)) {<a name="line.2142"></a> |
| <span class="sourceLineNo">2143</span> continue;<a name="line.2143"></a> |
| <span class="sourceLineNo">2144</span> }<a name="line.2144"></a> |
| <span class="sourceLineNo">2145</span> }<a name="line.2145"></a> |
| <span class="sourceLineNo">2146</span> result = Math.min(result, reader.getFileContext().getFileCreateTime());<a name="line.2146"></a> |
| <span class="sourceLineNo">2147</span> }<a name="line.2147"></a> |
| <span class="sourceLineNo">2148</span> }<a name="line.2148"></a> |
| <span class="sourceLineNo">2149</span> return result == Long.MAX_VALUE ? 0 : result;<a name="line.2149"></a> |
| <span class="sourceLineNo">2150</span> }<a name="line.2150"></a> |
| <span class="sourceLineNo">2151</span><a name="line.2151"></a> |
| <span class="sourceLineNo">2152</span> RegionLoad.Builder setCompleteSequenceId(RegionLoad.Builder regionLoadBldr) {<a name="line.2152"></a> |
| <span class="sourceLineNo">2153</span> long lastFlushOpSeqIdLocal = this.lastFlushOpSeqId;<a name="line.2153"></a> |
| <span class="sourceLineNo">2154</span> byte[] encodedRegionName = this.getRegionInfo().getEncodedNameAsBytes();<a name="line.2154"></a> |
| <span class="sourceLineNo">2155</span> regionLoadBldr.clearStoreCompleteSequenceId();<a name="line.2155"></a> |
| <span class="sourceLineNo">2156</span> for (byte[] familyName : this.stores.keySet()) {<a name="line.2156"></a> |
| <span class="sourceLineNo">2157</span> long earliest = this.wal.getEarliestMemStoreSeqNum(encodedRegionName, familyName);<a name="line.2157"></a> |
| <span class="sourceLineNo">2158</span> // Subtract - 1 to go earlier than the current oldest, unflushed edit in memstore; this will<a name="line.2158"></a> |
| <span class="sourceLineNo">2159</span> // give us a sequence id that is for sure flushed. We want edit replay to start after this<a name="line.2159"></a> |
| <span class="sourceLineNo">2160</span> // sequence id in this region. If NO_SEQNUM, use the regions maximum flush id.<a name="line.2160"></a> |
| <span class="sourceLineNo">2161</span> long csid = (earliest == HConstants.NO_SEQNUM)? lastFlushOpSeqIdLocal: earliest - 1;<a name="line.2161"></a> |
| <span class="sourceLineNo">2162</span> regionLoadBldr.addStoreCompleteSequenceId(StoreSequenceId.newBuilder()<a name="line.2162"></a> |
| <span class="sourceLineNo">2163</span> .setFamilyName(UnsafeByteOperations.unsafeWrap(familyName)).setSequenceId(csid).build());<a name="line.2163"></a> |
| <span class="sourceLineNo">2164</span> }<a name="line.2164"></a> |
| <span class="sourceLineNo">2165</span> return regionLoadBldr.setCompleteSequenceId(getMaxFlushedSeqId());<a name="line.2165"></a> |
| <span class="sourceLineNo">2166</span> }<a name="line.2166"></a> |
| <span class="sourceLineNo">2167</span><a name="line.2167"></a> |
| <span class="sourceLineNo">2168</span> //////////////////////////////////////////////////////////////////////////////<a name="line.2168"></a> |
| <span class="sourceLineNo">2169</span> // HRegion maintenance.<a name="line.2169"></a> |
| <span class="sourceLineNo">2170</span> //<a name="line.2170"></a> |
| <span class="sourceLineNo">2171</span> // These methods are meant to be called periodically by the HRegionServer for<a name="line.2171"></a> |
| <span class="sourceLineNo">2172</span> // upkeep.<a name="line.2172"></a> |
| <span class="sourceLineNo">2173</span> //////////////////////////////////////////////////////////////////////////////<a name="line.2173"></a> |
| <span class="sourceLineNo">2174</span> /**<a name="line.2174"></a> |
| <span class="sourceLineNo">2175</span> * Do preparation for pending compaction.<a name="line.2175"></a> |
| <span class="sourceLineNo">2176</span> * @throws IOException<a name="line.2176"></a> |
| <span class="sourceLineNo">2177</span> */<a name="line.2177"></a> |
| <span class="sourceLineNo">2178</span> protected void doRegionCompactionPrep() throws IOException {<a name="line.2178"></a> |
| <span class="sourceLineNo">2179</span> }<a name="line.2179"></a> |
| <span class="sourceLineNo">2180</span><a name="line.2180"></a> |
| <span class="sourceLineNo">2181</span> /**<a name="line.2181"></a> |
| <span class="sourceLineNo">2182</span> * Synchronously compact all stores in the region.<a name="line.2182"></a> |
| <span class="sourceLineNo">2183</span> * <p>This operation could block for a long time, so don't call it from a<a name="line.2183"></a> |
| <span class="sourceLineNo">2184</span> * time-sensitive thread.<a name="line.2184"></a> |
| <span class="sourceLineNo">2185</span> * <p>Note that no locks are taken to prevent possible conflicts between<a name="line.2185"></a> |
| <span class="sourceLineNo">2186</span> * compaction and splitting activities. The regionserver does not normally compact<a name="line.2186"></a> |
| <span class="sourceLineNo">2187</span> * and split in parallel. However by calling this method you may introduce<a name="line.2187"></a> |
| <span class="sourceLineNo">2188</span> * unexpected and unhandled concurrency. Don't do this unless you know what<a name="line.2188"></a> |
| <span class="sourceLineNo">2189</span> * you are doing.<a name="line.2189"></a> |
| <span class="sourceLineNo">2190</span> *<a name="line.2190"></a> |
| <span class="sourceLineNo">2191</span> * @param majorCompaction True to force a major compaction regardless of thresholds<a name="line.2191"></a> |
| <span class="sourceLineNo">2192</span> * @throws IOException<a name="line.2192"></a> |
| <span class="sourceLineNo">2193</span> */<a name="line.2193"></a> |
| <span class="sourceLineNo">2194</span> public void compact(boolean majorCompaction) throws IOException {<a name="line.2194"></a> |
| <span class="sourceLineNo">2195</span> if (majorCompaction) {<a name="line.2195"></a> |
| <span class="sourceLineNo">2196</span> stores.values().forEach(HStore::triggerMajorCompaction);<a name="line.2196"></a> |
| <span class="sourceLineNo">2197</span> }<a name="line.2197"></a> |
| <span class="sourceLineNo">2198</span> for (HStore s : stores.values()) {<a name="line.2198"></a> |
| <span class="sourceLineNo">2199</span> Optional<CompactionContext> compaction = s.requestCompaction();<a name="line.2199"></a> |
| <span class="sourceLineNo">2200</span> if (compaction.isPresent()) {<a name="line.2200"></a> |
| <span class="sourceLineNo">2201</span> ThroughputController controller = null;<a name="line.2201"></a> |
| <span class="sourceLineNo">2202</span> if (rsServices != null) {<a name="line.2202"></a> |
| <span class="sourceLineNo">2203</span> controller = CompactionThroughputControllerFactory.create(rsServices, conf);<a name="line.2203"></a> |
| <span class="sourceLineNo">2204</span> }<a name="line.2204"></a> |
| <span class="sourceLineNo">2205</span> if (controller == null) {<a name="line.2205"></a> |
| <span class="sourceLineNo">2206</span> controller = NoLimitThroughputController.INSTANCE;<a name="line.2206"></a> |
| <span class="sourceLineNo">2207</span> }<a name="line.2207"></a> |
| <span class="sourceLineNo">2208</span> compact(compaction.get(), s, controller, null);<a name="line.2208"></a> |
| <span class="sourceLineNo">2209</span> }<a name="line.2209"></a> |
| <span class="sourceLineNo">2210</span> }<a name="line.2210"></a> |
| <span class="sourceLineNo">2211</span> }<a name="line.2211"></a> |
| <span class="sourceLineNo">2212</span><a name="line.2212"></a> |
| <span class="sourceLineNo">2213</span> /**<a name="line.2213"></a> |
| <span class="sourceLineNo">2214</span> * This is a helper function that compact all the stores synchronously.<a name="line.2214"></a> |
| <span class="sourceLineNo">2215</span> * <p><a name="line.2215"></a> |
| <span class="sourceLineNo">2216</span> * It is used by utilities and testing<a name="line.2216"></a> |
| <span class="sourceLineNo">2217</span> */<a name="line.2217"></a> |
| <span class="sourceLineNo">2218</span> public void compactStores() throws IOException {<a name="line.2218"></a> |
| <span class="sourceLineNo">2219</span> for (HStore s : stores.values()) {<a name="line.2219"></a> |
| <span class="sourceLineNo">2220</span> Optional<CompactionContext> compaction = s.requestCompaction();<a name="line.2220"></a> |
| <span class="sourceLineNo">2221</span> if (compaction.isPresent()) {<a name="line.2221"></a> |
| <span class="sourceLineNo">2222</span> compact(compaction.get(), s, NoLimitThroughputController.INSTANCE, null);<a name="line.2222"></a> |
| <span class="sourceLineNo">2223</span> }<a name="line.2223"></a> |
| <span class="sourceLineNo">2224</span> }<a name="line.2224"></a> |
| <span class="sourceLineNo">2225</span> }<a name="line.2225"></a> |
| <span class="sourceLineNo">2226</span><a name="line.2226"></a> |
| <span class="sourceLineNo">2227</span> /**<a name="line.2227"></a> |
| <span class="sourceLineNo">2228</span> * This is a helper function that compact the given store.<a name="line.2228"></a> |
| <span class="sourceLineNo">2229</span> * <p><a name="line.2229"></a> |
| <span class="sourceLineNo">2230</span> * It is used by utilities and testing<a name="line.2230"></a> |
| <span class="sourceLineNo">2231</span> */<a name="line.2231"></a> |
| <span class="sourceLineNo">2232</span> void compactStore(byte[] family, ThroughputController throughputController) throws IOException {<a name="line.2232"></a> |
| <span class="sourceLineNo">2233</span> HStore s = getStore(family);<a name="line.2233"></a> |
| <span class="sourceLineNo">2234</span> Optional<CompactionContext> compaction = s.requestCompaction();<a name="line.2234"></a> |
| <span class="sourceLineNo">2235</span> if (compaction.isPresent()) {<a name="line.2235"></a> |
| <span class="sourceLineNo">2236</span> compact(compaction.get(), s, throughputController, null);<a name="line.2236"></a> |
| <span class="sourceLineNo">2237</span> }<a name="line.2237"></a> |
| <span class="sourceLineNo">2238</span> }<a name="line.2238"></a> |
| <span class="sourceLineNo">2239</span><a name="line.2239"></a> |
| <span class="sourceLineNo">2240</span> /**<a name="line.2240"></a> |
| <span class="sourceLineNo">2241</span> * Called by compaction thread and after region is opened to compact the<a name="line.2241"></a> |
| <span class="sourceLineNo">2242</span> * HStores if necessary.<a name="line.2242"></a> |
| <span class="sourceLineNo">2243</span> *<a name="line.2243"></a> |
| <span class="sourceLineNo">2244</span> * <p>This operation could block for a long time, so don't call it from a<a name="line.2244"></a> |
| <span class="sourceLineNo">2245</span> * time-sensitive thread.<a name="line.2245"></a> |
| <span class="sourceLineNo">2246</span> *<a name="line.2246"></a> |
| <span class="sourceLineNo">2247</span> * Note that no locking is necessary at this level because compaction only<a name="line.2247"></a> |
| <span class="sourceLineNo">2248</span> * conflicts with a region split, and that cannot happen because the region<a name="line.2248"></a> |
| <span class="sourceLineNo">2249</span> * server does them sequentially and not in parallel.<a name="line.2249"></a> |
| <span class="sourceLineNo">2250</span> *<a name="line.2250"></a> |
| <span class="sourceLineNo">2251</span> * @param compaction Compaction details, obtained by requestCompaction()<a name="line.2251"></a> |
| <span class="sourceLineNo">2252</span> * @param throughputController<a name="line.2252"></a> |
| <span class="sourceLineNo">2253</span> * @return whether the compaction completed<a name="line.2253"></a> |
| <span class="sourceLineNo">2254</span> */<a name="line.2254"></a> |
| <span class="sourceLineNo">2255</span> public boolean compact(CompactionContext compaction, HStore store,<a name="line.2255"></a> |
| <span class="sourceLineNo">2256</span> ThroughputController throughputController) throws IOException {<a name="line.2256"></a> |
| <span class="sourceLineNo">2257</span> return compact(compaction, store, throughputController, null);<a name="line.2257"></a> |
| <span class="sourceLineNo">2258</span> }<a name="line.2258"></a> |
| <span class="sourceLineNo">2259</span><a name="line.2259"></a> |
| <span class="sourceLineNo">2260</span> private boolean shouldForbidMajorCompaction() {<a name="line.2260"></a> |
| <span class="sourceLineNo">2261</span> if (rsServices != null && rsServices.getReplicationSourceService() != null) {<a name="line.2261"></a> |
| <span class="sourceLineNo">2262</span> return rsServices.getReplicationSourceService().getSyncReplicationPeerInfoProvider()<a name="line.2262"></a> |
| <span class="sourceLineNo">2263</span> .checkState(getRegionInfo().getTable(), ForbidMajorCompactionChecker.get());<a name="line.2263"></a> |
| <span class="sourceLineNo">2264</span> }<a name="line.2264"></a> |
| <span class="sourceLineNo">2265</span> return false;<a name="line.2265"></a> |
| <span class="sourceLineNo">2266</span> }<a name="line.2266"></a> |
| <span class="sourceLineNo">2267</span><a name="line.2267"></a> |
| <span class="sourceLineNo">2268</span> /**<a name="line.2268"></a> |
| <span class="sourceLineNo">2269</span> * We are trying to remove / relax the region read lock for compaction.<a name="line.2269"></a> |
| <span class="sourceLineNo">2270</span> * Let's see what are the potential race conditions among the operations (user scan,<a name="line.2270"></a> |
| <span class="sourceLineNo">2271</span> * region split, region close and region bulk load).<a name="line.2271"></a> |
| <span class="sourceLineNo">2272</span> *<a name="line.2272"></a> |
| <span class="sourceLineNo">2273</span> * user scan ---> region read lock<a name="line.2273"></a> |
| <span class="sourceLineNo">2274</span> * region split --> region close first --> region write lock<a name="line.2274"></a> |
| <span class="sourceLineNo">2275</span> * region close --> region write lock<a name="line.2275"></a> |
| <span class="sourceLineNo">2276</span> * region bulk load --> region write lock<a name="line.2276"></a> |
| <span class="sourceLineNo">2277</span> *<a name="line.2277"></a> |
| <span class="sourceLineNo">2278</span> * read lock is compatible with read lock. ---> no problem with user scan/read<a name="line.2278"></a> |
| <span class="sourceLineNo">2279</span> * region bulk load does not cause problem for compaction (no consistency problem, store lock<a name="line.2279"></a> |
| <span class="sourceLineNo">2280</span> * will help the store file accounting).<a name="line.2280"></a> |
| <span class="sourceLineNo">2281</span> * They can run almost concurrently at the region level.<a name="line.2281"></a> |
| <span class="sourceLineNo">2282</span> *<a name="line.2282"></a> |
| <span class="sourceLineNo">2283</span> * The only remaining race condition is between the region close and compaction.<a name="line.2283"></a> |
| <span class="sourceLineNo">2284</span> * So we will evaluate, below, how region close intervenes with compaction if compaction does<a name="line.2284"></a> |
| <span class="sourceLineNo">2285</span> * not acquire region read lock.<a name="line.2285"></a> |
| <span class="sourceLineNo">2286</span> *<a name="line.2286"></a> |
| <span class="sourceLineNo">2287</span> * Here are the steps for compaction:<a name="line.2287"></a> |
| <span class="sourceLineNo">2288</span> * 1. obtain list of StoreFile's<a name="line.2288"></a> |
| <span class="sourceLineNo">2289</span> * 2. create StoreFileScanner's based on list from #1<a name="line.2289"></a> |
| <span class="sourceLineNo">2290</span> * 3. perform compaction and save resulting files under tmp dir<a name="line.2290"></a> |
| <span class="sourceLineNo">2291</span> * 4. swap in compacted files<a name="line.2291"></a> |
| <span class="sourceLineNo">2292</span> *<a name="line.2292"></a> |
| <span class="sourceLineNo">2293</span> * #1 is guarded by store lock. This patch does not change this --> no worse or better<a name="line.2293"></a> |
| <span class="sourceLineNo">2294</span> * For #2, we obtain smallest read point (for region) across all the Scanners (for both default<a name="line.2294"></a> |
| <span class="sourceLineNo">2295</span> * compactor and stripe compactor).<a name="line.2295"></a> |
| <span class="sourceLineNo">2296</span> * The read points are for user scans. Region keeps the read points for all currently open<a name="line.2296"></a> |
| <span class="sourceLineNo">2297</span> * user scanners.<a name="line.2297"></a> |
| <span class="sourceLineNo">2298</span> * Compaction needs to know the smallest read point so that during re-write of the hfiles,<a name="line.2298"></a> |
| <span class="sourceLineNo">2299</span> * it can remove the mvcc points for the cells if their mvccs are older than the smallest<a name="line.2299"></a> |
| <span class="sourceLineNo">2300</span> * since they are not needed anymore.<a name="line.2300"></a> |
| <span class="sourceLineNo">2301</span> * This will not conflict with compaction.<a name="line.2301"></a> |
| <span class="sourceLineNo">2302</span> * For #3, it can be performed in parallel to other operations.<a name="line.2302"></a> |
| <span class="sourceLineNo">2303</span> * For #4 bulk load and compaction don't conflict with each other on the region level<a name="line.2303"></a> |
| <span class="sourceLineNo">2304</span> * (for multi-family atomicy).<a name="line.2304"></a> |
| <span class="sourceLineNo">2305</span> * Region close and compaction are guarded pretty well by the 'writestate'.<a name="line.2305"></a> |
| <span class="sourceLineNo">2306</span> * In HRegion#doClose(), we have :<a name="line.2306"></a> |
| <span class="sourceLineNo">2307</span> * synchronized (writestate) {<a name="line.2307"></a> |
| <span class="sourceLineNo">2308</span> * // Disable compacting and flushing by background threads for this<a name="line.2308"></a> |
| <span class="sourceLineNo">2309</span> * // region.<a name="line.2309"></a> |
| <span class="sourceLineNo">2310</span> * canFlush = !writestate.readOnly;<a name="line.2310"></a> |
| <span class="sourceLineNo">2311</span> * writestate.writesEnabled = false;<a name="line.2311"></a> |
| <span class="sourceLineNo">2312</span> * LOG.debug("Closing " + this + ": disabling compactions & flushes");<a name="line.2312"></a> |
| <span class="sourceLineNo">2313</span> * waitForFlushesAndCompactions();<a name="line.2313"></a> |
| <span class="sourceLineNo">2314</span> * }<a name="line.2314"></a> |
| <span class="sourceLineNo">2315</span> * waitForFlushesAndCompactions() would wait for writestate.compacting to come down to 0.<a name="line.2315"></a> |
| <span class="sourceLineNo">2316</span> * and in HRegion.compact()<a name="line.2316"></a> |
| <span class="sourceLineNo">2317</span> * try {<a name="line.2317"></a> |
| <span class="sourceLineNo">2318</span> * synchronized (writestate) {<a name="line.2318"></a> |
| <span class="sourceLineNo">2319</span> * if (writestate.writesEnabled) {<a name="line.2319"></a> |
| <span class="sourceLineNo">2320</span> * wasStateSet = true;<a name="line.2320"></a> |
| <span class="sourceLineNo">2321</span> * ++writestate.compacting;<a name="line.2321"></a> |
| <span class="sourceLineNo">2322</span> * } else {<a name="line.2322"></a> |
| <span class="sourceLineNo">2323</span> * String msg = "NOT compacting region " + this + ". Writes disabled.";<a name="line.2323"></a> |
| <span class="sourceLineNo">2324</span> * LOG.info(msg);<a name="line.2324"></a> |
| <span class="sourceLineNo">2325</span> * status.abort(msg);<a name="line.2325"></a> |
| <span class="sourceLineNo">2326</span> * return false;<a name="line.2326"></a> |
| <span class="sourceLineNo">2327</span> * }<a name="line.2327"></a> |
| <span class="sourceLineNo">2328</span> * }<a name="line.2328"></a> |
| <span class="sourceLineNo">2329</span> * Also in compactor.performCompaction():<a name="line.2329"></a> |
| <span class="sourceLineNo">2330</span> * check periodically to see if a system stop is requested<a name="line.2330"></a> |
| <span class="sourceLineNo">2331</span> * if (closeChecker != null && closeChecker.isTimeLimit(store, now)) {<a name="line.2331"></a> |
| <span class="sourceLineNo">2332</span> * progress.cancel();<a name="line.2332"></a> |
| <span class="sourceLineNo">2333</span> * return false;<a name="line.2333"></a> |
| <span class="sourceLineNo">2334</span> * }<a name="line.2334"></a> |
| <span class="sourceLineNo">2335</span> * if (closeChecker != null && closeChecker.isSizeLimit(store, len)) {<a name="line.2335"></a> |
| <span class="sourceLineNo">2336</span> * progress.cancel();<a name="line.2336"></a> |
| <span class="sourceLineNo">2337</span> * return false;<a name="line.2337"></a> |
| <span class="sourceLineNo">2338</span> * }<a name="line.2338"></a> |
| <span class="sourceLineNo">2339</span> */<a name="line.2339"></a> |
| <span class="sourceLineNo">2340</span> public boolean compact(CompactionContext compaction, HStore store,<a name="line.2340"></a> |
| <span class="sourceLineNo">2341</span> ThroughputController throughputController, User user) throws IOException {<a name="line.2341"></a> |
| <span class="sourceLineNo">2342</span> assert compaction != null && compaction.hasSelection();<a name="line.2342"></a> |
| <span class="sourceLineNo">2343</span> assert !compaction.getRequest().getFiles().isEmpty();<a name="line.2343"></a> |
| <span class="sourceLineNo">2344</span> if (this.closing.get() || this.closed.get()) {<a name="line.2344"></a> |
| <span class="sourceLineNo">2345</span> LOG.debug("Skipping compaction on " + this + " because closing/closed");<a name="line.2345"></a> |
| <span class="sourceLineNo">2346</span> store.cancelRequestedCompaction(compaction);<a name="line.2346"></a> |
| <span class="sourceLineNo">2347</span> return false;<a name="line.2347"></a> |
| <span class="sourceLineNo">2348</span> }<a name="line.2348"></a> |
| <span class="sourceLineNo">2349</span><a name="line.2349"></a> |
| <span class="sourceLineNo">2350</span> if (compaction.getRequest().isAllFiles() && shouldForbidMajorCompaction()) {<a name="line.2350"></a> |
| <span class="sourceLineNo">2351</span> LOG.warn("Skipping major compaction on " + this<a name="line.2351"></a> |
| <span class="sourceLineNo">2352</span> + " because this cluster is transiting sync replication state"<a name="line.2352"></a> |
| <span class="sourceLineNo">2353</span> + " from STANDBY to DOWNGRADE_ACTIVE");<a name="line.2353"></a> |
| <span class="sourceLineNo">2354</span> store.cancelRequestedCompaction(compaction);<a name="line.2354"></a> |
| <span class="sourceLineNo">2355</span> return false;<a name="line.2355"></a> |
| <span class="sourceLineNo">2356</span> }<a name="line.2356"></a> |
| <span class="sourceLineNo">2357</span><a name="line.2357"></a> |
| <span class="sourceLineNo">2358</span> MonitoredTask status = null;<a name="line.2358"></a> |
| <span class="sourceLineNo">2359</span> boolean requestNeedsCancellation = true;<a name="line.2359"></a> |
| <span class="sourceLineNo">2360</span> try {<a name="line.2360"></a> |
| <span class="sourceLineNo">2361</span> byte[] cf = Bytes.toBytes(store.getColumnFamilyName());<a name="line.2361"></a> |
| <span class="sourceLineNo">2362</span> if (stores.get(cf) != store) {<a name="line.2362"></a> |
| <span class="sourceLineNo">2363</span> LOG.warn("Store " + store.getColumnFamilyName() + " on region " + this<a name="line.2363"></a> |
| <span class="sourceLineNo">2364</span> + " has been re-instantiated, cancel this compaction request. "<a name="line.2364"></a> |
| <span class="sourceLineNo">2365</span> + " It may be caused by the roll back of split transaction");<a name="line.2365"></a> |
| <span class="sourceLineNo">2366</span> return false;<a name="line.2366"></a> |
| <span class="sourceLineNo">2367</span> }<a name="line.2367"></a> |
| <span class="sourceLineNo">2368</span><a name="line.2368"></a> |
| <span class="sourceLineNo">2369</span> status = TaskMonitor.get().createStatus("Compacting " + store + " in " + this);<a name="line.2369"></a> |
| <span class="sourceLineNo">2370</span> status.enableStatusJournal(false);<a name="line.2370"></a> |
| <span class="sourceLineNo">2371</span> if (this.closed.get()) {<a name="line.2371"></a> |
| <span class="sourceLineNo">2372</span> String msg = "Skipping compaction on " + this + " because closed";<a name="line.2372"></a> |
| <span class="sourceLineNo">2373</span> LOG.debug(msg);<a name="line.2373"></a> |
| <span class="sourceLineNo">2374</span> status.abort(msg);<a name="line.2374"></a> |
| <span class="sourceLineNo">2375</span> return false;<a name="line.2375"></a> |
| <span class="sourceLineNo">2376</span> }<a name="line.2376"></a> |
| <span class="sourceLineNo">2377</span> boolean wasStateSet = false;<a name="line.2377"></a> |
| <span class="sourceLineNo">2378</span> try {<a name="line.2378"></a> |
| <span class="sourceLineNo">2379</span> synchronized (writestate) {<a name="line.2379"></a> |
| <span class="sourceLineNo">2380</span> if (writestate.writesEnabled) {<a name="line.2380"></a> |
| <span class="sourceLineNo">2381</span> wasStateSet = true;<a name="line.2381"></a> |
| <span class="sourceLineNo">2382</span> writestate.compacting.incrementAndGet();<a name="line.2382"></a> |
| <span class="sourceLineNo">2383</span> } else {<a name="line.2383"></a> |
| <span class="sourceLineNo">2384</span> String msg = "NOT compacting region " + this + ". Writes disabled.";<a name="line.2384"></a> |
| <span class="sourceLineNo">2385</span> LOG.info(msg);<a name="line.2385"></a> |
| <span class="sourceLineNo">2386</span> status.abort(msg);<a name="line.2386"></a> |
| <span class="sourceLineNo">2387</span> return false;<a name="line.2387"></a> |
| <span class="sourceLineNo">2388</span> }<a name="line.2388"></a> |
| <span class="sourceLineNo">2389</span> }<a name="line.2389"></a> |
| <span class="sourceLineNo">2390</span> LOG.info("Starting compaction of {} in {}{}", store, this,<a name="line.2390"></a> |
| <span class="sourceLineNo">2391</span> (compaction.getRequest().isOffPeak()?" as an off-peak compaction":""));<a name="line.2391"></a> |
| <span class="sourceLineNo">2392</span> doRegionCompactionPrep();<a name="line.2392"></a> |
| <span class="sourceLineNo">2393</span> try {<a name="line.2393"></a> |
| <span class="sourceLineNo">2394</span> status.setStatus("Compacting store " + store);<a name="line.2394"></a> |
| <span class="sourceLineNo">2395</span> // We no longer need to cancel the request on the way out of this<a name="line.2395"></a> |
| <span class="sourceLineNo">2396</span> // method because Store#compact will clean up unconditionally<a name="line.2396"></a> |
| <span class="sourceLineNo">2397</span> requestNeedsCancellation = false;<a name="line.2397"></a> |
| <span class="sourceLineNo">2398</span> store.compact(compaction, throughputController, user);<a name="line.2398"></a> |
| <span class="sourceLineNo">2399</span> } catch (InterruptedIOException iioe) {<a name="line.2399"></a> |
| <span class="sourceLineNo">2400</span> String msg = "region " + this + " compaction interrupted";<a name="line.2400"></a> |
| <span class="sourceLineNo">2401</span> LOG.info(msg, iioe);<a name="line.2401"></a> |
| <span class="sourceLineNo">2402</span> status.abort(msg);<a name="line.2402"></a> |
| <span class="sourceLineNo">2403</span> return false;<a name="line.2403"></a> |
| <span class="sourceLineNo">2404</span> }<a name="line.2404"></a> |
| <span class="sourceLineNo">2405</span> } finally {<a name="line.2405"></a> |
| <span class="sourceLineNo">2406</span> if (wasStateSet) {<a name="line.2406"></a> |
| <span class="sourceLineNo">2407</span> synchronized (writestate) {<a name="line.2407"></a> |
| <span class="sourceLineNo">2408</span> writestate.compacting.decrementAndGet();<a name="line.2408"></a> |
| <span class="sourceLineNo">2409</span> if (writestate.compacting.get() <= 0) {<a name="line.2409"></a> |
| <span class="sourceLineNo">2410</span> writestate.notifyAll();<a name="line.2410"></a> |
| <span class="sourceLineNo">2411</span> }<a name="line.2411"></a> |
| <span class="sourceLineNo">2412</span> }<a name="line.2412"></a> |
| <span class="sourceLineNo">2413</span> }<a name="line.2413"></a> |
| <span class="sourceLineNo">2414</span> }<a name="line.2414"></a> |
| <span class="sourceLineNo">2415</span> status.markComplete("Compaction complete");<a name="line.2415"></a> |
| <span class="sourceLineNo">2416</span> return true;<a name="line.2416"></a> |
| <span class="sourceLineNo">2417</span> } finally {<a name="line.2417"></a> |
| <span class="sourceLineNo">2418</span> if (requestNeedsCancellation) store.cancelRequestedCompaction(compaction);<a name="line.2418"></a> |
| <span class="sourceLineNo">2419</span> if (status != null) {<a name="line.2419"></a> |
| <span class="sourceLineNo">2420</span> LOG.debug("Compaction status journal for {}:\n{}", this.getRegionInfo().getEncodedName(),<a name="line.2420"></a> |
| <span class="sourceLineNo">2421</span> status.prettyPrintJournal());<a name="line.2421"></a> |
| <span class="sourceLineNo">2422</span> status.cleanup();<a name="line.2422"></a> |
| <span class="sourceLineNo">2423</span> }<a name="line.2423"></a> |
| <span class="sourceLineNo">2424</span> }<a name="line.2424"></a> |
| <span class="sourceLineNo">2425</span> }<a name="line.2425"></a> |
| <span class="sourceLineNo">2426</span><a name="line.2426"></a> |
| <span class="sourceLineNo">2427</span> /**<a name="line.2427"></a> |
| <span class="sourceLineNo">2428</span> * Flush the cache.<a name="line.2428"></a> |
| <span class="sourceLineNo">2429</span> *<a name="line.2429"></a> |
| <span class="sourceLineNo">2430</span> * <p>When this method is called the cache will be flushed unless:<a name="line.2430"></a> |
| <span class="sourceLineNo">2431</span> * <ol><a name="line.2431"></a> |
| <span class="sourceLineNo">2432</span> * <li>the cache is empty</li><a name="line.2432"></a> |
| <span class="sourceLineNo">2433</span> * <li>the region is closed.</li><a name="line.2433"></a> |
| <span class="sourceLineNo">2434</span> * <li>a flush is already in progress</li><a name="line.2434"></a> |
| <span class="sourceLineNo">2435</span> * <li>writes are disabled</li><a name="line.2435"></a> |
| <span class="sourceLineNo">2436</span> * </ol><a name="line.2436"></a> |
| <span class="sourceLineNo">2437</span> *<a name="line.2437"></a> |
| <span class="sourceLineNo">2438</span> * <p>This method may block for some time, so it should not be called from a<a name="line.2438"></a> |
| <span class="sourceLineNo">2439</span> * time-sensitive thread.<a name="line.2439"></a> |
| <span class="sourceLineNo">2440</span> * @param flushAllStores whether we want to force a flush of all stores<a name="line.2440"></a> |
| <span class="sourceLineNo">2441</span> * @return FlushResult indicating whether the flush was successful or not and if<a name="line.2441"></a> |
| <span class="sourceLineNo">2442</span> * the region needs compacting<a name="line.2442"></a> |
| <span class="sourceLineNo">2443</span> *<a name="line.2443"></a> |
| <span class="sourceLineNo">2444</span> * @throws IOException general io exceptions<a name="line.2444"></a> |
| <span class="sourceLineNo">2445</span> * because a snapshot was not properly persisted.<a name="line.2445"></a> |
| <span class="sourceLineNo">2446</span> */<a name="line.2446"></a> |
| <span class="sourceLineNo">2447</span> // TODO HBASE-18905. We might have to expose a requestFlush API for CPs<a name="line.2447"></a> |
| <span class="sourceLineNo">2448</span> public FlushResult flush(boolean flushAllStores) throws IOException {<a name="line.2448"></a> |
| <span class="sourceLineNo">2449</span> return flushcache(flushAllStores, false, FlushLifeCycleTracker.DUMMY);<a name="line.2449"></a> |
| <span class="sourceLineNo">2450</span> }<a name="line.2450"></a> |
| <span class="sourceLineNo">2451</span><a name="line.2451"></a> |
| <span class="sourceLineNo">2452</span> public interface FlushResult {<a name="line.2452"></a> |
| <span class="sourceLineNo">2453</span> enum Result {<a name="line.2453"></a> |
| <span class="sourceLineNo">2454</span> FLUSHED_NO_COMPACTION_NEEDED,<a name="line.2454"></a> |
| <span class="sourceLineNo">2455</span> FLUSHED_COMPACTION_NEEDED,<a name="line.2455"></a> |
| <span class="sourceLineNo">2456</span> // Special case where a flush didn't run because there's nothing in the memstores. Used when<a name="line.2456"></a> |
| <span class="sourceLineNo">2457</span> // bulk loading to know when we can still load even if a flush didn't happen.<a name="line.2457"></a> |
| <span class="sourceLineNo">2458</span> CANNOT_FLUSH_MEMSTORE_EMPTY,<a name="line.2458"></a> |
| <span class="sourceLineNo">2459</span> CANNOT_FLUSH<a name="line.2459"></a> |
| <span class="sourceLineNo">2460</span> }<a name="line.2460"></a> |
| <span class="sourceLineNo">2461</span><a name="line.2461"></a> |
| <span class="sourceLineNo">2462</span> /** @return the detailed result code */<a name="line.2462"></a> |
| <span class="sourceLineNo">2463</span> Result getResult();<a name="line.2463"></a> |
| <span class="sourceLineNo">2464</span><a name="line.2464"></a> |
| <span class="sourceLineNo">2465</span> /** @return true if the memstores were flushed, else false */<a name="line.2465"></a> |
| <span class="sourceLineNo">2466</span> boolean isFlushSucceeded();<a name="line.2466"></a> |
| <span class="sourceLineNo">2467</span><a name="line.2467"></a> |
| <span class="sourceLineNo">2468</span> /** @return True if the flush requested a compaction, else false */<a name="line.2468"></a> |
| <span class="sourceLineNo">2469</span> boolean isCompactionNeeded();<a name="line.2469"></a> |
| <span class="sourceLineNo">2470</span> }<a name="line.2470"></a> |
| <span class="sourceLineNo">2471</span><a name="line.2471"></a> |
| <span class="sourceLineNo">2472</span> FlushResultImpl flushcache(boolean flushAllStores, boolean writeFlushRequestWalMarker,<a name="line.2472"></a> |
| <span class="sourceLineNo">2473</span> FlushLifeCycleTracker tracker) throws IOException {<a name="line.2473"></a> |
| <span class="sourceLineNo">2474</span> List<byte[]> families = null;<a name="line.2474"></a> |
| <span class="sourceLineNo">2475</span> if (flushAllStores) {<a name="line.2475"></a> |
| <span class="sourceLineNo">2476</span> families = new ArrayList<>();<a name="line.2476"></a> |
| <span class="sourceLineNo">2477</span> families.addAll(this.getTableDescriptor().getColumnFamilyNames());<a name="line.2477"></a> |
| <span class="sourceLineNo">2478</span> }<a name="line.2478"></a> |
| <span class="sourceLineNo">2479</span> return this.flushcache(families, writeFlushRequestWalMarker, tracker);<a name="line.2479"></a> |
| <span class="sourceLineNo">2480</span> }<a name="line.2480"></a> |
| <span class="sourceLineNo">2481</span><a name="line.2481"></a> |
| <span class="sourceLineNo">2482</span> /**<a name="line.2482"></a> |
| <span class="sourceLineNo">2483</span> * Flush the cache.<a name="line.2483"></a> |
| <span class="sourceLineNo">2484</span> *<a name="line.2484"></a> |
| <span class="sourceLineNo">2485</span> * When this method is called the cache will be flushed unless:<a name="line.2485"></a> |
| <span class="sourceLineNo">2486</span> * <ol><a name="line.2486"></a> |
| <span class="sourceLineNo">2487</span> * <li>the cache is empty</li><a name="line.2487"></a> |
| <span class="sourceLineNo">2488</span> * <li>the region is closed.</li><a name="line.2488"></a> |
| <span class="sourceLineNo">2489</span> * <li>a flush is already in progress</li><a name="line.2489"></a> |
| <span class="sourceLineNo">2490</span> * <li>writes are disabled</li><a name="line.2490"></a> |
| <span class="sourceLineNo">2491</span> * </ol><a name="line.2491"></a> |
| <span class="sourceLineNo">2492</span> *<a name="line.2492"></a> |
| <span class="sourceLineNo">2493</span> * <p>This method may block for some time, so it should not be called from a<a name="line.2493"></a> |
| <span class="sourceLineNo">2494</span> * time-sensitive thread.<a name="line.2494"></a> |
| <span class="sourceLineNo">2495</span> * @param families stores of region to flush.<a name="line.2495"></a> |
| <span class="sourceLineNo">2496</span> * @param writeFlushRequestWalMarker whether to write the flush request marker to WAL<a name="line.2496"></a> |
| <span class="sourceLineNo">2497</span> * @param tracker used to track the life cycle of this flush<a name="line.2497"></a> |
| <span class="sourceLineNo">2498</span> * @return whether the flush is success and whether the region needs compacting<a name="line.2498"></a> |
| <span class="sourceLineNo">2499</span> *<a name="line.2499"></a> |
| <span class="sourceLineNo">2500</span> * @throws IOException general io exceptions<a name="line.2500"></a> |
| <span class="sourceLineNo">2501</span> * @throws DroppedSnapshotException Thrown when replay of wal is required<a name="line.2501"></a> |
| <span class="sourceLineNo">2502</span> * because a Snapshot was not properly persisted. The region is put in closing mode, and the<a name="line.2502"></a> |
| <span class="sourceLineNo">2503</span> * caller MUST abort after this.<a name="line.2503"></a> |
| <span class="sourceLineNo">2504</span> */<a name="line.2504"></a> |
| <span class="sourceLineNo">2505</span> public FlushResultImpl flushcache(List<byte[]> families,<a name="line.2505"></a> |
| <span class="sourceLineNo">2506</span> boolean writeFlushRequestWalMarker, FlushLifeCycleTracker tracker) throws IOException {<a name="line.2506"></a> |
| <span class="sourceLineNo">2507</span> // fail-fast instead of waiting on the lock<a name="line.2507"></a> |
| <span class="sourceLineNo">2508</span> if (this.closing.get()) {<a name="line.2508"></a> |
| <span class="sourceLineNo">2509</span> String msg = "Skipping flush on " + this + " because closing";<a name="line.2509"></a> |
| <span class="sourceLineNo">2510</span> LOG.debug(msg);<a name="line.2510"></a> |
| <span class="sourceLineNo">2511</span> return new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, msg, false);<a name="line.2511"></a> |
| <span class="sourceLineNo">2512</span> }<a name="line.2512"></a> |
| <span class="sourceLineNo">2513</span> MonitoredTask status = TaskMonitor.get().createStatus("Flushing " + this);<a name="line.2513"></a> |
| <span class="sourceLineNo">2514</span> status.enableStatusJournal(false);<a name="line.2514"></a> |
| <span class="sourceLineNo">2515</span> status.setStatus("Acquiring readlock on region");<a name="line.2515"></a> |
| <span class="sourceLineNo">2516</span> // block waiting for the lock for flushing cache<a name="line.2516"></a> |
| <span class="sourceLineNo">2517</span> lock.readLock().lock();<a name="line.2517"></a> |
| <span class="sourceLineNo">2518</span> boolean flushed = true;<a name="line.2518"></a> |
| <span class="sourceLineNo">2519</span> try {<a name="line.2519"></a> |
| <span class="sourceLineNo">2520</span> if (this.closed.get()) {<a name="line.2520"></a> |
| <span class="sourceLineNo">2521</span> String msg = "Skipping flush on " + this + " because closed";<a name="line.2521"></a> |
| <span class="sourceLineNo">2522</span> LOG.debug(msg);<a name="line.2522"></a> |
| <span class="sourceLineNo">2523</span> status.abort(msg);<a name="line.2523"></a> |
| <span class="sourceLineNo">2524</span> flushed = false;<a name="line.2524"></a> |
| <span class="sourceLineNo">2525</span> return new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, msg, false);<a name="line.2525"></a> |
| <span class="sourceLineNo">2526</span> }<a name="line.2526"></a> |
| <span class="sourceLineNo">2527</span> if (coprocessorHost != null) {<a name="line.2527"></a> |
| <span class="sourceLineNo">2528</span> status.setStatus("Running coprocessor pre-flush hooks");<a name="line.2528"></a> |
| <span class="sourceLineNo">2529</span> coprocessorHost.preFlush(tracker);<a name="line.2529"></a> |
| <span class="sourceLineNo">2530</span> }<a name="line.2530"></a> |
| <span class="sourceLineNo">2531</span> // TODO: this should be managed within memstore with the snapshot, updated only after flush<a name="line.2531"></a> |
| <span class="sourceLineNo">2532</span> // successful<a name="line.2532"></a> |
| <span class="sourceLineNo">2533</span> if (numMutationsWithoutWAL.sum() > 0) {<a name="line.2533"></a> |
| <span class="sourceLineNo">2534</span> numMutationsWithoutWAL.reset();<a name="line.2534"></a> |
| <span class="sourceLineNo">2535</span> dataInMemoryWithoutWAL.reset();<a name="line.2535"></a> |
| <span class="sourceLineNo">2536</span> }<a name="line.2536"></a> |
| <span class="sourceLineNo">2537</span> synchronized (writestate) {<a name="line.2537"></a> |
| <span class="sourceLineNo">2538</span> if (!writestate.flushing && writestate.writesEnabled) {<a name="line.2538"></a> |
| <span class="sourceLineNo">2539</span> this.writestate.flushing = true;<a name="line.2539"></a> |
| <span class="sourceLineNo">2540</span> } else {<a name="line.2540"></a> |
| <span class="sourceLineNo">2541</span> String msg = "NOT flushing " + this + " as " + (writestate.flushing ? "already flushing"<a name="line.2541"></a> |
| <span class="sourceLineNo">2542</span> : "writes are not enabled");<a name="line.2542"></a> |
| <span class="sourceLineNo">2543</span> LOG.debug(msg);<a name="line.2543"></a> |
| <span class="sourceLineNo">2544</span> status.abort(msg);<a name="line.2544"></a> |
| <span class="sourceLineNo">2545</span> flushed = false;<a name="line.2545"></a> |
| <span class="sourceLineNo">2546</span> return new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, msg, false);<a name="line.2546"></a> |
| <span class="sourceLineNo">2547</span> }<a name="line.2547"></a> |
| <span class="sourceLineNo">2548</span> }<a name="line.2548"></a> |
| <span class="sourceLineNo">2549</span><a name="line.2549"></a> |
| <span class="sourceLineNo">2550</span> try {<a name="line.2550"></a> |
| <span class="sourceLineNo">2551</span> // The reason that we do not always use flushPolicy is, when the flush is<a name="line.2551"></a> |
| <span class="sourceLineNo">2552</span> // caused by logRoller, we should select stores which must be flushed<a name="line.2552"></a> |
| <span class="sourceLineNo">2553</span> // rather than could be flushed.<a name="line.2553"></a> |
| <span class="sourceLineNo">2554</span> Collection<HStore> specificStoresToFlush = null;<a name="line.2554"></a> |
| <span class="sourceLineNo">2555</span> if (families != null) {<a name="line.2555"></a> |
| <span class="sourceLineNo">2556</span> specificStoresToFlush = getSpecificStores(families);<a name="line.2556"></a> |
| <span class="sourceLineNo">2557</span> } else {<a name="line.2557"></a> |
| <span class="sourceLineNo">2558</span> specificStoresToFlush = flushPolicy.selectStoresToFlush();<a name="line.2558"></a> |
| <span class="sourceLineNo">2559</span> }<a name="line.2559"></a> |
| <span class="sourceLineNo">2560</span> FlushResultImpl fs =<a name="line.2560"></a> |
| <span class="sourceLineNo">2561</span> internalFlushcache(specificStoresToFlush, status, writeFlushRequestWalMarker, tracker);<a name="line.2561"></a> |
| <span class="sourceLineNo">2562</span><a name="line.2562"></a> |
| <span class="sourceLineNo">2563</span> if (coprocessorHost != null) {<a name="line.2563"></a> |
| <span class="sourceLineNo">2564</span> status.setStatus("Running post-flush coprocessor hooks");<a name="line.2564"></a> |
| <span class="sourceLineNo">2565</span> coprocessorHost.postFlush(tracker);<a name="line.2565"></a> |
| <span class="sourceLineNo">2566</span> }<a name="line.2566"></a> |
| <span class="sourceLineNo">2567</span><a name="line.2567"></a> |
| <span class="sourceLineNo">2568</span> if(fs.isFlushSucceeded()) {<a name="line.2568"></a> |
| <span class="sourceLineNo">2569</span> flushesQueued.reset();<a name="line.2569"></a> |
| <span class="sourceLineNo">2570</span> }<a name="line.2570"></a> |
| <span class="sourceLineNo">2571</span><a name="line.2571"></a> |
| <span class="sourceLineNo">2572</span> status.markComplete("Flush successful " + fs.toString());<a name="line.2572"></a> |
| <span class="sourceLineNo">2573</span> return fs;<a name="line.2573"></a> |
| <span class="sourceLineNo">2574</span> } finally {<a name="line.2574"></a> |
| <span class="sourceLineNo">2575</span> synchronized (writestate) {<a name="line.2575"></a> |
| <span class="sourceLineNo">2576</span> writestate.flushing = false;<a name="line.2576"></a> |
| <span class="sourceLineNo">2577</span> this.writestate.flushRequested = false;<a name="line.2577"></a> |
| <span class="sourceLineNo">2578</span> writestate.notifyAll();<a name="line.2578"></a> |
| <span class="sourceLineNo">2579</span> }<a name="line.2579"></a> |
| <span class="sourceLineNo">2580</span> }<a name="line.2580"></a> |
| <span class="sourceLineNo">2581</span> } finally {<a name="line.2581"></a> |
| <span class="sourceLineNo">2582</span> lock.readLock().unlock();<a name="line.2582"></a> |
| <span class="sourceLineNo">2583</span> if (flushed) {<a name="line.2583"></a> |
| <span class="sourceLineNo">2584</span> // Don't log this journal stuff if no flush -- confusing.<a name="line.2584"></a> |
| <span class="sourceLineNo">2585</span> LOG.debug("Flush status journal for {}:\n{}", this.getRegionInfo().getEncodedName(),<a name="line.2585"></a> |
| <span class="sourceLineNo">2586</span> status.prettyPrintJournal());<a name="line.2586"></a> |
| <span class="sourceLineNo">2587</span> }<a name="line.2587"></a> |
| <span class="sourceLineNo">2588</span> status.cleanup();<a name="line.2588"></a> |
| <span class="sourceLineNo">2589</span> }<a name="line.2589"></a> |
| <span class="sourceLineNo">2590</span> }<a name="line.2590"></a> |
| <span class="sourceLineNo">2591</span><a name="line.2591"></a> |
| <span class="sourceLineNo">2592</span> /**<a name="line.2592"></a> |
| <span class="sourceLineNo">2593</span> * get stores which matches the specified families<a name="line.2593"></a> |
| <span class="sourceLineNo">2594</span> *<a name="line.2594"></a> |
| <span class="sourceLineNo">2595</span> * @return the stores need to be flushed.<a name="line.2595"></a> |
| <span class="sourceLineNo">2596</span> */<a name="line.2596"></a> |
| <span class="sourceLineNo">2597</span> private Collection<HStore> getSpecificStores(List<byte[]> families) {<a name="line.2597"></a> |
| <span class="sourceLineNo">2598</span> Collection<HStore> specificStoresToFlush = new ArrayList<>();<a name="line.2598"></a> |
| <span class="sourceLineNo">2599</span> for (byte[] family : families) {<a name="line.2599"></a> |
| <span class="sourceLineNo">2600</span> specificStoresToFlush.add(stores.get(family));<a name="line.2600"></a> |
| <span class="sourceLineNo">2601</span> }<a name="line.2601"></a> |
| <span class="sourceLineNo">2602</span> return specificStoresToFlush;<a name="line.2602"></a> |
| <span class="sourceLineNo">2603</span> }<a name="line.2603"></a> |
| <span class="sourceLineNo">2604</span><a name="line.2604"></a> |
| <span class="sourceLineNo">2605</span> /**<a name="line.2605"></a> |
| <span class="sourceLineNo">2606</span> * Should the store be flushed because it is old enough.<a name="line.2606"></a> |
| <span class="sourceLineNo">2607</span> * <p><a name="line.2607"></a> |
| <span class="sourceLineNo">2608</span> * Every FlushPolicy should call this to determine whether a store is old enough to flush (except<a name="line.2608"></a> |
| <span class="sourceLineNo">2609</span> * that you always flush all stores). Otherwise the method will always<a name="line.2609"></a> |
| <span class="sourceLineNo">2610</span> * returns true which will make a lot of flush requests.<a name="line.2610"></a> |
| <span class="sourceLineNo">2611</span> */<a name="line.2611"></a> |
| <span class="sourceLineNo">2612</span> boolean shouldFlushStore(HStore store) {<a name="line.2612"></a> |
| <span class="sourceLineNo">2613</span> long earliest = this.wal.getEarliestMemStoreSeqNum(getRegionInfo().getEncodedNameAsBytes(),<a name="line.2613"></a> |
| <span class="sourceLineNo">2614</span> store.getColumnFamilyDescriptor().getName()) - 1;<a name="line.2614"></a> |
| <span class="sourceLineNo">2615</span> if (earliest > 0 && earliest + flushPerChanges < mvcc.getReadPoint()) {<a name="line.2615"></a> |
| <span class="sourceLineNo">2616</span> if (LOG.isDebugEnabled()) {<a name="line.2616"></a> |
| <span class="sourceLineNo">2617</span> LOG.debug("Flush column family " + store.getColumnFamilyName() + " of " +<a name="line.2617"></a> |
| <span class="sourceLineNo">2618</span> getRegionInfo().getEncodedName() + " because unflushed sequenceid=" + earliest +<a name="line.2618"></a> |
| <span class="sourceLineNo">2619</span> " is > " + this.flushPerChanges + " from current=" + mvcc.getReadPoint());<a name="line.2619"></a> |
| <span class="sourceLineNo">2620</span> }<a name="line.2620"></a> |
| <span class="sourceLineNo">2621</span> return true;<a name="line.2621"></a> |
| <span class="sourceLineNo">2622</span> }<a name="line.2622"></a> |
| <span class="sourceLineNo">2623</span> if (this.flushCheckInterval <= 0) {<a name="line.2623"></a> |
| <span class="sourceLineNo">2624</span> return false;<a name="line.2624"></a> |
| <span class="sourceLineNo">2625</span> }<a name="line.2625"></a> |
| <span class="sourceLineNo">2626</span> long now = EnvironmentEdgeManager.currentTime();<a name="line.2626"></a> |
| <span class="sourceLineNo">2627</span> if (store.timeOfOldestEdit() < now - this.flushCheckInterval) {<a name="line.2627"></a> |
| <span class="sourceLineNo">2628</span> if (LOG.isDebugEnabled()) {<a name="line.2628"></a> |
| <span class="sourceLineNo">2629</span> LOG.debug("Flush column family: " + store.getColumnFamilyName() + " of " +<a name="line.2629"></a> |
| <span class="sourceLineNo">2630</span> getRegionInfo().getEncodedName() + " because time of oldest edit=" +<a name="line.2630"></a> |
| <span class="sourceLineNo">2631</span> store.timeOfOldestEdit() + " is > " + this.flushCheckInterval + " from now =" + now);<a name="line.2631"></a> |
| <span class="sourceLineNo">2632</span> }<a name="line.2632"></a> |
| <span class="sourceLineNo">2633</span> return true;<a name="line.2633"></a> |
| <span class="sourceLineNo">2634</span> }<a name="line.2634"></a> |
| <span class="sourceLineNo">2635</span> return false;<a name="line.2635"></a> |
| <span class="sourceLineNo">2636</span> }<a name="line.2636"></a> |
| <span class="sourceLineNo">2637</span><a name="line.2637"></a> |
| <span class="sourceLineNo">2638</span> /**<a name="line.2638"></a> |
| <span class="sourceLineNo">2639</span> * Should the memstore be flushed now<a name="line.2639"></a> |
| <span class="sourceLineNo">2640</span> */<a name="line.2640"></a> |
| <span class="sourceLineNo">2641</span> boolean shouldFlush(final StringBuilder whyFlush) {<a name="line.2641"></a> |
| <span class="sourceLineNo">2642</span> whyFlush.setLength(0);<a name="line.2642"></a> |
| <span class="sourceLineNo">2643</span> // This is a rough measure.<a name="line.2643"></a> |
| <span class="sourceLineNo">2644</span> if (this.maxFlushedSeqId > 0<a name="line.2644"></a> |
| <span class="sourceLineNo">2645</span> && (this.maxFlushedSeqId + this.flushPerChanges < this.mvcc.getReadPoint())) {<a name="line.2645"></a> |
| <span class="sourceLineNo">2646</span> whyFlush.append("more than max edits, " + this.flushPerChanges + ", since last flush");<a name="line.2646"></a> |
| <span class="sourceLineNo">2647</span> return true;<a name="line.2647"></a> |
| <span class="sourceLineNo">2648</span> }<a name="line.2648"></a> |
| <span class="sourceLineNo">2649</span> long modifiedFlushCheckInterval = flushCheckInterval;<a name="line.2649"></a> |
| <span class="sourceLineNo">2650</span> if (getRegionInfo().getTable().isSystemTable() &&<a name="line.2650"></a> |
| <span class="sourceLineNo">2651</span> getRegionInfo().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {<a name="line.2651"></a> |
| <span class="sourceLineNo">2652</span> modifiedFlushCheckInterval = SYSTEM_CACHE_FLUSH_INTERVAL;<a name="line.2652"></a> |
| <span class="sourceLineNo">2653</span> }<a name="line.2653"></a> |
| <span class="sourceLineNo">2654</span> if (modifiedFlushCheckInterval <= 0) { //disabled<a name="line.2654"></a> |
| <span class="sourceLineNo">2655</span> return false;<a name="line.2655"></a> |
| <span class="sourceLineNo">2656</span> }<a name="line.2656"></a> |
| <span class="sourceLineNo">2657</span> long now = EnvironmentEdgeManager.currentTime();<a name="line.2657"></a> |
| <span class="sourceLineNo">2658</span> //if we flushed in the recent past, we don't need to do again now<a name="line.2658"></a> |
| <span class="sourceLineNo">2659</span> if ((now - getEarliestFlushTimeForAllStores() < modifiedFlushCheckInterval)) {<a name="line.2659"></a> |
| <span class="sourceLineNo">2660</span> return false;<a name="line.2660"></a> |
| <span class="sourceLineNo">2661</span> }<a name="line.2661"></a> |
| <span class="sourceLineNo">2662</span> //since we didn't flush in the recent past, flush now if certain conditions<a name="line.2662"></a> |
| <span class="sourceLineNo">2663</span> //are met. Return true on first such memstore hit.<a name="line.2663"></a> |
| <span class="sourceLineNo">2664</span> for (HStore s : stores.values()) {<a name="line.2664"></a> |
| <span class="sourceLineNo">2665</span> if (s.timeOfOldestEdit() < now - modifiedFlushCheckInterval) {<a name="line.2665"></a> |
| <span class="sourceLineNo">2666</span> // we have an old enough edit in the memstore, flush<a name="line.2666"></a> |
| <span class="sourceLineNo">2667</span> whyFlush.append(s.toString() + " has an old edit so flush to free WALs");<a name="line.2667"></a> |
| <span class="sourceLineNo">2668</span> return true;<a name="line.2668"></a> |
| <span class="sourceLineNo">2669</span> }<a name="line.2669"></a> |
| <span class="sourceLineNo">2670</span> }<a name="line.2670"></a> |
| <span class="sourceLineNo">2671</span> return false;<a name="line.2671"></a> |
| <span class="sourceLineNo">2672</span> }<a name="line.2672"></a> |
| <span class="sourceLineNo">2673</span><a name="line.2673"></a> |
| <span class="sourceLineNo">2674</span> /**<a name="line.2674"></a> |
| <span class="sourceLineNo">2675</span> * Flushing all stores.<a name="line.2675"></a> |
| <span class="sourceLineNo">2676</span> * @see #internalFlushcache(Collection, MonitoredTask, boolean, FlushLifeCycleTracker)<a name="line.2676"></a> |
| <span class="sourceLineNo">2677</span> */<a name="line.2677"></a> |
| <span class="sourceLineNo">2678</span> private FlushResult internalFlushcache(MonitoredTask status) throws IOException {<a name="line.2678"></a> |
| <span class="sourceLineNo">2679</span> return internalFlushcache(stores.values(), status, false, FlushLifeCycleTracker.DUMMY);<a name="line.2679"></a> |
| <span class="sourceLineNo">2680</span> }<a name="line.2680"></a> |
| <span class="sourceLineNo">2681</span><a name="line.2681"></a> |
| <span class="sourceLineNo">2682</span> /**<a name="line.2682"></a> |
| <span class="sourceLineNo">2683</span> * Flushing given stores.<a name="line.2683"></a> |
| <span class="sourceLineNo">2684</span> * @see #internalFlushcache(WAL, long, Collection, MonitoredTask, boolean, FlushLifeCycleTracker)<a name="line.2684"></a> |
| <span class="sourceLineNo">2685</span> */<a name="line.2685"></a> |
| <span class="sourceLineNo">2686</span> private FlushResultImpl internalFlushcache(Collection<HStore> storesToFlush, MonitoredTask status,<a name="line.2686"></a> |
| <span class="sourceLineNo">2687</span> boolean writeFlushWalMarker, FlushLifeCycleTracker tracker) throws IOException {<a name="line.2687"></a> |
| <span class="sourceLineNo">2688</span> return internalFlushcache(this.wal, HConstants.NO_SEQNUM, storesToFlush, status,<a name="line.2688"></a> |
| <span class="sourceLineNo">2689</span> writeFlushWalMarker, tracker);<a name="line.2689"></a> |
| <span class="sourceLineNo">2690</span> }<a name="line.2690"></a> |
| <span class="sourceLineNo">2691</span><a name="line.2691"></a> |
| <span class="sourceLineNo">2692</span> /**<a name="line.2692"></a> |
| <span class="sourceLineNo">2693</span> * Flush the memstore. Flushing the memstore is a little tricky. We have a lot of updates in the<a name="line.2693"></a> |
| <span class="sourceLineNo">2694</span> * memstore, all of which have also been written to the wal. We need to write those updates in the<a name="line.2694"></a> |
| <span class="sourceLineNo">2695</span> * memstore out to disk, while being able to process reads/writes as much as possible during the<a name="line.2695"></a> |
| <span class="sourceLineNo">2696</span> * flush operation.<a name="line.2696"></a> |
| <span class="sourceLineNo">2697</span> * <p><a name="line.2697"></a> |
| <span class="sourceLineNo">2698</span> * This method may block for some time. Every time you call it, we up the regions sequence id even<a name="line.2698"></a> |
| <span class="sourceLineNo">2699</span> * if we don't flush; i.e. the returned region id will be at least one larger than the last edit<a name="line.2699"></a> |
| <span class="sourceLineNo">2700</span> * applied to this region. The returned id does not refer to an actual edit. The returned id can<a name="line.2700"></a> |
| <span class="sourceLineNo">2701</span> * be used for say installing a bulk loaded file just ahead of the last hfile that was the result<a name="line.2701"></a> |
| <span class="sourceLineNo">2702</span> * of this flush, etc.<a name="line.2702"></a> |
| <span class="sourceLineNo">2703</span> * @param wal Null if we're NOT to go via wal.<a name="line.2703"></a> |
| <span class="sourceLineNo">2704</span> * @param myseqid The seqid to use if <code>wal</code> is null writing out flush file.<a name="line.2704"></a> |
| <span class="sourceLineNo">2705</span> * @param storesToFlush The list of stores to flush.<a name="line.2705"></a> |
| <span class="sourceLineNo">2706</span> * @return object describing the flush's state<a name="line.2706"></a> |
| <span class="sourceLineNo">2707</span> * @throws IOException general io exceptions<a name="line.2707"></a> |
| <span class="sourceLineNo">2708</span> * @throws DroppedSnapshotException Thrown when replay of WAL is required.<a name="line.2708"></a> |
| <span class="sourceLineNo">2709</span> */<a name="line.2709"></a> |
| <span class="sourceLineNo">2710</span> protected FlushResultImpl internalFlushcache(WAL wal, long myseqid,<a name="line.2710"></a> |
| <span class="sourceLineNo">2711</span> Collection<HStore> storesToFlush, MonitoredTask status, boolean writeFlushWalMarker,<a name="line.2711"></a> |
| <span class="sourceLineNo">2712</span> FlushLifeCycleTracker tracker) throws IOException {<a name="line.2712"></a> |
| <span class="sourceLineNo">2713</span> PrepareFlushResult result =<a name="line.2713"></a> |
| <span class="sourceLineNo">2714</span> internalPrepareFlushCache(wal, myseqid, storesToFlush, status, writeFlushWalMarker, tracker);<a name="line.2714"></a> |
| <span class="sourceLineNo">2715</span> if (result.result == null) {<a name="line.2715"></a> |
| <span class="sourceLineNo">2716</span> return internalFlushCacheAndCommit(wal, status, result, storesToFlush);<a name="line.2716"></a> |
| <span class="sourceLineNo">2717</span> } else {<a name="line.2717"></a> |
| <span class="sourceLineNo">2718</span> return result.result; // early exit due to failure from prepare stage<a name="line.2718"></a> |
| <span class="sourceLineNo">2719</span> }<a name="line.2719"></a> |
| <span class="sourceLineNo">2720</span> }<a name="line.2720"></a> |
| <span class="sourceLineNo">2721</span><a name="line.2721"></a> |
| <span class="sourceLineNo">2722</span> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="DLS_DEAD_LOCAL_STORE",<a name="line.2722"></a> |
| <span class="sourceLineNo">2723</span> justification="FindBugs seems confused about trxId")<a name="line.2723"></a> |
| <span class="sourceLineNo">2724</span> protected PrepareFlushResult internalPrepareFlushCache(WAL wal, long myseqid,<a name="line.2724"></a> |
| <span class="sourceLineNo">2725</span> Collection<HStore> storesToFlush, MonitoredTask status, boolean writeFlushWalMarker,<a name="line.2725"></a> |
| <span class="sourceLineNo">2726</span> FlushLifeCycleTracker tracker) throws IOException {<a name="line.2726"></a> |
| <span class="sourceLineNo">2727</span> if (this.rsServices != null && this.rsServices.isAborted()) {<a name="line.2727"></a> |
| <span class="sourceLineNo">2728</span> // Don't flush when server aborting, it's unsafe<a name="line.2728"></a> |
| <span class="sourceLineNo">2729</span> throw new IOException("Aborting flush because server is aborted...");<a name="line.2729"></a> |
| <span class="sourceLineNo">2730</span> }<a name="line.2730"></a> |
| <span class="sourceLineNo">2731</span> final long startTime = EnvironmentEdgeManager.currentTime();<a name="line.2731"></a> |
| <span class="sourceLineNo">2732</span> // If nothing to flush, return, but return with a valid unused sequenceId.<a name="line.2732"></a> |
| <span class="sourceLineNo">2733</span> // Its needed by bulk upload IIRC. It flushes until no edits in memory so it can insert a<a name="line.2733"></a> |
| <span class="sourceLineNo">2734</span> // bulk loaded file between memory and existing hfiles. It wants a good seqeunceId that belongs<a name="line.2734"></a> |
| <span class="sourceLineNo">2735</span> // to no other that it can use to associate with the bulk load. Hence this little dance below<a name="line.2735"></a> |
| <span class="sourceLineNo">2736</span> // to go get one.<a name="line.2736"></a> |
| <span class="sourceLineNo">2737</span> if (this.memStoreSizing.getDataSize() <= 0) {<a name="line.2737"></a> |
| <span class="sourceLineNo">2738</span> // Take an update lock so no edits can come into memory just yet.<a name="line.2738"></a> |
| <span class="sourceLineNo">2739</span> this.updatesLock.writeLock().lock();<a name="line.2739"></a> |
| <span class="sourceLineNo">2740</span> WriteEntry writeEntry = null;<a name="line.2740"></a> |
| <span class="sourceLineNo">2741</span> try {<a name="line.2741"></a> |
| <span class="sourceLineNo">2742</span> if (this.memStoreSizing.getDataSize() <= 0) {<a name="line.2742"></a> |
| <span class="sourceLineNo">2743</span> // Presume that if there are still no edits in the memstore, then there are no edits for<a name="line.2743"></a> |
| <span class="sourceLineNo">2744</span> // this region out in the WAL subsystem so no need to do any trickery clearing out<a name="line.2744"></a> |
| <span class="sourceLineNo">2745</span> // edits in the WAL sub-system. Up the sequence number so the resulting flush id is for<a name="line.2745"></a> |
| <span class="sourceLineNo">2746</span> // sure just beyond the last appended region edit and not associated with any edit<a name="line.2746"></a> |
| <span class="sourceLineNo">2747</span> // (useful as marker when bulk loading, etc.).<a name="line.2747"></a> |
| <span class="sourceLineNo">2748</span> if (wal != null) {<a name="line.2748"></a> |
| <span class="sourceLineNo">2749</span> writeEntry = mvcc.begin();<a name="line.2749"></a> |
| <span class="sourceLineNo">2750</span> long flushOpSeqId = writeEntry.getWriteNumber();<a name="line.2750"></a> |
| <span class="sourceLineNo">2751</span> FlushResultImpl flushResult =<a name="line.2751"></a> |
| <span class="sourceLineNo">2752</span> new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY, flushOpSeqId,<a name="line.2752"></a> |
| <span class="sourceLineNo">2753</span> "Nothing to flush", writeFlushRequestMarkerToWAL(wal, writeFlushWalMarker));<a name="line.2753"></a> |
| <span class="sourceLineNo">2754</span> mvcc.completeAndWait(writeEntry);<a name="line.2754"></a> |
| <span class="sourceLineNo">2755</span> // Set to null so we don't complete it again down in finally block.<a name="line.2755"></a> |
| <span class="sourceLineNo">2756</span> writeEntry = null;<a name="line.2756"></a> |
| <span class="sourceLineNo">2757</span> return new PrepareFlushResult(flushResult, myseqid);<a name="line.2757"></a> |
| <span class="sourceLineNo">2758</span> } else {<a name="line.2758"></a> |
| <span class="sourceLineNo">2759</span> return new PrepareFlushResult(new FlushResultImpl(<a name="line.2759"></a> |
| <span class="sourceLineNo">2760</span> FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY, "Nothing to flush", false), myseqid);<a name="line.2760"></a> |
| <span class="sourceLineNo">2761</span> }<a name="line.2761"></a> |
| <span class="sourceLineNo">2762</span> }<a name="line.2762"></a> |
| <span class="sourceLineNo">2763</span> } finally {<a name="line.2763"></a> |
| <span class="sourceLineNo">2764</span> if (writeEntry != null) {<a name="line.2764"></a> |
| <span class="sourceLineNo">2765</span> // If writeEntry is non-null, this operation failed; the mvcc transaction failed...<a name="line.2765"></a> |
| <span class="sourceLineNo">2766</span> // but complete it anyways so it doesn't block the mvcc queue.<a name="line.2766"></a> |
| <span class="sourceLineNo">2767</span> mvcc.complete(writeEntry);<a name="line.2767"></a> |
| <span class="sourceLineNo">2768</span> }<a name="line.2768"></a> |
| <span class="sourceLineNo">2769</span> this.updatesLock.writeLock().unlock();<a name="line.2769"></a> |
| <span class="sourceLineNo">2770</span> }<a name="line.2770"></a> |
| <span class="sourceLineNo">2771</span> }<a name="line.2771"></a> |
| <span class="sourceLineNo">2772</span> logFatLineOnFlush(storesToFlush, myseqid);<a name="line.2772"></a> |
| <span class="sourceLineNo">2773</span> // Stop updates while we snapshot the memstore of all of these regions' stores. We only have<a name="line.2773"></a> |
| <span class="sourceLineNo">2774</span> // to do this for a moment. It is quick. We also set the memstore size to zero here before we<a name="line.2774"></a> |
| <span class="sourceLineNo">2775</span> // allow updates again so its value will represent the size of the updates received<a name="line.2775"></a> |
| <span class="sourceLineNo">2776</span> // during flush<a name="line.2776"></a> |
| <span class="sourceLineNo">2777</span><a name="line.2777"></a> |
| <span class="sourceLineNo">2778</span> // We have to take an update lock during snapshot, or else a write could end up in both snapshot<a name="line.2778"></a> |
| <span class="sourceLineNo">2779</span> // and memstore (makes it difficult to do atomic rows then)<a name="line.2779"></a> |
| <span class="sourceLineNo">2780</span> status.setStatus("Obtaining lock to block concurrent updates");<a name="line.2780"></a> |
| <span class="sourceLineNo">2781</span> // block waiting for the lock for internal flush<a name="line.2781"></a> |
| <span class="sourceLineNo">2782</span> this.updatesLock.writeLock().lock();<a name="line.2782"></a> |
| <span class="sourceLineNo">2783</span> status.setStatus("Preparing flush snapshotting stores in " + getRegionInfo().getEncodedName());<a name="line.2783"></a> |
| <span class="sourceLineNo">2784</span> MemStoreSizing totalSizeOfFlushableStores = new NonThreadSafeMemStoreSizing();<a name="line.2784"></a> |
| <span class="sourceLineNo">2785</span><a name="line.2785"></a> |
| <span class="sourceLineNo">2786</span> Map<byte[], Long> flushedFamilyNamesToSeq = new HashMap<>();<a name="line.2786"></a> |
| <span class="sourceLineNo">2787</span> for (HStore store : storesToFlush) {<a name="line.2787"></a> |
| <span class="sourceLineNo">2788</span> flushedFamilyNamesToSeq.put(store.getColumnFamilyDescriptor().getName(),<a name="line.2788"></a> |
| <span class="sourceLineNo">2789</span> store.preFlushSeqIDEstimation());<a name="line.2789"></a> |
| <span class="sourceLineNo">2790</span> }<a name="line.2790"></a> |
| <span class="sourceLineNo">2791</span><a name="line.2791"></a> |
| <span class="sourceLineNo">2792</span> TreeMap<byte[], StoreFlushContext> storeFlushCtxs = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.2792"></a> |
| <span class="sourceLineNo">2793</span> TreeMap<byte[], List<Path>> committedFiles = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.2793"></a> |
| <span class="sourceLineNo">2794</span> TreeMap<byte[], MemStoreSize> storeFlushableSize = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.2794"></a> |
| <span class="sourceLineNo">2795</span> // The sequence id of this flush operation which is used to log FlushMarker and pass to<a name="line.2795"></a> |
| <span class="sourceLineNo">2796</span> // createFlushContext to use as the store file's sequence id. It can be in advance of edits<a name="line.2796"></a> |
| <span class="sourceLineNo">2797</span> // still in the memstore, edits that are in other column families yet to be flushed.<a name="line.2797"></a> |
| <span class="sourceLineNo">2798</span> long flushOpSeqId = HConstants.NO_SEQNUM;<a name="line.2798"></a> |
| <span class="sourceLineNo">2799</span> // The max flushed sequence id after this flush operation completes. All edits in memstore<a name="line.2799"></a> |
| <span class="sourceLineNo">2800</span> // will be in advance of this sequence id.<a name="line.2800"></a> |
| <span class="sourceLineNo">2801</span> long flushedSeqId = HConstants.NO_SEQNUM;<a name="line.2801"></a> |
| <span class="sourceLineNo">2802</span> byte[] encodedRegionName = getRegionInfo().getEncodedNameAsBytes();<a name="line.2802"></a> |
| <span class="sourceLineNo">2803</span> try {<a name="line.2803"></a> |
| <span class="sourceLineNo">2804</span> if (wal != null) {<a name="line.2804"></a> |
| <span class="sourceLineNo">2805</span> Long earliestUnflushedSequenceIdForTheRegion =<a name="line.2805"></a> |
| <span class="sourceLineNo">2806</span> wal.startCacheFlush(encodedRegionName, flushedFamilyNamesToSeq);<a name="line.2806"></a> |
| <span class="sourceLineNo">2807</span> if (earliestUnflushedSequenceIdForTheRegion == null) {<a name="line.2807"></a> |
| <span class="sourceLineNo">2808</span> // This should never happen. This is how startCacheFlush signals flush cannot proceed.<a name="line.2808"></a> |
| <span class="sourceLineNo">2809</span> String msg = this.getRegionInfo().getEncodedName() + " flush aborted; WAL closing.";<a name="line.2809"></a> |
| <span class="sourceLineNo">2810</span> status.setStatus(msg);<a name="line.2810"></a> |
| <span class="sourceLineNo">2811</span> return new PrepareFlushResult(<a name="line.2811"></a> |
| <span class="sourceLineNo">2812</span> new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, msg, false),<a name="line.2812"></a> |
| <span class="sourceLineNo">2813</span> myseqid);<a name="line.2813"></a> |
| <span class="sourceLineNo">2814</span> }<a name="line.2814"></a> |
| <span class="sourceLineNo">2815</span> flushOpSeqId = getNextSequenceId(wal);<a name="line.2815"></a> |
| <span class="sourceLineNo">2816</span> // Back up 1, minus 1 from oldest sequence id in memstore to get last 'flushed' edit<a name="line.2816"></a> |
| <span class="sourceLineNo">2817</span> flushedSeqId =<a name="line.2817"></a> |
| <span class="sourceLineNo">2818</span> earliestUnflushedSequenceIdForTheRegion.longValue() == HConstants.NO_SEQNUM?<a name="line.2818"></a> |
| <span class="sourceLineNo">2819</span> flushOpSeqId: earliestUnflushedSequenceIdForTheRegion.longValue() - 1;<a name="line.2819"></a> |
| <span class="sourceLineNo">2820</span> } else {<a name="line.2820"></a> |
| <span class="sourceLineNo">2821</span> // use the provided sequence Id as WAL is not being used for this flush.<a name="line.2821"></a> |
| <span class="sourceLineNo">2822</span> flushedSeqId = flushOpSeqId = myseqid;<a name="line.2822"></a> |
| <span class="sourceLineNo">2823</span> }<a name="line.2823"></a> |
| <span class="sourceLineNo">2824</span><a name="line.2824"></a> |
| <span class="sourceLineNo">2825</span> for (HStore s : storesToFlush) {<a name="line.2825"></a> |
| <span class="sourceLineNo">2826</span> storeFlushCtxs.put(s.getColumnFamilyDescriptor().getName(),<a name="line.2826"></a> |
| <span class="sourceLineNo">2827</span> s.createFlushContext(flushOpSeqId, tracker));<a name="line.2827"></a> |
| <span class="sourceLineNo">2828</span> // for writing stores to WAL<a name="line.2828"></a> |
| <span class="sourceLineNo">2829</span> committedFiles.put(s.getColumnFamilyDescriptor().getName(), null);<a name="line.2829"></a> |
| <span class="sourceLineNo">2830</span> }<a name="line.2830"></a> |
| <span class="sourceLineNo">2831</span><a name="line.2831"></a> |
| <span class="sourceLineNo">2832</span> // write the snapshot start to WAL<a name="line.2832"></a> |
| <span class="sourceLineNo">2833</span> if (wal != null && !writestate.readOnly) {<a name="line.2833"></a> |
| <span class="sourceLineNo">2834</span> FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.START_FLUSH,<a name="line.2834"></a> |
| <span class="sourceLineNo">2835</span> getRegionInfo(), flushOpSeqId, committedFiles);<a name="line.2835"></a> |
| <span class="sourceLineNo">2836</span> // No sync. Sync is below where no updates lock and we do FlushAction.COMMIT_FLUSH<a name="line.2836"></a> |
| <span class="sourceLineNo">2837</span> WALUtil.writeFlushMarker(wal, this.getReplicationScope(), getRegionInfo(), desc, false,<a name="line.2837"></a> |
| <span class="sourceLineNo">2838</span> mvcc);<a name="line.2838"></a> |
| <span class="sourceLineNo">2839</span> }<a name="line.2839"></a> |
| <span class="sourceLineNo">2840</span><a name="line.2840"></a> |
| <span class="sourceLineNo">2841</span> // Prepare flush (take a snapshot)<a name="line.2841"></a> |
| <span class="sourceLineNo">2842</span> storeFlushCtxs.forEach((name, flush) -> {<a name="line.2842"></a> |
| <span class="sourceLineNo">2843</span> MemStoreSize snapshotSize = flush.prepare();<a name="line.2843"></a> |
| <span class="sourceLineNo">2844</span> totalSizeOfFlushableStores.incMemStoreSize(snapshotSize);<a name="line.2844"></a> |
| <span class="sourceLineNo">2845</span> storeFlushableSize.put(name, snapshotSize);<a name="line.2845"></a> |
| <span class="sourceLineNo">2846</span> });<a name="line.2846"></a> |
| <span class="sourceLineNo">2847</span> } catch (IOException ex) {<a name="line.2847"></a> |
| <span class="sourceLineNo">2848</span> doAbortFlushToWAL(wal, flushOpSeqId, committedFiles);<a name="line.2848"></a> |
| <span class="sourceLineNo">2849</span> throw ex;<a name="line.2849"></a> |
| <span class="sourceLineNo">2850</span> } finally {<a name="line.2850"></a> |
| <span class="sourceLineNo">2851</span> this.updatesLock.writeLock().unlock();<a name="line.2851"></a> |
| <span class="sourceLineNo">2852</span> }<a name="line.2852"></a> |
| <span class="sourceLineNo">2853</span> String s = "Finished memstore snapshotting " + this + ", syncing WAL and waiting on mvcc, " +<a name="line.2853"></a> |
| <span class="sourceLineNo">2854</span> "flushsize=" + totalSizeOfFlushableStores;<a name="line.2854"></a> |
| <span class="sourceLineNo">2855</span> status.setStatus(s);<a name="line.2855"></a> |
| <span class="sourceLineNo">2856</span> doSyncOfUnflushedWALChanges(wal, getRegionInfo());<a name="line.2856"></a> |
| <span class="sourceLineNo">2857</span> return new PrepareFlushResult(storeFlushCtxs, committedFiles, storeFlushableSize, startTime,<a name="line.2857"></a> |
| <span class="sourceLineNo">2858</span> flushOpSeqId, flushedSeqId, totalSizeOfFlushableStores);<a name="line.2858"></a> |
| <span class="sourceLineNo">2859</span> }<a name="line.2859"></a> |
| <span class="sourceLineNo">2860</span><a name="line.2860"></a> |
| <span class="sourceLineNo">2861</span> /**<a name="line.2861"></a> |
| <span class="sourceLineNo">2862</span> * Utility method broken out of internalPrepareFlushCache so that method is smaller.<a name="line.2862"></a> |
| <span class="sourceLineNo">2863</span> */<a name="line.2863"></a> |
| <span class="sourceLineNo">2864</span> private void logFatLineOnFlush(Collection<HStore> storesToFlush, long sequenceId) {<a name="line.2864"></a> |
| <span class="sourceLineNo">2865</span> if (!LOG.isInfoEnabled()) {<a name="line.2865"></a> |
| <span class="sourceLineNo">2866</span> return;<a name="line.2866"></a> |
| <span class="sourceLineNo">2867</span> }<a name="line.2867"></a> |
| <span class="sourceLineNo">2868</span> // Log a fat line detailing what is being flushed.<a name="line.2868"></a> |
| <span class="sourceLineNo">2869</span> StringBuilder perCfExtras = null;<a name="line.2869"></a> |
| <span class="sourceLineNo">2870</span> if (!isAllFamilies(storesToFlush)) {<a name="line.2870"></a> |
| <span class="sourceLineNo">2871</span> perCfExtras = new StringBuilder();<a name="line.2871"></a> |
| <span class="sourceLineNo">2872</span> for (HStore store: storesToFlush) {<a name="line.2872"></a> |
| <span class="sourceLineNo">2873</span> MemStoreSize mss = store.getFlushableSize();<a name="line.2873"></a> |
| <span class="sourceLineNo">2874</span> perCfExtras.append("; ").append(store.getColumnFamilyName());<a name="line.2874"></a> |
| <span class="sourceLineNo">2875</span> perCfExtras.append("={dataSize=")<a name="line.2875"></a> |
| <span class="sourceLineNo">2876</span> .append(StringUtils.byteDesc(mss.getDataSize()));<a name="line.2876"></a> |
| <span class="sourceLineNo">2877</span> perCfExtras.append(", heapSize=")<a name="line.2877"></a> |
| <span class="sourceLineNo">2878</span> .append(StringUtils.byteDesc(mss.getHeapSize()));<a name="line.2878"></a> |
| <span class="sourceLineNo">2879</span> perCfExtras.append(", offHeapSize=")<a name="line.2879"></a> |
| <span class="sourceLineNo">2880</span> .append(StringUtils.byteDesc(mss.getOffHeapSize()));<a name="line.2880"></a> |
| <span class="sourceLineNo">2881</span> perCfExtras.append("}");<a name="line.2881"></a> |
| <span class="sourceLineNo">2882</span> }<a name="line.2882"></a> |
| <span class="sourceLineNo">2883</span> }<a name="line.2883"></a> |
| <span class="sourceLineNo">2884</span> MemStoreSize mss = this.memStoreSizing.getMemStoreSize();<a name="line.2884"></a> |
| <span class="sourceLineNo">2885</span> LOG.info("Flushing " + this.getRegionInfo().getEncodedName() + " " +<a name="line.2885"></a> |
| <span class="sourceLineNo">2886</span> storesToFlush.size() + "/" + stores.size() + " column families," +<a name="line.2886"></a> |
| <span class="sourceLineNo">2887</span> " dataSize=" + StringUtils.byteDesc(mss.getDataSize()) +<a name="line.2887"></a> |
| <span class="sourceLineNo">2888</span> " heapSize=" + StringUtils.byteDesc(mss.getHeapSize()) +<a name="line.2888"></a> |
| <span class="sourceLineNo">2889</span> ((perCfExtras != null && perCfExtras.length() > 0)? perCfExtras.toString(): "") +<a name="line.2889"></a> |
| <span class="sourceLineNo">2890</span> ((wal != null) ? "" : "; WAL is null, using passed sequenceid=" + sequenceId));<a name="line.2890"></a> |
| <span class="sourceLineNo">2891</span> }<a name="line.2891"></a> |
| <span class="sourceLineNo">2892</span><a name="line.2892"></a> |
| <span class="sourceLineNo">2893</span> private void doAbortFlushToWAL(final WAL wal, final long flushOpSeqId,<a name="line.2893"></a> |
| <span class="sourceLineNo">2894</span> final Map<byte[], List<Path>> committedFiles) {<a name="line.2894"></a> |
| <span class="sourceLineNo">2895</span> if (wal == null) return;<a name="line.2895"></a> |
| <span class="sourceLineNo">2896</span> try {<a name="line.2896"></a> |
| <span class="sourceLineNo">2897</span> FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.ABORT_FLUSH,<a name="line.2897"></a> |
| <span class="sourceLineNo">2898</span> getRegionInfo(), flushOpSeqId, committedFiles);<a name="line.2898"></a> |
| <span class="sourceLineNo">2899</span> WALUtil.writeFlushMarker(wal, this.getReplicationScope(), getRegionInfo(), desc, false,<a name="line.2899"></a> |
| <span class="sourceLineNo">2900</span> mvcc);<a name="line.2900"></a> |
| <span class="sourceLineNo">2901</span> } catch (Throwable t) {<a name="line.2901"></a> |
| <span class="sourceLineNo">2902</span> LOG.warn("Received unexpected exception trying to write ABORT_FLUSH marker to WAL: {} in "<a name="line.2902"></a> |
| <span class="sourceLineNo">2903</span> + " region {}", StringUtils.stringifyException(t), this);<a name="line.2903"></a> |
| <span class="sourceLineNo">2904</span> // ignore this since we will be aborting the RS with DSE.<a name="line.2904"></a> |
| <span class="sourceLineNo">2905</span> }<a name="line.2905"></a> |
| <span class="sourceLineNo">2906</span> // we have called wal.startCacheFlush(), now we have to abort it<a name="line.2906"></a> |
| <span class="sourceLineNo">2907</span> wal.abortCacheFlush(this.getRegionInfo().getEncodedNameAsBytes());<a name="line.2907"></a> |
| <span class="sourceLineNo">2908</span> }<a name="line.2908"></a> |
| <span class="sourceLineNo">2909</span><a name="line.2909"></a> |
| <span class="sourceLineNo">2910</span> /**<a name="line.2910"></a> |
| <span class="sourceLineNo">2911</span> * Sync unflushed WAL changes. See HBASE-8208 for details<a name="line.2911"></a> |
| <span class="sourceLineNo">2912</span> */<a name="line.2912"></a> |
| <span class="sourceLineNo">2913</span> private static void doSyncOfUnflushedWALChanges(final WAL wal, final RegionInfo hri)<a name="line.2913"></a> |
| <span class="sourceLineNo">2914</span> throws IOException {<a name="line.2914"></a> |
| <span class="sourceLineNo">2915</span> if (wal == null) {<a name="line.2915"></a> |
| <span class="sourceLineNo">2916</span> return;<a name="line.2916"></a> |
| <span class="sourceLineNo">2917</span> }<a name="line.2917"></a> |
| <span class="sourceLineNo">2918</span> try {<a name="line.2918"></a> |
| <span class="sourceLineNo">2919</span> wal.sync(); // ensure that flush marker is sync'ed<a name="line.2919"></a> |
| <span class="sourceLineNo">2920</span> } catch (IOException ioe) {<a name="line.2920"></a> |
| <span class="sourceLineNo">2921</span> wal.abortCacheFlush(hri.getEncodedNameAsBytes());<a name="line.2921"></a> |
| <span class="sourceLineNo">2922</span> throw ioe;<a name="line.2922"></a> |
| <span class="sourceLineNo">2923</span> }<a name="line.2923"></a> |
| <span class="sourceLineNo">2924</span> }<a name="line.2924"></a> |
| <span class="sourceLineNo">2925</span><a name="line.2925"></a> |
| <span class="sourceLineNo">2926</span> /**<a name="line.2926"></a> |
| <span class="sourceLineNo">2927</span> * @return True if passed Set is all families in the region.<a name="line.2927"></a> |
| <span class="sourceLineNo">2928</span> */<a name="line.2928"></a> |
| <span class="sourceLineNo">2929</span> private boolean isAllFamilies(Collection<HStore> families) {<a name="line.2929"></a> |
| <span class="sourceLineNo">2930</span> return families == null || this.stores.size() == families.size();<a name="line.2930"></a> |
| <span class="sourceLineNo">2931</span> }<a name="line.2931"></a> |
| <span class="sourceLineNo">2932</span><a name="line.2932"></a> |
| <span class="sourceLineNo">2933</span> /**<a name="line.2933"></a> |
| <span class="sourceLineNo">2934</span> * Writes a marker to WAL indicating a flush is requested but cannot be complete due to various<a name="line.2934"></a> |
| <span class="sourceLineNo">2935</span> * reasons. Ignores exceptions from WAL. Returns whether the write succeeded.<a name="line.2935"></a> |
| <span class="sourceLineNo">2936</span> * @param wal<a name="line.2936"></a> |
| <span class="sourceLineNo">2937</span> * @return whether WAL write was successful<a name="line.2937"></a> |
| <span class="sourceLineNo">2938</span> */<a name="line.2938"></a> |
| <span class="sourceLineNo">2939</span> private boolean writeFlushRequestMarkerToWAL(WAL wal, boolean writeFlushWalMarker) {<a name="line.2939"></a> |
| <span class="sourceLineNo">2940</span> if (writeFlushWalMarker && wal != null && !writestate.readOnly) {<a name="line.2940"></a> |
| <span class="sourceLineNo">2941</span> FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.CANNOT_FLUSH,<a name="line.2941"></a> |
| <span class="sourceLineNo">2942</span> getRegionInfo(), -1, new TreeMap<>(Bytes.BYTES_COMPARATOR));<a name="line.2942"></a> |
| <span class="sourceLineNo">2943</span> try {<a name="line.2943"></a> |
| <span class="sourceLineNo">2944</span> WALUtil.writeFlushMarker(wal, this.getReplicationScope(), getRegionInfo(), desc, true,<a name="line.2944"></a> |
| <span class="sourceLineNo">2945</span> mvcc);<a name="line.2945"></a> |
| <span class="sourceLineNo">2946</span> return true;<a name="line.2946"></a> |
| <span class="sourceLineNo">2947</span> } catch (IOException e) {<a name="line.2947"></a> |
| <span class="sourceLineNo">2948</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.2948"></a> |
| <span class="sourceLineNo">2949</span> + "Received exception while trying to write the flush request to wal", e);<a name="line.2949"></a> |
| <span class="sourceLineNo">2950</span> }<a name="line.2950"></a> |
| <span class="sourceLineNo">2951</span> }<a name="line.2951"></a> |
| <span class="sourceLineNo">2952</span> return false;<a name="line.2952"></a> |
| <span class="sourceLineNo">2953</span> }<a name="line.2953"></a> |
| <span class="sourceLineNo">2954</span><a name="line.2954"></a> |
| <span class="sourceLineNo">2955</span> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NN_NAKED_NOTIFY",<a name="line.2955"></a> |
| <span class="sourceLineNo">2956</span> justification="Intentional; notify is about completed flush")<a name="line.2956"></a> |
| <span class="sourceLineNo">2957</span> FlushResultImpl internalFlushCacheAndCommit(WAL wal, MonitoredTask status,<a name="line.2957"></a> |
| <span class="sourceLineNo">2958</span> PrepareFlushResult prepareResult, Collection<HStore> storesToFlush) throws IOException {<a name="line.2958"></a> |
| <span class="sourceLineNo">2959</span> // prepare flush context is carried via PrepareFlushResult<a name="line.2959"></a> |
| <span class="sourceLineNo">2960</span> TreeMap<byte[], StoreFlushContext> storeFlushCtxs = prepareResult.storeFlushCtxs;<a name="line.2960"></a> |
| <span class="sourceLineNo">2961</span> TreeMap<byte[], List<Path>> committedFiles = prepareResult.committedFiles;<a name="line.2961"></a> |
| <span class="sourceLineNo">2962</span> long startTime = prepareResult.startTime;<a name="line.2962"></a> |
| <span class="sourceLineNo">2963</span> long flushOpSeqId = prepareResult.flushOpSeqId;<a name="line.2963"></a> |
| <span class="sourceLineNo">2964</span> long flushedSeqId = prepareResult.flushedSeqId;<a name="line.2964"></a> |
| <span class="sourceLineNo">2965</span><a name="line.2965"></a> |
| <span class="sourceLineNo">2966</span> String s = "Flushing stores of " + this;<a name="line.2966"></a> |
| <span class="sourceLineNo">2967</span> status.setStatus(s);<a name="line.2967"></a> |
| <span class="sourceLineNo">2968</span> if (LOG.isTraceEnabled()) LOG.trace(s);<a name="line.2968"></a> |
| <span class="sourceLineNo">2969</span><a name="line.2969"></a> |
| <span class="sourceLineNo">2970</span> // Any failure from here on out will be catastrophic requiring server<a name="line.2970"></a> |
| <span class="sourceLineNo">2971</span> // restart so wal content can be replayed and put back into the memstore.<a name="line.2971"></a> |
| <span class="sourceLineNo">2972</span> // Otherwise, the snapshot content while backed up in the wal, it will not<a name="line.2972"></a> |
| <span class="sourceLineNo">2973</span> // be part of the current running servers state.<a name="line.2973"></a> |
| <span class="sourceLineNo">2974</span> boolean compactionRequested = false;<a name="line.2974"></a> |
| <span class="sourceLineNo">2975</span> long flushedOutputFileSize = 0;<a name="line.2975"></a> |
| <span class="sourceLineNo">2976</span> try {<a name="line.2976"></a> |
| <span class="sourceLineNo">2977</span> // A. Flush memstore to all the HStores.<a name="line.2977"></a> |
| <span class="sourceLineNo">2978</span> // Keep running vector of all store files that includes both old and the<a name="line.2978"></a> |
| <span class="sourceLineNo">2979</span> // just-made new flush store file. The new flushed file is still in the<a name="line.2979"></a> |
| <span class="sourceLineNo">2980</span> // tmp directory.<a name="line.2980"></a> |
| <span class="sourceLineNo">2981</span><a name="line.2981"></a> |
| <span class="sourceLineNo">2982</span> for (StoreFlushContext flush : storeFlushCtxs.values()) {<a name="line.2982"></a> |
| <span class="sourceLineNo">2983</span> flush.flushCache(status);<a name="line.2983"></a> |
| <span class="sourceLineNo">2984</span> }<a name="line.2984"></a> |
| <span class="sourceLineNo">2985</span><a name="line.2985"></a> |
| <span class="sourceLineNo">2986</span> // Switch snapshot (in memstore) -> new hfile (thus causing<a name="line.2986"></a> |
| <span class="sourceLineNo">2987</span> // all the store scanners to reset/reseek).<a name="line.2987"></a> |
| <span class="sourceLineNo">2988</span> for (Map.Entry<byte[], StoreFlushContext> flushEntry : storeFlushCtxs.entrySet()) {<a name="line.2988"></a> |
| <span class="sourceLineNo">2989</span> StoreFlushContext sfc = flushEntry.getValue();<a name="line.2989"></a> |
| <span class="sourceLineNo">2990</span> boolean needsCompaction = sfc.commit(status);<a name="line.2990"></a> |
| <span class="sourceLineNo">2991</span> if (needsCompaction) {<a name="line.2991"></a> |
| <span class="sourceLineNo">2992</span> compactionRequested = true;<a name="line.2992"></a> |
| <span class="sourceLineNo">2993</span> }<a name="line.2993"></a> |
| <span class="sourceLineNo">2994</span> byte[] storeName = flushEntry.getKey();<a name="line.2994"></a> |
| <span class="sourceLineNo">2995</span> List<Path> storeCommittedFiles = sfc.getCommittedFiles();<a name="line.2995"></a> |
| <span class="sourceLineNo">2996</span> committedFiles.put(storeName, storeCommittedFiles);<a name="line.2996"></a> |
| <span class="sourceLineNo">2997</span> // Flush committed no files, indicating flush is empty or flush was canceled<a name="line.2997"></a> |
| <span class="sourceLineNo">2998</span> if (storeCommittedFiles == null || storeCommittedFiles.isEmpty()) {<a name="line.2998"></a> |
| <span class="sourceLineNo">2999</span> MemStoreSize storeFlushableSize = prepareResult.storeFlushableSize.get(storeName);<a name="line.2999"></a> |
| <span class="sourceLineNo">3000</span> prepareResult.totalFlushableSize.decMemStoreSize(storeFlushableSize);<a name="line.3000"></a> |
| <span class="sourceLineNo">3001</span> }<a name="line.3001"></a> |
| <span class="sourceLineNo">3002</span> flushedOutputFileSize += sfc.getOutputFileSize();<a name="line.3002"></a> |
| <span class="sourceLineNo">3003</span> }<a name="line.3003"></a> |
| <span class="sourceLineNo">3004</span> storeFlushCtxs.clear();<a name="line.3004"></a> |
| <span class="sourceLineNo">3005</span><a name="line.3005"></a> |
| <span class="sourceLineNo">3006</span> // Set down the memstore size by amount of flush.<a name="line.3006"></a> |
| <span class="sourceLineNo">3007</span> MemStoreSize mss = prepareResult.totalFlushableSize.getMemStoreSize();<a name="line.3007"></a> |
| <span class="sourceLineNo">3008</span> this.decrMemStoreSize(mss);<a name="line.3008"></a> |
| <span class="sourceLineNo">3009</span><a name="line.3009"></a> |
| <span class="sourceLineNo">3010</span> // Increase the size of this Region for the purposes of quota. Noop if quotas are disabled.<a name="line.3010"></a> |
| <span class="sourceLineNo">3011</span> // During startup, quota manager may not be initialized yet.<a name="line.3011"></a> |
| <span class="sourceLineNo">3012</span> if (rsServices != null) {<a name="line.3012"></a> |
| <span class="sourceLineNo">3013</span> RegionServerSpaceQuotaManager quotaManager = rsServices.getRegionServerSpaceQuotaManager();<a name="line.3013"></a> |
| <span class="sourceLineNo">3014</span> if (quotaManager != null) {<a name="line.3014"></a> |
| <span class="sourceLineNo">3015</span> quotaManager.getRegionSizeStore().incrementRegionSize(<a name="line.3015"></a> |
| <span class="sourceLineNo">3016</span> this.getRegionInfo(), flushedOutputFileSize);<a name="line.3016"></a> |
| <span class="sourceLineNo">3017</span> }<a name="line.3017"></a> |
| <span class="sourceLineNo">3018</span> }<a name="line.3018"></a> |
| <span class="sourceLineNo">3019</span><a name="line.3019"></a> |
| <span class="sourceLineNo">3020</span> if (wal != null) {<a name="line.3020"></a> |
| <span class="sourceLineNo">3021</span> // write flush marker to WAL. If fail, we should throw DroppedSnapshotException<a name="line.3021"></a> |
| <span class="sourceLineNo">3022</span> FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.COMMIT_FLUSH,<a name="line.3022"></a> |
| <span class="sourceLineNo">3023</span> getRegionInfo(), flushOpSeqId, committedFiles);<a name="line.3023"></a> |
| <span class="sourceLineNo">3024</span> WALUtil.writeFlushMarker(wal, this.getReplicationScope(), getRegionInfo(), desc, true,<a name="line.3024"></a> |
| <span class="sourceLineNo">3025</span> mvcc);<a name="line.3025"></a> |
| <span class="sourceLineNo">3026</span> }<a name="line.3026"></a> |
| <span class="sourceLineNo">3027</span> } catch (Throwable t) {<a name="line.3027"></a> |
| <span class="sourceLineNo">3028</span> // An exception here means that the snapshot was not persisted.<a name="line.3028"></a> |
| <span class="sourceLineNo">3029</span> // The wal needs to be replayed so its content is restored to memstore.<a name="line.3029"></a> |
| <span class="sourceLineNo">3030</span> // Currently, only a server restart will do this.<a name="line.3030"></a> |
| <span class="sourceLineNo">3031</span> // We used to only catch IOEs but its possible that we'd get other<a name="line.3031"></a> |
| <span class="sourceLineNo">3032</span> // exceptions -- e.g. HBASE-659 was about an NPE -- so now we catch<a name="line.3032"></a> |
| <span class="sourceLineNo">3033</span> // all and sundry.<a name="line.3033"></a> |
| <span class="sourceLineNo">3034</span> if (wal != null) {<a name="line.3034"></a> |
| <span class="sourceLineNo">3035</span> try {<a name="line.3035"></a> |
| <span class="sourceLineNo">3036</span> FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.ABORT_FLUSH,<a name="line.3036"></a> |
| <span class="sourceLineNo">3037</span> getRegionInfo(), flushOpSeqId, committedFiles);<a name="line.3037"></a> |
| <span class="sourceLineNo">3038</span> WALUtil.writeFlushMarker(wal, this.replicationScope, getRegionInfo(), desc, false, mvcc);<a name="line.3038"></a> |
| <span class="sourceLineNo">3039</span> } catch (Throwable ex) {<a name="line.3039"></a> |
| <span class="sourceLineNo">3040</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.3040"></a> |
| <span class="sourceLineNo">3041</span> + "failed writing ABORT_FLUSH marker to WAL", ex);<a name="line.3041"></a> |
| <span class="sourceLineNo">3042</span> // ignore this since we will be aborting the RS with DSE.<a name="line.3042"></a> |
| <span class="sourceLineNo">3043</span> }<a name="line.3043"></a> |
| <span class="sourceLineNo">3044</span> wal.abortCacheFlush(this.getRegionInfo().getEncodedNameAsBytes());<a name="line.3044"></a> |
| <span class="sourceLineNo">3045</span> }<a name="line.3045"></a> |
| <span class="sourceLineNo">3046</span> DroppedSnapshotException dse = new DroppedSnapshotException("region: " +<a name="line.3046"></a> |
| <span class="sourceLineNo">3047</span> Bytes.toStringBinary(getRegionInfo().getRegionName()), t);<a name="line.3047"></a> |
| <span class="sourceLineNo">3048</span> status.abort("Flush failed: " + StringUtils.stringifyException(t));<a name="line.3048"></a> |
| <span class="sourceLineNo">3049</span><a name="line.3049"></a> |
| <span class="sourceLineNo">3050</span> // Callers for flushcache() should catch DroppedSnapshotException and abort the region server.<a name="line.3050"></a> |
| <span class="sourceLineNo">3051</span> // However, since we may have the region read lock, we cannot call close(true) here since<a name="line.3051"></a> |
| <span class="sourceLineNo">3052</span> // we cannot promote to a write lock. Instead we are setting closing so that all other region<a name="line.3052"></a> |
| <span class="sourceLineNo">3053</span> // operations except for close will be rejected.<a name="line.3053"></a> |
| <span class="sourceLineNo">3054</span> this.closing.set(true);<a name="line.3054"></a> |
| <span class="sourceLineNo">3055</span><a name="line.3055"></a> |
| <span class="sourceLineNo">3056</span> if (rsServices != null) {<a name="line.3056"></a> |
| <span class="sourceLineNo">3057</span> // This is a safeguard against the case where the caller fails to explicitly handle aborting<a name="line.3057"></a> |
| <span class="sourceLineNo">3058</span> rsServices.abort("Replay of WAL required. Forcing server shutdown", dse);<a name="line.3058"></a> |
| <span class="sourceLineNo">3059</span> }<a name="line.3059"></a> |
| <span class="sourceLineNo">3060</span><a name="line.3060"></a> |
| <span class="sourceLineNo">3061</span> throw dse;<a name="line.3061"></a> |
| <span class="sourceLineNo">3062</span> }<a name="line.3062"></a> |
| <span class="sourceLineNo">3063</span><a name="line.3063"></a> |
| <span class="sourceLineNo">3064</span> // If we get to here, the HStores have been written.<a name="line.3064"></a> |
| <span class="sourceLineNo">3065</span> if (wal != null) {<a name="line.3065"></a> |
| <span class="sourceLineNo">3066</span> wal.completeCacheFlush(this.getRegionInfo().getEncodedNameAsBytes(), flushedSeqId);<a name="line.3066"></a> |
| <span class="sourceLineNo">3067</span> }<a name="line.3067"></a> |
| <span class="sourceLineNo">3068</span><a name="line.3068"></a> |
| <span class="sourceLineNo">3069</span> // Record latest flush time<a name="line.3069"></a> |
| <span class="sourceLineNo">3070</span> for (HStore store: storesToFlush) {<a name="line.3070"></a> |
| <span class="sourceLineNo">3071</span> this.lastStoreFlushTimeMap.put(store, startTime);<a name="line.3071"></a> |
| <span class="sourceLineNo">3072</span> }<a name="line.3072"></a> |
| <span class="sourceLineNo">3073</span><a name="line.3073"></a> |
| <span class="sourceLineNo">3074</span> this.maxFlushedSeqId = flushedSeqId;<a name="line.3074"></a> |
| <span class="sourceLineNo">3075</span> this.lastFlushOpSeqId = flushOpSeqId;<a name="line.3075"></a> |
| <span class="sourceLineNo">3076</span><a name="line.3076"></a> |
| <span class="sourceLineNo">3077</span> // C. Finally notify anyone waiting on memstore to clear:<a name="line.3077"></a> |
| <span class="sourceLineNo">3078</span> // e.g. checkResources().<a name="line.3078"></a> |
| <span class="sourceLineNo">3079</span> synchronized (this) {<a name="line.3079"></a> |
| <span class="sourceLineNo">3080</span> notifyAll(); // FindBugs NN_NAKED_NOTIFY<a name="line.3080"></a> |
| <span class="sourceLineNo">3081</span> }<a name="line.3081"></a> |
| <span class="sourceLineNo">3082</span><a name="line.3082"></a> |
| <span class="sourceLineNo">3083</span> long time = EnvironmentEdgeManager.currentTime() - startTime;<a name="line.3083"></a> |
| <span class="sourceLineNo">3084</span> MemStoreSize mss = prepareResult.totalFlushableSize.getMemStoreSize();<a name="line.3084"></a> |
| <span class="sourceLineNo">3085</span> long memstoresize = this.memStoreSizing.getMemStoreSize().getDataSize();<a name="line.3085"></a> |
| <span class="sourceLineNo">3086</span> String msg = "Finished flush of"<a name="line.3086"></a> |
| <span class="sourceLineNo">3087</span> + " dataSize ~" + StringUtils.byteDesc(mss.getDataSize()) + "/" + mss.getDataSize()<a name="line.3087"></a> |
| <span class="sourceLineNo">3088</span> + ", heapSize ~" + StringUtils.byteDesc(mss.getHeapSize()) + "/" + mss.getHeapSize()<a name="line.3088"></a> |
| <span class="sourceLineNo">3089</span> + ", currentSize=" + StringUtils.byteDesc(memstoresize) + "/" + memstoresize<a name="line.3089"></a> |
| <span class="sourceLineNo">3090</span> + " for " + this.getRegionInfo().getEncodedName() + " in " + time + "ms, sequenceid="<a name="line.3090"></a> |
| <span class="sourceLineNo">3091</span> + flushOpSeqId + ", compaction requested=" + compactionRequested<a name="line.3091"></a> |
| <span class="sourceLineNo">3092</span> + ((wal == null) ? "; wal=null" : "");<a name="line.3092"></a> |
| <span class="sourceLineNo">3093</span> LOG.info(msg);<a name="line.3093"></a> |
| <span class="sourceLineNo">3094</span> status.setStatus(msg);<a name="line.3094"></a> |
| <span class="sourceLineNo">3095</span><a name="line.3095"></a> |
| <span class="sourceLineNo">3096</span> if (rsServices != null && rsServices.getMetrics() != null) {<a name="line.3096"></a> |
| <span class="sourceLineNo">3097</span> rsServices.getMetrics().updateFlush(getTableDescriptor().getTableName().getNameAsString(),<a name="line.3097"></a> |
| <span class="sourceLineNo">3098</span> time,<a name="line.3098"></a> |
| <span class="sourceLineNo">3099</span> mss.getDataSize(), flushedOutputFileSize);<a name="line.3099"></a> |
| <span class="sourceLineNo">3100</span> }<a name="line.3100"></a> |
| <span class="sourceLineNo">3101</span><a name="line.3101"></a> |
| <span class="sourceLineNo">3102</span> return new FlushResultImpl(compactionRequested ?<a name="line.3102"></a> |
| <span class="sourceLineNo">3103</span> FlushResult.Result.FLUSHED_COMPACTION_NEEDED :<a name="line.3103"></a> |
| <span class="sourceLineNo">3104</span> FlushResult.Result.FLUSHED_NO_COMPACTION_NEEDED, flushOpSeqId);<a name="line.3104"></a> |
| <span class="sourceLineNo">3105</span> }<a name="line.3105"></a> |
| <span class="sourceLineNo">3106</span><a name="line.3106"></a> |
| <span class="sourceLineNo">3107</span> /**<a name="line.3107"></a> |
| <span class="sourceLineNo">3108</span> * Method to safely get the next sequence number.<a name="line.3108"></a> |
| <span class="sourceLineNo">3109</span> * @return Next sequence number unassociated with any actual edit.<a name="line.3109"></a> |
| <span class="sourceLineNo">3110</span> * @throws IOException<a name="line.3110"></a> |
| <span class="sourceLineNo">3111</span> */<a name="line.3111"></a> |
| <span class="sourceLineNo">3112</span> protected long getNextSequenceId(final WAL wal) throws IOException {<a name="line.3112"></a> |
| <span class="sourceLineNo">3113</span> WriteEntry we = mvcc.begin();<a name="line.3113"></a> |
| <span class="sourceLineNo">3114</span> mvcc.completeAndWait(we);<a name="line.3114"></a> |
| <span class="sourceLineNo">3115</span> return we.getWriteNumber();<a name="line.3115"></a> |
| <span class="sourceLineNo">3116</span> }<a name="line.3116"></a> |
| <span class="sourceLineNo">3117</span><a name="line.3117"></a> |
| <span class="sourceLineNo">3118</span> //////////////////////////////////////////////////////////////////////////////<a name="line.3118"></a> |
| <span class="sourceLineNo">3119</span> // get() methods for client use.<a name="line.3119"></a> |
| <span class="sourceLineNo">3120</span> //////////////////////////////////////////////////////////////////////////////<a name="line.3120"></a> |
| <span class="sourceLineNo">3121</span><a name="line.3121"></a> |
| <span class="sourceLineNo">3122</span> @Override<a name="line.3122"></a> |
| <span class="sourceLineNo">3123</span> public RegionScannerImpl getScanner(Scan scan) throws IOException {<a name="line.3123"></a> |
| <span class="sourceLineNo">3124</span> return getScanner(scan, null);<a name="line.3124"></a> |
| <span class="sourceLineNo">3125</span> }<a name="line.3125"></a> |
| <span class="sourceLineNo">3126</span><a name="line.3126"></a> |
| <span class="sourceLineNo">3127</span> @Override<a name="line.3127"></a> |
| <span class="sourceLineNo">3128</span> public RegionScannerImpl getScanner(Scan scan, List<KeyValueScanner> additionalScanners)<a name="line.3128"></a> |
| <span class="sourceLineNo">3129</span> throws IOException {<a name="line.3129"></a> |
| <span class="sourceLineNo">3130</span> return getScanner(scan, additionalScanners, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.3130"></a> |
| <span class="sourceLineNo">3131</span> }<a name="line.3131"></a> |
| <span class="sourceLineNo">3132</span><a name="line.3132"></a> |
| <span class="sourceLineNo">3133</span> private RegionScannerImpl getScanner(Scan scan, List<KeyValueScanner> additionalScanners,<a name="line.3133"></a> |
| <span class="sourceLineNo">3134</span> long nonceGroup, long nonce) throws IOException {<a name="line.3134"></a> |
| <span class="sourceLineNo">3135</span> startRegionOperation(Operation.SCAN);<a name="line.3135"></a> |
| <span class="sourceLineNo">3136</span> try {<a name="line.3136"></a> |
| <span class="sourceLineNo">3137</span> // Verify families are all valid<a name="line.3137"></a> |
| <span class="sourceLineNo">3138</span> if (!scan.hasFamilies()) {<a name="line.3138"></a> |
| <span class="sourceLineNo">3139</span> // Adding all families to scanner<a name="line.3139"></a> |
| <span class="sourceLineNo">3140</span> for (byte[] family : this.htableDescriptor.getColumnFamilyNames()) {<a name="line.3140"></a> |
| <span class="sourceLineNo">3141</span> scan.addFamily(family);<a name="line.3141"></a> |
| <span class="sourceLineNo">3142</span> }<a name="line.3142"></a> |
| <span class="sourceLineNo">3143</span> } else {<a name="line.3143"></a> |
| <span class="sourceLineNo">3144</span> for (byte[] family : scan.getFamilyMap().keySet()) {<a name="line.3144"></a> |
| <span class="sourceLineNo">3145</span> checkFamily(family);<a name="line.3145"></a> |
| <span class="sourceLineNo">3146</span> }<a name="line.3146"></a> |
| <span class="sourceLineNo">3147</span> }<a name="line.3147"></a> |
| <span class="sourceLineNo">3148</span> return instantiateRegionScanner(scan, additionalScanners, nonceGroup, nonce);<a name="line.3148"></a> |
| <span class="sourceLineNo">3149</span> } finally {<a name="line.3149"></a> |
| <span class="sourceLineNo">3150</span> closeRegionOperation(Operation.SCAN);<a name="line.3150"></a> |
| <span class="sourceLineNo">3151</span> }<a name="line.3151"></a> |
| <span class="sourceLineNo">3152</span> }<a name="line.3152"></a> |
| <span class="sourceLineNo">3153</span><a name="line.3153"></a> |
| <span class="sourceLineNo">3154</span> protected RegionScannerImpl instantiateRegionScanner(Scan scan,<a name="line.3154"></a> |
| <span class="sourceLineNo">3155</span> List<KeyValueScanner> additionalScanners, long nonceGroup, long nonce) throws IOException {<a name="line.3155"></a> |
| <span class="sourceLineNo">3156</span> if (scan.isReversed()) {<a name="line.3156"></a> |
| <span class="sourceLineNo">3157</span> if (scan.getFilter() != null) {<a name="line.3157"></a> |
| <span class="sourceLineNo">3158</span> scan.getFilter().setReversed(true);<a name="line.3158"></a> |
| <span class="sourceLineNo">3159</span> }<a name="line.3159"></a> |
| <span class="sourceLineNo">3160</span> return new ReversedRegionScannerImpl(scan, additionalScanners, this, nonceGroup, nonce);<a name="line.3160"></a> |
| <span class="sourceLineNo">3161</span> }<a name="line.3161"></a> |
| <span class="sourceLineNo">3162</span> return new RegionScannerImpl(scan, additionalScanners, this, nonceGroup, nonce);<a name="line.3162"></a> |
| <span class="sourceLineNo">3163</span> }<a name="line.3163"></a> |
| <span class="sourceLineNo">3164</span><a name="line.3164"></a> |
| <span class="sourceLineNo">3165</span> /**<a name="line.3165"></a> |
| <span class="sourceLineNo">3166</span> * Prepare a delete for a row mutation processor<a name="line.3166"></a> |
| <span class="sourceLineNo">3167</span> * @param delete The passed delete is modified by this method. WARNING!<a name="line.3167"></a> |
| <span class="sourceLineNo">3168</span> */<a name="line.3168"></a> |
| <span class="sourceLineNo">3169</span> private void prepareDelete(Delete delete) throws IOException {<a name="line.3169"></a> |
| <span class="sourceLineNo">3170</span> // Check to see if this is a deleteRow insert<a name="line.3170"></a> |
| <span class="sourceLineNo">3171</span> if(delete.getFamilyCellMap().isEmpty()){<a name="line.3171"></a> |
| <span class="sourceLineNo">3172</span> for(byte [] family : this.htableDescriptor.getColumnFamilyNames()){<a name="line.3172"></a> |
| <span class="sourceLineNo">3173</span> // Don't eat the timestamp<a name="line.3173"></a> |
| <span class="sourceLineNo">3174</span> delete.addFamily(family, delete.getTimestamp());<a name="line.3174"></a> |
| <span class="sourceLineNo">3175</span> }<a name="line.3175"></a> |
| <span class="sourceLineNo">3176</span> } else {<a name="line.3176"></a> |
| <span class="sourceLineNo">3177</span> for(byte [] family : delete.getFamilyCellMap().keySet()) {<a name="line.3177"></a> |
| <span class="sourceLineNo">3178</span> if(family == null) {<a name="line.3178"></a> |
| <span class="sourceLineNo">3179</span> throw new NoSuchColumnFamilyException("Empty family is invalid");<a name="line.3179"></a> |
| <span class="sourceLineNo">3180</span> }<a name="line.3180"></a> |
| <span class="sourceLineNo">3181</span> checkFamily(family, delete.getDurability());<a name="line.3181"></a> |
| <span class="sourceLineNo">3182</span> }<a name="line.3182"></a> |
| <span class="sourceLineNo">3183</span> }<a name="line.3183"></a> |
| <span class="sourceLineNo">3184</span> }<a name="line.3184"></a> |
| <span class="sourceLineNo">3185</span><a name="line.3185"></a> |
| <span class="sourceLineNo">3186</span> @Override<a name="line.3186"></a> |
| <span class="sourceLineNo">3187</span> public void delete(Delete delete) throws IOException {<a name="line.3187"></a> |
| <span class="sourceLineNo">3188</span> checkReadOnly();<a name="line.3188"></a> |
| <span class="sourceLineNo">3189</span> checkResources();<a name="line.3189"></a> |
| <span class="sourceLineNo">3190</span> startRegionOperation(Operation.DELETE);<a name="line.3190"></a> |
| <span class="sourceLineNo">3191</span> try {<a name="line.3191"></a> |
| <span class="sourceLineNo">3192</span> // All edits for the given row (across all column families) must happen atomically.<a name="line.3192"></a> |
| <span class="sourceLineNo">3193</span> mutate(delete);<a name="line.3193"></a> |
| <span class="sourceLineNo">3194</span> } finally {<a name="line.3194"></a> |
| <span class="sourceLineNo">3195</span> closeRegionOperation(Operation.DELETE);<a name="line.3195"></a> |
| <span class="sourceLineNo">3196</span> }<a name="line.3196"></a> |
| <span class="sourceLineNo">3197</span> }<a name="line.3197"></a> |
| <span class="sourceLineNo">3198</span><a name="line.3198"></a> |
| <span class="sourceLineNo">3199</span> /**<a name="line.3199"></a> |
| <span class="sourceLineNo">3200</span> * Set up correct timestamps in the KVs in Delete object.<a name="line.3200"></a> |
| <span class="sourceLineNo">3201</span> * <p/><a name="line.3201"></a> |
| <span class="sourceLineNo">3202</span> * Caller should have the row and region locks.<a name="line.3202"></a> |
| <span class="sourceLineNo">3203</span> */<a name="line.3203"></a> |
| <span class="sourceLineNo">3204</span> private void prepareDeleteTimestamps(Mutation mutation, Map<byte[], List<Cell>> familyMap,<a name="line.3204"></a> |
| <span class="sourceLineNo">3205</span> byte[] byteNow) throws IOException {<a name="line.3205"></a> |
| <span class="sourceLineNo">3206</span> for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) {<a name="line.3206"></a> |
| <span class="sourceLineNo">3207</span><a name="line.3207"></a> |
| <span class="sourceLineNo">3208</span> byte[] family = e.getKey();<a name="line.3208"></a> |
| <span class="sourceLineNo">3209</span> List<Cell> cells = e.getValue();<a name="line.3209"></a> |
| <span class="sourceLineNo">3210</span> assert cells instanceof RandomAccess;<a name="line.3210"></a> |
| <span class="sourceLineNo">3211</span><a name="line.3211"></a> |
| <span class="sourceLineNo">3212</span> Map<byte[], Integer> kvCount = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.3212"></a> |
| <span class="sourceLineNo">3213</span> int listSize = cells.size();<a name="line.3213"></a> |
| <span class="sourceLineNo">3214</span> for (int i=0; i < listSize; i++) {<a name="line.3214"></a> |
| <span class="sourceLineNo">3215</span> Cell cell = cells.get(i);<a name="line.3215"></a> |
| <span class="sourceLineNo">3216</span> // Check if time is LATEST, change to time of most recent addition if so<a name="line.3216"></a> |
| <span class="sourceLineNo">3217</span> // This is expensive.<a name="line.3217"></a> |
| <span class="sourceLineNo">3218</span> if (cell.getTimestamp() == HConstants.LATEST_TIMESTAMP<a name="line.3218"></a> |
| <span class="sourceLineNo">3219</span> && PrivateCellUtil.isDeleteType(cell)) {<a name="line.3219"></a> |
| <span class="sourceLineNo">3220</span> byte[] qual = CellUtil.cloneQualifier(cell);<a name="line.3220"></a> |
| <span class="sourceLineNo">3221</span><a name="line.3221"></a> |
| <span class="sourceLineNo">3222</span> Integer count = kvCount.get(qual);<a name="line.3222"></a> |
| <span class="sourceLineNo">3223</span> if (count == null) {<a name="line.3223"></a> |
| <span class="sourceLineNo">3224</span> kvCount.put(qual, 1);<a name="line.3224"></a> |
| <span class="sourceLineNo">3225</span> } else {<a name="line.3225"></a> |
| <span class="sourceLineNo">3226</span> kvCount.put(qual, count + 1);<a name="line.3226"></a> |
| <span class="sourceLineNo">3227</span> }<a name="line.3227"></a> |
| <span class="sourceLineNo">3228</span> count = kvCount.get(qual);<a name="line.3228"></a> |
| <span class="sourceLineNo">3229</span><a name="line.3229"></a> |
| <span class="sourceLineNo">3230</span> Get get = new Get(CellUtil.cloneRow(cell));<a name="line.3230"></a> |
| <span class="sourceLineNo">3231</span> get.readVersions(count);<a name="line.3231"></a> |
| <span class="sourceLineNo">3232</span> get.addColumn(family, qual);<a name="line.3232"></a> |
| <span class="sourceLineNo">3233</span> if (coprocessorHost != null) {<a name="line.3233"></a> |
| <span class="sourceLineNo">3234</span> if (!coprocessorHost.prePrepareTimeStampForDeleteVersion(mutation, cell,<a name="line.3234"></a> |
| <span class="sourceLineNo">3235</span> byteNow, get)) {<a name="line.3235"></a> |
| <span class="sourceLineNo">3236</span> updateDeleteLatestVersionTimestamp(cell, get, count, byteNow);<a name="line.3236"></a> |
| <span class="sourceLineNo">3237</span> }<a name="line.3237"></a> |
| <span class="sourceLineNo">3238</span> } else {<a name="line.3238"></a> |
| <span class="sourceLineNo">3239</span> updateDeleteLatestVersionTimestamp(cell, get, count, byteNow);<a name="line.3239"></a> |
| <span class="sourceLineNo">3240</span> }<a name="line.3240"></a> |
| <span class="sourceLineNo">3241</span> } else {<a name="line.3241"></a> |
| <span class="sourceLineNo">3242</span> PrivateCellUtil.updateLatestStamp(cell, byteNow);<a name="line.3242"></a> |
| <span class="sourceLineNo">3243</span> }<a name="line.3243"></a> |
| <span class="sourceLineNo">3244</span> }<a name="line.3244"></a> |
| <span class="sourceLineNo">3245</span> }<a name="line.3245"></a> |
| <span class="sourceLineNo">3246</span> }<a name="line.3246"></a> |
| <span class="sourceLineNo">3247</span><a name="line.3247"></a> |
| <span class="sourceLineNo">3248</span> private void updateDeleteLatestVersionTimestamp(Cell cell, Get get, int count, byte[] byteNow)<a name="line.3248"></a> |
| <span class="sourceLineNo">3249</span> throws IOException {<a name="line.3249"></a> |
| <span class="sourceLineNo">3250</span> List<Cell> result = get(get, false);<a name="line.3250"></a> |
| <span class="sourceLineNo">3251</span><a name="line.3251"></a> |
| <span class="sourceLineNo">3252</span> if (result.size() < count) {<a name="line.3252"></a> |
| <span class="sourceLineNo">3253</span> // Nothing to delete<a name="line.3253"></a> |
| <span class="sourceLineNo">3254</span> PrivateCellUtil.updateLatestStamp(cell, byteNow);<a name="line.3254"></a> |
| <span class="sourceLineNo">3255</span> return;<a name="line.3255"></a> |
| <span class="sourceLineNo">3256</span> }<a name="line.3256"></a> |
| <span class="sourceLineNo">3257</span> if (result.size() > count) {<a name="line.3257"></a> |
| <span class="sourceLineNo">3258</span> throw new RuntimeException("Unexpected size: " + result.size());<a name="line.3258"></a> |
| <span class="sourceLineNo">3259</span> }<a name="line.3259"></a> |
| <span class="sourceLineNo">3260</span> Cell getCell = result.get(count - 1);<a name="line.3260"></a> |
| <span class="sourceLineNo">3261</span> PrivateCellUtil.setTimestamp(cell, getCell.getTimestamp());<a name="line.3261"></a> |
| <span class="sourceLineNo">3262</span> }<a name="line.3262"></a> |
| <span class="sourceLineNo">3263</span><a name="line.3263"></a> |
| <span class="sourceLineNo">3264</span> @Override<a name="line.3264"></a> |
| <span class="sourceLineNo">3265</span> public void put(Put put) throws IOException {<a name="line.3265"></a> |
| <span class="sourceLineNo">3266</span> checkReadOnly();<a name="line.3266"></a> |
| <span class="sourceLineNo">3267</span><a name="line.3267"></a> |
| <span class="sourceLineNo">3268</span> // Do a rough check that we have resources to accept a write. The check is<a name="line.3268"></a> |
| <span class="sourceLineNo">3269</span> // 'rough' in that between the resource check and the call to obtain a<a name="line.3269"></a> |
| <span class="sourceLineNo">3270</span> // read lock, resources may run out. For now, the thought is that this<a name="line.3270"></a> |
| <span class="sourceLineNo">3271</span> // will be extremely rare; we'll deal with it when it happens.<a name="line.3271"></a> |
| <span class="sourceLineNo">3272</span> checkResources();<a name="line.3272"></a> |
| <span class="sourceLineNo">3273</span> startRegionOperation(Operation.PUT);<a name="line.3273"></a> |
| <span class="sourceLineNo">3274</span> try {<a name="line.3274"></a> |
| <span class="sourceLineNo">3275</span> // All edits for the given row (across all column families) must happen atomically.<a name="line.3275"></a> |
| <span class="sourceLineNo">3276</span> mutate(put);<a name="line.3276"></a> |
| <span class="sourceLineNo">3277</span> } finally {<a name="line.3277"></a> |
| <span class="sourceLineNo">3278</span> closeRegionOperation(Operation.PUT);<a name="line.3278"></a> |
| <span class="sourceLineNo">3279</span> }<a name="line.3279"></a> |
| <span class="sourceLineNo">3280</span> }<a name="line.3280"></a> |
| <span class="sourceLineNo">3281</span><a name="line.3281"></a> |
| <span class="sourceLineNo">3282</span> /**<a name="line.3282"></a> |
| <span class="sourceLineNo">3283</span> * Class that tracks the progress of a batch operations, accumulating status codes and tracking<a name="line.3283"></a> |
| <span class="sourceLineNo">3284</span> * the index at which processing is proceeding. These batch operations may get split into<a name="line.3284"></a> |
| <span class="sourceLineNo">3285</span> * mini-batches for processing.<a name="line.3285"></a> |
| <span class="sourceLineNo">3286</span> */<a name="line.3286"></a> |
| <span class="sourceLineNo">3287</span> private abstract static class BatchOperation<T> {<a name="line.3287"></a> |
| <span class="sourceLineNo">3288</span> protected final T[] operations;<a name="line.3288"></a> |
| <span class="sourceLineNo">3289</span> protected final OperationStatus[] retCodeDetails;<a name="line.3289"></a> |
| <span class="sourceLineNo">3290</span> protected final WALEdit[] walEditsFromCoprocessors;<a name="line.3290"></a> |
| <span class="sourceLineNo">3291</span> // reference family cell maps directly so coprocessors can mutate them if desired<a name="line.3291"></a> |
| <span class="sourceLineNo">3292</span> protected final Map<byte[], List<Cell>>[] familyCellMaps;<a name="line.3292"></a> |
| <span class="sourceLineNo">3293</span> // For Increment/Append operations<a name="line.3293"></a> |
| <span class="sourceLineNo">3294</span> protected final Result[] results;<a name="line.3294"></a> |
| <span class="sourceLineNo">3295</span> // For nonce operations<a name="line.3295"></a> |
| <span class="sourceLineNo">3296</span> protected final boolean[] canProceed;<a name="line.3296"></a> |
| <span class="sourceLineNo">3297</span><a name="line.3297"></a> |
| <span class="sourceLineNo">3298</span> protected final HRegion region;<a name="line.3298"></a> |
| <span class="sourceLineNo">3299</span> protected int nextIndexToProcess = 0;<a name="line.3299"></a> |
| <span class="sourceLineNo">3300</span> protected final ObservedExceptionsInBatch observedExceptions;<a name="line.3300"></a> |
| <span class="sourceLineNo">3301</span> //Durability of the batch (highest durability of all operations)<a name="line.3301"></a> |
| <span class="sourceLineNo">3302</span> protected Durability durability;<a name="line.3302"></a> |
| <span class="sourceLineNo">3303</span> protected boolean atomic = false;<a name="line.3303"></a> |
| <span class="sourceLineNo">3304</span><a name="line.3304"></a> |
| <span class="sourceLineNo">3305</span> public BatchOperation(final HRegion region, T[] operations) {<a name="line.3305"></a> |
| <span class="sourceLineNo">3306</span> this.operations = operations;<a name="line.3306"></a> |
| <span class="sourceLineNo">3307</span> this.retCodeDetails = new OperationStatus[operations.length];<a name="line.3307"></a> |
| <span class="sourceLineNo">3308</span> Arrays.fill(this.retCodeDetails, OperationStatus.NOT_RUN);<a name="line.3308"></a> |
| <span class="sourceLineNo">3309</span> this.walEditsFromCoprocessors = new WALEdit[operations.length];<a name="line.3309"></a> |
| <span class="sourceLineNo">3310</span> familyCellMaps = new Map[operations.length];<a name="line.3310"></a> |
| <span class="sourceLineNo">3311</span> this.results = new Result[operations.length];<a name="line.3311"></a> |
| <span class="sourceLineNo">3312</span> this.canProceed = new boolean[operations.length];<a name="line.3312"></a> |
| <span class="sourceLineNo">3313</span><a name="line.3313"></a> |
| <span class="sourceLineNo">3314</span> this.region = region;<a name="line.3314"></a> |
| <span class="sourceLineNo">3315</span> observedExceptions = new ObservedExceptionsInBatch();<a name="line.3315"></a> |
| <span class="sourceLineNo">3316</span> durability = Durability.USE_DEFAULT;<a name="line.3316"></a> |
| <span class="sourceLineNo">3317</span> }<a name="line.3317"></a> |
| <span class="sourceLineNo">3318</span><a name="line.3318"></a> |
| <span class="sourceLineNo">3319</span> /**<a name="line.3319"></a> |
| <span class="sourceLineNo">3320</span> * Visitor interface for batch operations<a name="line.3320"></a> |
| <span class="sourceLineNo">3321</span> */<a name="line.3321"></a> |
| <span class="sourceLineNo">3322</span> @FunctionalInterface<a name="line.3322"></a> |
| <span class="sourceLineNo">3323</span> interface Visitor {<a name="line.3323"></a> |
| <span class="sourceLineNo">3324</span> /**<a name="line.3324"></a> |
| <span class="sourceLineNo">3325</span> * @param index operation index<a name="line.3325"></a> |
| <span class="sourceLineNo">3326</span> * @return If true continue visiting remaining entries, break otherwise<a name="line.3326"></a> |
| <span class="sourceLineNo">3327</span> */<a name="line.3327"></a> |
| <span class="sourceLineNo">3328</span> boolean visit(int index) throws IOException;<a name="line.3328"></a> |
| <span class="sourceLineNo">3329</span> }<a name="line.3329"></a> |
| <span class="sourceLineNo">3330</span><a name="line.3330"></a> |
| <span class="sourceLineNo">3331</span> /**<a name="line.3331"></a> |
| <span class="sourceLineNo">3332</span> * Helper method for visiting pending/ all batch operations<a name="line.3332"></a> |
| <span class="sourceLineNo">3333</span> */<a name="line.3333"></a> |
| <span class="sourceLineNo">3334</span> public void visitBatchOperations(boolean pendingOnly, int lastIndexExclusive, Visitor visitor)<a name="line.3334"></a> |
| <span class="sourceLineNo">3335</span> throws IOException {<a name="line.3335"></a> |
| <span class="sourceLineNo">3336</span> assert lastIndexExclusive <= this.size();<a name="line.3336"></a> |
| <span class="sourceLineNo">3337</span> for (int i = nextIndexToProcess; i < lastIndexExclusive; i++) {<a name="line.3337"></a> |
| <span class="sourceLineNo">3338</span> if (!pendingOnly || isOperationPending(i)) {<a name="line.3338"></a> |
| <span class="sourceLineNo">3339</span> if (!visitor.visit(i)) {<a name="line.3339"></a> |
| <span class="sourceLineNo">3340</span> break;<a name="line.3340"></a> |
| <span class="sourceLineNo">3341</span> }<a name="line.3341"></a> |
| <span class="sourceLineNo">3342</span> }<a name="line.3342"></a> |
| <span class="sourceLineNo">3343</span> }<a name="line.3343"></a> |
| <span class="sourceLineNo">3344</span> }<a name="line.3344"></a> |
| <span class="sourceLineNo">3345</span><a name="line.3345"></a> |
| <span class="sourceLineNo">3346</span> public abstract Mutation getMutation(int index);<a name="line.3346"></a> |
| <span class="sourceLineNo">3347</span><a name="line.3347"></a> |
| <span class="sourceLineNo">3348</span> public abstract long getNonceGroup(int index);<a name="line.3348"></a> |
| <span class="sourceLineNo">3349</span><a name="line.3349"></a> |
| <span class="sourceLineNo">3350</span> public abstract long getNonce(int index);<a name="line.3350"></a> |
| <span class="sourceLineNo">3351</span><a name="line.3351"></a> |
| <span class="sourceLineNo">3352</span> /**<a name="line.3352"></a> |
| <span class="sourceLineNo">3353</span> * This method is potentially expensive and useful mostly for non-replay CP path.<a name="line.3353"></a> |
| <span class="sourceLineNo">3354</span> */<a name="line.3354"></a> |
| <span class="sourceLineNo">3355</span> public abstract Mutation[] getMutationsForCoprocs();<a name="line.3355"></a> |
| <span class="sourceLineNo">3356</span><a name="line.3356"></a> |
| <span class="sourceLineNo">3357</span> public abstract boolean isInReplay();<a name="line.3357"></a> |
| <span class="sourceLineNo">3358</span><a name="line.3358"></a> |
| <span class="sourceLineNo">3359</span> public abstract long getOrigLogSeqNum();<a name="line.3359"></a> |
| <span class="sourceLineNo">3360</span><a name="line.3360"></a> |
| <span class="sourceLineNo">3361</span> public abstract void startRegionOperation() throws IOException;<a name="line.3361"></a> |
| <span class="sourceLineNo">3362</span><a name="line.3362"></a> |
| <span class="sourceLineNo">3363</span> public abstract void closeRegionOperation() throws IOException;<a name="line.3363"></a> |
| <span class="sourceLineNo">3364</span><a name="line.3364"></a> |
| <span class="sourceLineNo">3365</span> /**<a name="line.3365"></a> |
| <span class="sourceLineNo">3366</span> * Validates each mutation and prepares a batch for write. If necessary (non-replay case), runs<a name="line.3366"></a> |
| <span class="sourceLineNo">3367</span> * CP prePut()/preDelete()/preIncrement()/preAppend() hooks for all mutations in a batch. This<a name="line.3367"></a> |
| <span class="sourceLineNo">3368</span> * is intended to operate on entire batch and will be called from outside of class to check<a name="line.3368"></a> |
| <span class="sourceLineNo">3369</span> * and prepare batch. This can be implemented by calling helper method<a name="line.3369"></a> |
| <span class="sourceLineNo">3370</span> * {@link #checkAndPrepareMutation(int, long)} in a 'for' loop over mutations.<a name="line.3370"></a> |
| <span class="sourceLineNo">3371</span> */<a name="line.3371"></a> |
| <span class="sourceLineNo">3372</span> public abstract void checkAndPrepare() throws IOException;<a name="line.3372"></a> |
| <span class="sourceLineNo">3373</span><a name="line.3373"></a> |
| <span class="sourceLineNo">3374</span> /**<a name="line.3374"></a> |
| <span class="sourceLineNo">3375</span> * Implement any Put request specific check and prepare logic here. Please refer to<a name="line.3375"></a> |
| <span class="sourceLineNo">3376</span> * {@link #checkAndPrepareMutation(Mutation, long)} for how its used.<a name="line.3376"></a> |
| <span class="sourceLineNo">3377</span> */<a name="line.3377"></a> |
| <span class="sourceLineNo">3378</span> protected abstract void checkAndPreparePut(final Put p) throws IOException;<a name="line.3378"></a> |
| <span class="sourceLineNo">3379</span><a name="line.3379"></a> |
| <span class="sourceLineNo">3380</span> /**<a name="line.3380"></a> |
| <span class="sourceLineNo">3381</span> * If necessary, calls preBatchMutate() CP hook for a mini-batch and updates metrics, cell<a name="line.3381"></a> |
| <span class="sourceLineNo">3382</span> * count, tags and timestamp for all cells of all operations in a mini-batch.<a name="line.3382"></a> |
| <span class="sourceLineNo">3383</span> */<a name="line.3383"></a> |
| <span class="sourceLineNo">3384</span> public abstract void prepareMiniBatchOperations(MiniBatchOperationInProgress<Mutation><a name="line.3384"></a> |
| <span class="sourceLineNo">3385</span> miniBatchOp, long timestamp, final List<RowLock> acquiredRowLocks) throws IOException;<a name="line.3385"></a> |
| <span class="sourceLineNo">3386</span><a name="line.3386"></a> |
| <span class="sourceLineNo">3387</span> /**<a name="line.3387"></a> |
| <span class="sourceLineNo">3388</span> * Write mini-batch operations to MemStore<a name="line.3388"></a> |
| <span class="sourceLineNo">3389</span> */<a name="line.3389"></a> |
| <span class="sourceLineNo">3390</span> public abstract WriteEntry writeMiniBatchOperationsToMemStore(<a name="line.3390"></a> |
| <span class="sourceLineNo">3391</span> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final WriteEntry writeEntry)<a name="line.3391"></a> |
| <span class="sourceLineNo">3392</span> throws IOException;<a name="line.3392"></a> |
| <span class="sourceLineNo">3393</span><a name="line.3393"></a> |
| <span class="sourceLineNo">3394</span> protected void writeMiniBatchOperationsToMemStore(<a name="line.3394"></a> |
| <span class="sourceLineNo">3395</span> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final long writeNumber)<a name="line.3395"></a> |
| <span class="sourceLineNo">3396</span> throws IOException {<a name="line.3396"></a> |
| <span class="sourceLineNo">3397</span> MemStoreSizing memStoreAccounting = new NonThreadSafeMemStoreSizing();<a name="line.3397"></a> |
| <span class="sourceLineNo">3398</span> visitBatchOperations(true, miniBatchOp.getLastIndexExclusive(), (int index) -> {<a name="line.3398"></a> |
| <span class="sourceLineNo">3399</span> // We need to update the sequence id for following reasons.<a name="line.3399"></a> |
| <span class="sourceLineNo">3400</span> // 1) If the op is in replay mode, FSWALEntry#stampRegionSequenceId won't stamp sequence id.<a name="line.3400"></a> |
| <span class="sourceLineNo">3401</span> // 2) If no WAL, FSWALEntry won't be used<a name="line.3401"></a> |
| <span class="sourceLineNo">3402</span> // we use durability of the original mutation for the mutation passed by CP.<a name="line.3402"></a> |
| <span class="sourceLineNo">3403</span> if (isInReplay() || getMutation(index).getDurability() == Durability.SKIP_WAL) {<a name="line.3403"></a> |
| <span class="sourceLineNo">3404</span> region.updateSequenceId(familyCellMaps[index].values(), writeNumber);<a name="line.3404"></a> |
| <span class="sourceLineNo">3405</span> }<a name="line.3405"></a> |
| <span class="sourceLineNo">3406</span> applyFamilyMapToMemStore(familyCellMaps[index], memStoreAccounting);<a name="line.3406"></a> |
| <span class="sourceLineNo">3407</span> return true;<a name="line.3407"></a> |
| <span class="sourceLineNo">3408</span> });<a name="line.3408"></a> |
| <span class="sourceLineNo">3409</span> // update memStore size<a name="line.3409"></a> |
| <span class="sourceLineNo">3410</span> region.incMemStoreSize(memStoreAccounting.getDataSize(), memStoreAccounting.getHeapSize(),<a name="line.3410"></a> |
| <span class="sourceLineNo">3411</span> memStoreAccounting.getOffHeapSize(), memStoreAccounting.getCellsCount());<a name="line.3411"></a> |
| <span class="sourceLineNo">3412</span> }<a name="line.3412"></a> |
| <span class="sourceLineNo">3413</span><a name="line.3413"></a> |
| <span class="sourceLineNo">3414</span> public boolean isDone() {<a name="line.3414"></a> |
| <span class="sourceLineNo">3415</span> return nextIndexToProcess == operations.length;<a name="line.3415"></a> |
| <span class="sourceLineNo">3416</span> }<a name="line.3416"></a> |
| <span class="sourceLineNo">3417</span><a name="line.3417"></a> |
| <span class="sourceLineNo">3418</span> public int size() {<a name="line.3418"></a> |
| <span class="sourceLineNo">3419</span> return operations.length;<a name="line.3419"></a> |
| <span class="sourceLineNo">3420</span> }<a name="line.3420"></a> |
| <span class="sourceLineNo">3421</span><a name="line.3421"></a> |
| <span class="sourceLineNo">3422</span> public boolean isOperationPending(int index) {<a name="line.3422"></a> |
| <span class="sourceLineNo">3423</span> return retCodeDetails[index].getOperationStatusCode() == OperationStatusCode.NOT_RUN;<a name="line.3423"></a> |
| <span class="sourceLineNo">3424</span> }<a name="line.3424"></a> |
| <span class="sourceLineNo">3425</span><a name="line.3425"></a> |
| <span class="sourceLineNo">3426</span> public List<UUID> getClusterIds() {<a name="line.3426"></a> |
| <span class="sourceLineNo">3427</span> assert size() != 0;<a name="line.3427"></a> |
| <span class="sourceLineNo">3428</span> return getMutation(0).getClusterIds();<a name="line.3428"></a> |
| <span class="sourceLineNo">3429</span> }<a name="line.3429"></a> |
| <span class="sourceLineNo">3430</span><a name="line.3430"></a> |
| <span class="sourceLineNo">3431</span> boolean isAtomic() {<a name="line.3431"></a> |
| <span class="sourceLineNo">3432</span> return atomic;<a name="line.3432"></a> |
| <span class="sourceLineNo">3433</span> }<a name="line.3433"></a> |
| <span class="sourceLineNo">3434</span><a name="line.3434"></a> |
| <span class="sourceLineNo">3435</span> /**<a name="line.3435"></a> |
| <span class="sourceLineNo">3436</span> * Helper method that checks and prepares only one mutation. This can be used to implement<a name="line.3436"></a> |
| <span class="sourceLineNo">3437</span> * {@link #checkAndPrepare()} for entire Batch.<a name="line.3437"></a> |
| <span class="sourceLineNo">3438</span> * NOTE: As CP prePut()/preDelete()/preIncrement()/preAppend() hooks may modify mutations,<a name="line.3438"></a> |
| <span class="sourceLineNo">3439</span> * this method should be called after prePut()/preDelete()/preIncrement()/preAppend() CP hooks<a name="line.3439"></a> |
| <span class="sourceLineNo">3440</span> * are run for the mutation<a name="line.3440"></a> |
| <span class="sourceLineNo">3441</span> */<a name="line.3441"></a> |
| <span class="sourceLineNo">3442</span> protected void checkAndPrepareMutation(Mutation mutation, final long timestamp)<a name="line.3442"></a> |
| <span class="sourceLineNo">3443</span> throws IOException {<a name="line.3443"></a> |
| <span class="sourceLineNo">3444</span> region.checkRow(mutation.getRow(), "batchMutate");<a name="line.3444"></a> |
| <span class="sourceLineNo">3445</span> if (mutation instanceof Put) {<a name="line.3445"></a> |
| <span class="sourceLineNo">3446</span> // Check the families in the put. If bad, skip this one.<a name="line.3446"></a> |
| <span class="sourceLineNo">3447</span> checkAndPreparePut((Put) mutation);<a name="line.3447"></a> |
| <span class="sourceLineNo">3448</span> region.checkTimestamps(mutation.getFamilyCellMap(), timestamp);<a name="line.3448"></a> |
| <span class="sourceLineNo">3449</span> } else if (mutation instanceof Delete) {<a name="line.3449"></a> |
| <span class="sourceLineNo">3450</span> region.prepareDelete((Delete) mutation);<a name="line.3450"></a> |
| <span class="sourceLineNo">3451</span> } else if (mutation instanceof Increment || mutation instanceof Append) {<a name="line.3451"></a> |
| <span class="sourceLineNo">3452</span> region.checkFamilies(mutation.getFamilyCellMap().keySet(), mutation.getDurability());<a name="line.3452"></a> |
| <span class="sourceLineNo">3453</span> }<a name="line.3453"></a> |
| <span class="sourceLineNo">3454</span> }<a name="line.3454"></a> |
| <span class="sourceLineNo">3455</span><a name="line.3455"></a> |
| <span class="sourceLineNo">3456</span> protected void checkAndPrepareMutation(int index, long timestamp) throws IOException {<a name="line.3456"></a> |
| <span class="sourceLineNo">3457</span> Mutation mutation = getMutation(index);<a name="line.3457"></a> |
| <span class="sourceLineNo">3458</span> try {<a name="line.3458"></a> |
| <span class="sourceLineNo">3459</span> this.checkAndPrepareMutation(mutation, timestamp);<a name="line.3459"></a> |
| <span class="sourceLineNo">3460</span><a name="line.3460"></a> |
| <span class="sourceLineNo">3461</span> if (mutation instanceof Put || mutation instanceof Delete) {<a name="line.3461"></a> |
| <span class="sourceLineNo">3462</span> // store the family map reference to allow for mutations<a name="line.3462"></a> |
| <span class="sourceLineNo">3463</span> familyCellMaps[index] = mutation.getFamilyCellMap();<a name="line.3463"></a> |
| <span class="sourceLineNo">3464</span> }<a name="line.3464"></a> |
| <span class="sourceLineNo">3465</span><a name="line.3465"></a> |
| <span class="sourceLineNo">3466</span> // store durability for the batch (highest durability of all operations in the batch)<a name="line.3466"></a> |
| <span class="sourceLineNo">3467</span> Durability tmpDur = region.getEffectiveDurability(mutation.getDurability());<a name="line.3467"></a> |
| <span class="sourceLineNo">3468</span> if (tmpDur.ordinal() > durability.ordinal()) {<a name="line.3468"></a> |
| <span class="sourceLineNo">3469</span> durability = tmpDur;<a name="line.3469"></a> |
| <span class="sourceLineNo">3470</span> }<a name="line.3470"></a> |
| <span class="sourceLineNo">3471</span> } catch (NoSuchColumnFamilyException nscfe) {<a name="line.3471"></a> |
| <span class="sourceLineNo">3472</span> final String msg = "No such column family in batch mutation in region " + this;<a name="line.3472"></a> |
| <span class="sourceLineNo">3473</span> if (observedExceptions.hasSeenNoSuchFamily()) {<a name="line.3473"></a> |
| <span class="sourceLineNo">3474</span> LOG.warn(msg + nscfe.getMessage());<a name="line.3474"></a> |
| <span class="sourceLineNo">3475</span> } else {<a name="line.3475"></a> |
| <span class="sourceLineNo">3476</span> LOG.warn(msg, nscfe);<a name="line.3476"></a> |
| <span class="sourceLineNo">3477</span> observedExceptions.sawNoSuchFamily();<a name="line.3477"></a> |
| <span class="sourceLineNo">3478</span> }<a name="line.3478"></a> |
| <span class="sourceLineNo">3479</span> retCodeDetails[index] = new OperationStatus(<a name="line.3479"></a> |
| <span class="sourceLineNo">3480</span> OperationStatusCode.BAD_FAMILY, nscfe.getMessage());<a name="line.3480"></a> |
| <span class="sourceLineNo">3481</span> if (isAtomic()) { // fail, atomic means all or none<a name="line.3481"></a> |
| <span class="sourceLineNo">3482</span> throw nscfe;<a name="line.3482"></a> |
| <span class="sourceLineNo">3483</span> }<a name="line.3483"></a> |
| <span class="sourceLineNo">3484</span> } catch (FailedSanityCheckException fsce) {<a name="line.3484"></a> |
| <span class="sourceLineNo">3485</span> final String msg = "Batch Mutation did not pass sanity check in region " + this;<a name="line.3485"></a> |
| <span class="sourceLineNo">3486</span> if (observedExceptions.hasSeenFailedSanityCheck()) {<a name="line.3486"></a> |
| <span class="sourceLineNo">3487</span> LOG.warn(msg + fsce.getMessage());<a name="line.3487"></a> |
| <span class="sourceLineNo">3488</span> } else {<a name="line.3488"></a> |
| <span class="sourceLineNo">3489</span> LOG.warn(msg, fsce);<a name="line.3489"></a> |
| <span class="sourceLineNo">3490</span> observedExceptions.sawFailedSanityCheck();<a name="line.3490"></a> |
| <span class="sourceLineNo">3491</span> }<a name="line.3491"></a> |
| <span class="sourceLineNo">3492</span> retCodeDetails[index] = new OperationStatus(<a name="line.3492"></a> |
| <span class="sourceLineNo">3493</span> OperationStatusCode.SANITY_CHECK_FAILURE, fsce.getMessage());<a name="line.3493"></a> |
| <span class="sourceLineNo">3494</span> if (isAtomic()) {<a name="line.3494"></a> |
| <span class="sourceLineNo">3495</span> throw fsce;<a name="line.3495"></a> |
| <span class="sourceLineNo">3496</span> }<a name="line.3496"></a> |
| <span class="sourceLineNo">3497</span> } catch (WrongRegionException we) {<a name="line.3497"></a> |
| <span class="sourceLineNo">3498</span> final String msg = "Batch mutation had a row that does not belong to this region " + this;<a name="line.3498"></a> |
| <span class="sourceLineNo">3499</span> if (observedExceptions.hasSeenWrongRegion()) {<a name="line.3499"></a> |
| <span class="sourceLineNo">3500</span> LOG.warn(msg + we.getMessage());<a name="line.3500"></a> |
| <span class="sourceLineNo">3501</span> } else {<a name="line.3501"></a> |
| <span class="sourceLineNo">3502</span> LOG.warn(msg, we);<a name="line.3502"></a> |
| <span class="sourceLineNo">3503</span> observedExceptions.sawWrongRegion();<a name="line.3503"></a> |
| <span class="sourceLineNo">3504</span> }<a name="line.3504"></a> |
| <span class="sourceLineNo">3505</span> retCodeDetails[index] = new OperationStatus(<a name="line.3505"></a> |
| <span class="sourceLineNo">3506</span> OperationStatusCode.SANITY_CHECK_FAILURE, we.getMessage());<a name="line.3506"></a> |
| <span class="sourceLineNo">3507</span> if (isAtomic()) {<a name="line.3507"></a> |
| <span class="sourceLineNo">3508</span> throw we;<a name="line.3508"></a> |
| <span class="sourceLineNo">3509</span> }<a name="line.3509"></a> |
| <span class="sourceLineNo">3510</span> }<a name="line.3510"></a> |
| <span class="sourceLineNo">3511</span> }<a name="line.3511"></a> |
| <span class="sourceLineNo">3512</span><a name="line.3512"></a> |
| <span class="sourceLineNo">3513</span> /**<a name="line.3513"></a> |
| <span class="sourceLineNo">3514</span> * Creates Mini-batch of all operations [nextIndexToProcess, lastIndexExclusive) for which<a name="line.3514"></a> |
| <span class="sourceLineNo">3515</span> * a row lock can be acquired. All mutations with locked rows are considered to be<a name="line.3515"></a> |
| <span class="sourceLineNo">3516</span> * In-progress operations and hence the name {@link MiniBatchOperationInProgress}. Mini batch<a name="line.3516"></a> |
| <span class="sourceLineNo">3517</span> * is window over {@link BatchOperation} and contains contiguous pending operations.<a name="line.3517"></a> |
| <span class="sourceLineNo">3518</span> *<a name="line.3518"></a> |
| <span class="sourceLineNo">3519</span> * @param acquiredRowLocks keeps track of rowLocks acquired.<a name="line.3519"></a> |
| <span class="sourceLineNo">3520</span> */<a name="line.3520"></a> |
| <span class="sourceLineNo">3521</span> public MiniBatchOperationInProgress<Mutation> lockRowsAndBuildMiniBatch(<a name="line.3521"></a> |
| <span class="sourceLineNo">3522</span> List<RowLock> acquiredRowLocks) throws IOException {<a name="line.3522"></a> |
| <span class="sourceLineNo">3523</span> int readyToWriteCount = 0;<a name="line.3523"></a> |
| <span class="sourceLineNo">3524</span> int lastIndexExclusive = 0;<a name="line.3524"></a> |
| <span class="sourceLineNo">3525</span> RowLock prevRowLock = null;<a name="line.3525"></a> |
| <span class="sourceLineNo">3526</span> for (; lastIndexExclusive < size(); lastIndexExclusive++) {<a name="line.3526"></a> |
| <span class="sourceLineNo">3527</span> // It reaches the miniBatchSize, stop here and process the miniBatch<a name="line.3527"></a> |
| <span class="sourceLineNo">3528</span> // This only applies to non-atomic batch operations.<a name="line.3528"></a> |
| <span class="sourceLineNo">3529</span> if (!isAtomic() && (readyToWriteCount == region.miniBatchSize)) {<a name="line.3529"></a> |
| <span class="sourceLineNo">3530</span> break;<a name="line.3530"></a> |
| <span class="sourceLineNo">3531</span> }<a name="line.3531"></a> |
| <span class="sourceLineNo">3532</span><a name="line.3532"></a> |
| <span class="sourceLineNo">3533</span> if (!isOperationPending(lastIndexExclusive)) {<a name="line.3533"></a> |
| <span class="sourceLineNo">3534</span> continue;<a name="line.3534"></a> |
| <span class="sourceLineNo">3535</span> }<a name="line.3535"></a> |
| <span class="sourceLineNo">3536</span><a name="line.3536"></a> |
| <span class="sourceLineNo">3537</span> // HBASE-19389 Limit concurrency of put with dense (hundreds) columns to avoid exhausting<a name="line.3537"></a> |
| <span class="sourceLineNo">3538</span> // RS handlers, covering both MutationBatchOperation and ReplayBatchOperation<a name="line.3538"></a> |
| <span class="sourceLineNo">3539</span> // The BAD_FAMILY/SANITY_CHECK_FAILURE cases are handled in checkAndPrepare phase and won't<a name="line.3539"></a> |
| <span class="sourceLineNo">3540</span> // pass the isOperationPending check<a name="line.3540"></a> |
| <span class="sourceLineNo">3541</span> Map<byte[], List<Cell>> curFamilyCellMap =<a name="line.3541"></a> |
| <span class="sourceLineNo">3542</span> getMutation(lastIndexExclusive).getFamilyCellMap();<a name="line.3542"></a> |
| <span class="sourceLineNo">3543</span> try {<a name="line.3543"></a> |
| <span class="sourceLineNo">3544</span> // start the protector before acquiring row lock considering performance, and will finish<a name="line.3544"></a> |
| <span class="sourceLineNo">3545</span> // it when encountering exception<a name="line.3545"></a> |
| <span class="sourceLineNo">3546</span> region.storeHotnessProtector.start(curFamilyCellMap);<a name="line.3546"></a> |
| <span class="sourceLineNo">3547</span> } catch (RegionTooBusyException rtbe) {<a name="line.3547"></a> |
| <span class="sourceLineNo">3548</span> region.storeHotnessProtector.finish(curFamilyCellMap);<a name="line.3548"></a> |
| <span class="sourceLineNo">3549</span> if (isAtomic()) {<a name="line.3549"></a> |
| <span class="sourceLineNo">3550</span> throw rtbe;<a name="line.3550"></a> |
| <span class="sourceLineNo">3551</span> }<a name="line.3551"></a> |
| <span class="sourceLineNo">3552</span> retCodeDetails[lastIndexExclusive] =<a name="line.3552"></a> |
| <span class="sourceLineNo">3553</span> new OperationStatus(OperationStatusCode.STORE_TOO_BUSY, rtbe.getMessage());<a name="line.3553"></a> |
| <span class="sourceLineNo">3554</span> continue;<a name="line.3554"></a> |
| <span class="sourceLineNo">3555</span> }<a name="line.3555"></a> |
| <span class="sourceLineNo">3556</span><a name="line.3556"></a> |
| <span class="sourceLineNo">3557</span> Mutation mutation = getMutation(lastIndexExclusive);<a name="line.3557"></a> |
| <span class="sourceLineNo">3558</span> // If we haven't got any rows in our batch, we should block to get the next one.<a name="line.3558"></a> |
| <span class="sourceLineNo">3559</span> RowLock rowLock = null;<a name="line.3559"></a> |
| <span class="sourceLineNo">3560</span> boolean throwException = false;<a name="line.3560"></a> |
| <span class="sourceLineNo">3561</span> try {<a name="line.3561"></a> |
| <span class="sourceLineNo">3562</span> // if atomic then get exclusive lock, else shared lock<a name="line.3562"></a> |
| <span class="sourceLineNo">3563</span> rowLock = region.getRowLockInternal(mutation.getRow(), !isAtomic(), prevRowLock);<a name="line.3563"></a> |
| <span class="sourceLineNo">3564</span> } catch (TimeoutIOException | InterruptedIOException e) {<a name="line.3564"></a> |
| <span class="sourceLineNo">3565</span> // NOTE: We will retry when other exceptions, but we should stop if we receive<a name="line.3565"></a> |
| <span class="sourceLineNo">3566</span> // TimeoutIOException or InterruptedIOException as operation has timed out or<a name="line.3566"></a> |
| <span class="sourceLineNo">3567</span> // interrupted respectively.<a name="line.3567"></a> |
| <span class="sourceLineNo">3568</span> throwException = true;<a name="line.3568"></a> |
| <span class="sourceLineNo">3569</span> throw e;<a name="line.3569"></a> |
| <span class="sourceLineNo">3570</span> } catch (IOException ioe) {<a name="line.3570"></a> |
| <span class="sourceLineNo">3571</span> LOG.warn("Failed getting lock, row={}, in region {}",<a name="line.3571"></a> |
| <span class="sourceLineNo">3572</span> Bytes.toStringBinary(mutation.getRow()), this, ioe);<a name="line.3572"></a> |
| <span class="sourceLineNo">3573</span> if (isAtomic()) { // fail, atomic means all or none<a name="line.3573"></a> |
| <span class="sourceLineNo">3574</span> throwException = true;<a name="line.3574"></a> |
| <span class="sourceLineNo">3575</span> throw ioe;<a name="line.3575"></a> |
| <span class="sourceLineNo">3576</span> }<a name="line.3576"></a> |
| <span class="sourceLineNo">3577</span> } catch (Throwable throwable) {<a name="line.3577"></a> |
| <span class="sourceLineNo">3578</span> throwException = true;<a name="line.3578"></a> |
| <span class="sourceLineNo">3579</span> throw throwable;<a name="line.3579"></a> |
| <span class="sourceLineNo">3580</span> } finally {<a name="line.3580"></a> |
| <span class="sourceLineNo">3581</span> if (throwException) {<a name="line.3581"></a> |
| <span class="sourceLineNo">3582</span> region.storeHotnessProtector.finish(curFamilyCellMap);<a name="line.3582"></a> |
| <span class="sourceLineNo">3583</span> }<a name="line.3583"></a> |
| <span class="sourceLineNo">3584</span> }<a name="line.3584"></a> |
| <span class="sourceLineNo">3585</span> if (rowLock == null) {<a name="line.3585"></a> |
| <span class="sourceLineNo">3586</span> // We failed to grab another lock<a name="line.3586"></a> |
| <span class="sourceLineNo">3587</span> if (isAtomic()) {<a name="line.3587"></a> |
| <span class="sourceLineNo">3588</span> region.storeHotnessProtector.finish(curFamilyCellMap);<a name="line.3588"></a> |
| <span class="sourceLineNo">3589</span> throw new IOException("Can't apply all operations atomically!");<a name="line.3589"></a> |
| <span class="sourceLineNo">3590</span> }<a name="line.3590"></a> |
| <span class="sourceLineNo">3591</span> break; // Stop acquiring more rows for this batch<a name="line.3591"></a> |
| <span class="sourceLineNo">3592</span> } else {<a name="line.3592"></a> |
| <span class="sourceLineNo">3593</span> if (rowLock != prevRowLock) {<a name="line.3593"></a> |
| <span class="sourceLineNo">3594</span> // It is a different row now, add this to the acquiredRowLocks and<a name="line.3594"></a> |
| <span class="sourceLineNo">3595</span> // set prevRowLock to the new returned rowLock<a name="line.3595"></a> |
| <span class="sourceLineNo">3596</span> acquiredRowLocks.add(rowLock);<a name="line.3596"></a> |
| <span class="sourceLineNo">3597</span> prevRowLock = rowLock;<a name="line.3597"></a> |
| <span class="sourceLineNo">3598</span> }<a name="line.3598"></a> |
| <span class="sourceLineNo">3599</span> }<a name="line.3599"></a> |
| <span class="sourceLineNo">3600</span><a name="line.3600"></a> |
| <span class="sourceLineNo">3601</span> readyToWriteCount++;<a name="line.3601"></a> |
| <span class="sourceLineNo">3602</span> }<a name="line.3602"></a> |
| <span class="sourceLineNo">3603</span> return createMiniBatch(lastIndexExclusive, readyToWriteCount);<a name="line.3603"></a> |
| <span class="sourceLineNo">3604</span> }<a name="line.3604"></a> |
| <span class="sourceLineNo">3605</span><a name="line.3605"></a> |
| <span class="sourceLineNo">3606</span> protected MiniBatchOperationInProgress<Mutation> createMiniBatch(final int lastIndexExclusive,<a name="line.3606"></a> |
| <span class="sourceLineNo">3607</span> final int readyToWriteCount) {<a name="line.3607"></a> |
| <span class="sourceLineNo">3608</span> return new MiniBatchOperationInProgress<>(getMutationsForCoprocs(), retCodeDetails,<a name="line.3608"></a> |
| <span class="sourceLineNo">3609</span> walEditsFromCoprocessors, nextIndexToProcess, lastIndexExclusive, readyToWriteCount);<a name="line.3609"></a> |
| <span class="sourceLineNo">3610</span> }<a name="line.3610"></a> |
| <span class="sourceLineNo">3611</span><a name="line.3611"></a> |
| <span class="sourceLineNo">3612</span> /**<a name="line.3612"></a> |
| <span class="sourceLineNo">3613</span> * Builds separate WALEdit per nonce by applying input mutations. If WALEdits from CP are<a name="line.3613"></a> |
| <span class="sourceLineNo">3614</span> * present, they are merged to result WALEdit.<a name="line.3614"></a> |
| <span class="sourceLineNo">3615</span> */<a name="line.3615"></a> |
| <span class="sourceLineNo">3616</span> public List<Pair<NonceKey, WALEdit>> buildWALEdits(<a name="line.3616"></a> |
| <span class="sourceLineNo">3617</span> final MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {<a name="line.3617"></a> |
| <span class="sourceLineNo">3618</span> List<Pair<NonceKey, WALEdit>> walEdits = new ArrayList<>();<a name="line.3618"></a> |
| <span class="sourceLineNo">3619</span><a name="line.3619"></a> |
| <span class="sourceLineNo">3620</span> visitBatchOperations(true, nextIndexToProcess + miniBatchOp.size(), new Visitor() {<a name="line.3620"></a> |
| <span class="sourceLineNo">3621</span> private Pair<NonceKey, WALEdit> curWALEditForNonce;<a name="line.3621"></a> |
| <span class="sourceLineNo">3622</span><a name="line.3622"></a> |
| <span class="sourceLineNo">3623</span> @Override<a name="line.3623"></a> |
| <span class="sourceLineNo">3624</span> public boolean visit(int index) throws IOException {<a name="line.3624"></a> |
| <span class="sourceLineNo">3625</span> Mutation m = getMutation(index);<a name="line.3625"></a> |
| <span class="sourceLineNo">3626</span> // we use durability of the original mutation for the mutation passed by CP.<a name="line.3626"></a> |
| <span class="sourceLineNo">3627</span> if (region.getEffectiveDurability(m.getDurability()) == Durability.SKIP_WAL) {<a name="line.3627"></a> |
| <span class="sourceLineNo">3628</span> region.recordMutationWithoutWal(m.getFamilyCellMap());<a name="line.3628"></a> |
| <span class="sourceLineNo">3629</span> return true;<a name="line.3629"></a> |
| <span class="sourceLineNo">3630</span> }<a name="line.3630"></a> |
| <span class="sourceLineNo">3631</span><a name="line.3631"></a> |
| <span class="sourceLineNo">3632</span> // the batch may contain multiple nonce keys (replay case). If so, write WALEdit for each.<a name="line.3632"></a> |
| <span class="sourceLineNo">3633</span> // Given how nonce keys are originally written, these should be contiguous.<a name="line.3633"></a> |
| <span class="sourceLineNo">3634</span> // They don't have to be, it will still work, just write more WALEdits than needed.<a name="line.3634"></a> |
| <span class="sourceLineNo">3635</span> long nonceGroup = getNonceGroup(index);<a name="line.3635"></a> |
| <span class="sourceLineNo">3636</span> long nonce = getNonce(index);<a name="line.3636"></a> |
| <span class="sourceLineNo">3637</span> if (curWALEditForNonce == null ||<a name="line.3637"></a> |
| <span class="sourceLineNo">3638</span> curWALEditForNonce.getFirst().getNonceGroup() != nonceGroup ||<a name="line.3638"></a> |
| <span class="sourceLineNo">3639</span> curWALEditForNonce.getFirst().getNonce() != nonce) {<a name="line.3639"></a> |
| <span class="sourceLineNo">3640</span> curWALEditForNonce = new Pair<>(new NonceKey(nonceGroup, nonce),<a name="line.3640"></a> |
| <span class="sourceLineNo">3641</span> new WALEdit(miniBatchOp.getCellCount(), isInReplay()));<a name="line.3641"></a> |
| <span class="sourceLineNo">3642</span> walEdits.add(curWALEditForNonce);<a name="line.3642"></a> |
| <span class="sourceLineNo">3643</span> }<a name="line.3643"></a> |
| <span class="sourceLineNo">3644</span> WALEdit walEdit = curWALEditForNonce.getSecond();<a name="line.3644"></a> |
| <span class="sourceLineNo">3645</span><a name="line.3645"></a> |
| <span class="sourceLineNo">3646</span> // Add WAL edits from CPs.<a name="line.3646"></a> |
| <span class="sourceLineNo">3647</span> WALEdit fromCP = walEditsFromCoprocessors[index];<a name="line.3647"></a> |
| <span class="sourceLineNo">3648</span> if (fromCP != null) {<a name="line.3648"></a> |
| <span class="sourceLineNo">3649</span> for (Cell cell : fromCP.getCells()) {<a name="line.3649"></a> |
| <span class="sourceLineNo">3650</span> walEdit.add(cell);<a name="line.3650"></a> |
| <span class="sourceLineNo">3651</span> }<a name="line.3651"></a> |
| <span class="sourceLineNo">3652</span> }<a name="line.3652"></a> |
| <span class="sourceLineNo">3653</span> walEdit.add(familyCellMaps[index]);<a name="line.3653"></a> |
| <span class="sourceLineNo">3654</span><a name="line.3654"></a> |
| <span class="sourceLineNo">3655</span> return true;<a name="line.3655"></a> |
| <span class="sourceLineNo">3656</span> }<a name="line.3656"></a> |
| <span class="sourceLineNo">3657</span> });<a name="line.3657"></a> |
| <span class="sourceLineNo">3658</span> return walEdits;<a name="line.3658"></a> |
| <span class="sourceLineNo">3659</span> }<a name="line.3659"></a> |
| <span class="sourceLineNo">3660</span><a name="line.3660"></a> |
| <span class="sourceLineNo">3661</span> /**<a name="line.3661"></a> |
| <span class="sourceLineNo">3662</span> * This method completes mini-batch operations by calling postBatchMutate() CP hook (if<a name="line.3662"></a> |
| <span class="sourceLineNo">3663</span> * required) and completing mvcc.<a name="line.3663"></a> |
| <span class="sourceLineNo">3664</span> */<a name="line.3664"></a> |
| <span class="sourceLineNo">3665</span> public void completeMiniBatchOperations(<a name="line.3665"></a> |
| <span class="sourceLineNo">3666</span> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final WriteEntry writeEntry)<a name="line.3666"></a> |
| <span class="sourceLineNo">3667</span> throws IOException {<a name="line.3667"></a> |
| <span class="sourceLineNo">3668</span> if (writeEntry != null) {<a name="line.3668"></a> |
| <span class="sourceLineNo">3669</span> region.mvcc.completeAndWait(writeEntry);<a name="line.3669"></a> |
| <span class="sourceLineNo">3670</span> }<a name="line.3670"></a> |
| <span class="sourceLineNo">3671</span> }<a name="line.3671"></a> |
| <span class="sourceLineNo">3672</span><a name="line.3672"></a> |
| <span class="sourceLineNo">3673</span> public void doPostOpCleanupForMiniBatch(<a name="line.3673"></a> |
| <span class="sourceLineNo">3674</span> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final WALEdit walEdit,<a name="line.3674"></a> |
| <span class="sourceLineNo">3675</span> boolean success) throws IOException {<a name="line.3675"></a> |
| <span class="sourceLineNo">3676</span> doFinishHotnessProtector(miniBatchOp);<a name="line.3676"></a> |
| <span class="sourceLineNo">3677</span> }<a name="line.3677"></a> |
| <span class="sourceLineNo">3678</span><a name="line.3678"></a> |
| <span class="sourceLineNo">3679</span> private void doFinishHotnessProtector(<a name="line.3679"></a> |
| <span class="sourceLineNo">3680</span> final MiniBatchOperationInProgress<Mutation> miniBatchOp) {<a name="line.3680"></a> |
| <span class="sourceLineNo">3681</span> // check and return if the protector is not enabled<a name="line.3681"></a> |
| <span class="sourceLineNo">3682</span> if (!region.storeHotnessProtector.isEnable()) {<a name="line.3682"></a> |
| <span class="sourceLineNo">3683</span> return;<a name="line.3683"></a> |
| <span class="sourceLineNo">3684</span> }<a name="line.3684"></a> |
| <span class="sourceLineNo">3685</span> // miniBatchOp is null, if and only if lockRowsAndBuildMiniBatch throwing exception.<a name="line.3685"></a> |
| <span class="sourceLineNo">3686</span> // This case was handled.<a name="line.3686"></a> |
| <span class="sourceLineNo">3687</span> if (miniBatchOp == null) {<a name="line.3687"></a> |
| <span class="sourceLineNo">3688</span> return;<a name="line.3688"></a> |
| <span class="sourceLineNo">3689</span> }<a name="line.3689"></a> |
| <span class="sourceLineNo">3690</span><a name="line.3690"></a> |
| <span class="sourceLineNo">3691</span> final int finalLastIndexExclusive = miniBatchOp.getLastIndexExclusive();<a name="line.3691"></a> |
| <span class="sourceLineNo">3692</span><a name="line.3692"></a> |
| <span class="sourceLineNo">3693</span> for (int i = nextIndexToProcess; i < finalLastIndexExclusive; i++) {<a name="line.3693"></a> |
| <span class="sourceLineNo">3694</span> switch (retCodeDetails[i].getOperationStatusCode()) {<a name="line.3694"></a> |
| <span class="sourceLineNo">3695</span> case SUCCESS:<a name="line.3695"></a> |
| <span class="sourceLineNo">3696</span> case FAILURE:<a name="line.3696"></a> |
| <span class="sourceLineNo">3697</span> region.storeHotnessProtector.finish(getMutation(i).getFamilyCellMap());<a name="line.3697"></a> |
| <span class="sourceLineNo">3698</span> break;<a name="line.3698"></a> |
| <span class="sourceLineNo">3699</span> default:<a name="line.3699"></a> |
| <span class="sourceLineNo">3700</span> // do nothing<a name="line.3700"></a> |
| <span class="sourceLineNo">3701</span> // We won't start the protector for NOT_RUN/BAD_FAMILY/SANITY_CHECK_FAILURE and the<a name="line.3701"></a> |
| <span class="sourceLineNo">3702</span> // STORE_TOO_BUSY case is handled in StoreHotnessProtector#start<a name="line.3702"></a> |
| <span class="sourceLineNo">3703</span> break;<a name="line.3703"></a> |
| <span class="sourceLineNo">3704</span> }<a name="line.3704"></a> |
| <span class="sourceLineNo">3705</span> }<a name="line.3705"></a> |
| <span class="sourceLineNo">3706</span> }<a name="line.3706"></a> |
| <span class="sourceLineNo">3707</span><a name="line.3707"></a> |
| <span class="sourceLineNo">3708</span> /**<a name="line.3708"></a> |
| <span class="sourceLineNo">3709</span> * Atomically apply the given map of family->edits to the memstore.<a name="line.3709"></a> |
| <span class="sourceLineNo">3710</span> * This handles the consistency control on its own, but the caller<a name="line.3710"></a> |
| <span class="sourceLineNo">3711</span> * should already have locked updatesLock.readLock(). This also does<a name="line.3711"></a> |
| <span class="sourceLineNo">3712</span> * <b>not</b> check the families for validity.<a name="line.3712"></a> |
| <span class="sourceLineNo">3713</span> *<a name="line.3713"></a> |
| <span class="sourceLineNo">3714</span> * @param familyMap Map of Cells by family<a name="line.3714"></a> |
| <span class="sourceLineNo">3715</span> */<a name="line.3715"></a> |
| <span class="sourceLineNo">3716</span> protected void applyFamilyMapToMemStore(Map<byte[], List<Cell>> familyMap,<a name="line.3716"></a> |
| <span class="sourceLineNo">3717</span> MemStoreSizing memstoreAccounting) throws IOException {<a name="line.3717"></a> |
| <span class="sourceLineNo">3718</span> for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) {<a name="line.3718"></a> |
| <span class="sourceLineNo">3719</span> byte[] family = e.getKey();<a name="line.3719"></a> |
| <span class="sourceLineNo">3720</span> List<Cell> cells = e.getValue();<a name="line.3720"></a> |
| <span class="sourceLineNo">3721</span> assert cells instanceof RandomAccess;<a name="line.3721"></a> |
| <span class="sourceLineNo">3722</span> region.applyToMemStore(region.getStore(family), cells, false, memstoreAccounting);<a name="line.3722"></a> |
| <span class="sourceLineNo">3723</span> }<a name="line.3723"></a> |
| <span class="sourceLineNo">3724</span> }<a name="line.3724"></a> |
| <span class="sourceLineNo">3725</span> }<a name="line.3725"></a> |
| <span class="sourceLineNo">3726</span><a name="line.3726"></a> |
| <span class="sourceLineNo">3727</span><a name="line.3727"></a> |
| <span class="sourceLineNo">3728</span> /**<a name="line.3728"></a> |
| <span class="sourceLineNo">3729</span> * Batch of mutation operations. Base class is shared with {@link ReplayBatchOperation} as most of<a name="line.3729"></a> |
| <span class="sourceLineNo">3730</span> * the logic is same.<a name="line.3730"></a> |
| <span class="sourceLineNo">3731</span> */<a name="line.3731"></a> |
| <span class="sourceLineNo">3732</span> private static class MutationBatchOperation extends BatchOperation<Mutation> {<a name="line.3732"></a> |
| <span class="sourceLineNo">3733</span><a name="line.3733"></a> |
| <span class="sourceLineNo">3734</span> private long nonceGroup;<a name="line.3734"></a> |
| <span class="sourceLineNo">3735</span><a name="line.3735"></a> |
| <span class="sourceLineNo">3736</span> private long nonce;<a name="line.3736"></a> |
| <span class="sourceLineNo">3737</span><a name="line.3737"></a> |
| <span class="sourceLineNo">3738</span> public MutationBatchOperation(final HRegion region, Mutation[] operations, boolean atomic,<a name="line.3738"></a> |
| <span class="sourceLineNo">3739</span> long nonceGroup, long nonce) {<a name="line.3739"></a> |
| <span class="sourceLineNo">3740</span> super(region, operations);<a name="line.3740"></a> |
| <span class="sourceLineNo">3741</span> this.atomic = atomic;<a name="line.3741"></a> |
| <span class="sourceLineNo">3742</span> this.nonceGroup = nonceGroup;<a name="line.3742"></a> |
| <span class="sourceLineNo">3743</span> this.nonce = nonce;<a name="line.3743"></a> |
| <span class="sourceLineNo">3744</span> }<a name="line.3744"></a> |
| <span class="sourceLineNo">3745</span><a name="line.3745"></a> |
| <span class="sourceLineNo">3746</span> @Override<a name="line.3746"></a> |
| <span class="sourceLineNo">3747</span> public Mutation getMutation(int index) {<a name="line.3747"></a> |
| <span class="sourceLineNo">3748</span> return this.operations[index];<a name="line.3748"></a> |
| <span class="sourceLineNo">3749</span> }<a name="line.3749"></a> |
| <span class="sourceLineNo">3750</span><a name="line.3750"></a> |
| <span class="sourceLineNo">3751</span> @Override<a name="line.3751"></a> |
| <span class="sourceLineNo">3752</span> public long getNonceGroup(int index) {<a name="line.3752"></a> |
| <span class="sourceLineNo">3753</span> return nonceGroup;<a name="line.3753"></a> |
| <span class="sourceLineNo">3754</span> }<a name="line.3754"></a> |
| <span class="sourceLineNo">3755</span><a name="line.3755"></a> |
| <span class="sourceLineNo">3756</span> @Override<a name="line.3756"></a> |
| <span class="sourceLineNo">3757</span> public long getNonce(int index) {<a name="line.3757"></a> |
| <span class="sourceLineNo">3758</span> return nonce;<a name="line.3758"></a> |
| <span class="sourceLineNo">3759</span> }<a name="line.3759"></a> |
| <span class="sourceLineNo">3760</span><a name="line.3760"></a> |
| <span class="sourceLineNo">3761</span> @Override<a name="line.3761"></a> |
| <span class="sourceLineNo">3762</span> public Mutation[] getMutationsForCoprocs() {<a name="line.3762"></a> |
| <span class="sourceLineNo">3763</span> return this.operations;<a name="line.3763"></a> |
| <span class="sourceLineNo">3764</span> }<a name="line.3764"></a> |
| <span class="sourceLineNo">3765</span><a name="line.3765"></a> |
| <span class="sourceLineNo">3766</span> @Override<a name="line.3766"></a> |
| <span class="sourceLineNo">3767</span> public boolean isInReplay() {<a name="line.3767"></a> |
| <span class="sourceLineNo">3768</span> return false;<a name="line.3768"></a> |
| <span class="sourceLineNo">3769</span> }<a name="line.3769"></a> |
| <span class="sourceLineNo">3770</span><a name="line.3770"></a> |
| <span class="sourceLineNo">3771</span> @Override<a name="line.3771"></a> |
| <span class="sourceLineNo">3772</span> public long getOrigLogSeqNum() {<a name="line.3772"></a> |
| <span class="sourceLineNo">3773</span> return SequenceId.NO_SEQUENCE_ID;<a name="line.3773"></a> |
| <span class="sourceLineNo">3774</span> }<a name="line.3774"></a> |
| <span class="sourceLineNo">3775</span><a name="line.3775"></a> |
| <span class="sourceLineNo">3776</span> @Override<a name="line.3776"></a> |
| <span class="sourceLineNo">3777</span> public void startRegionOperation() throws IOException {<a name="line.3777"></a> |
| <span class="sourceLineNo">3778</span> region.startRegionOperation(Operation.BATCH_MUTATE);<a name="line.3778"></a> |
| <span class="sourceLineNo">3779</span> }<a name="line.3779"></a> |
| <span class="sourceLineNo">3780</span><a name="line.3780"></a> |
| <span class="sourceLineNo">3781</span> @Override<a name="line.3781"></a> |
| <span class="sourceLineNo">3782</span> public void closeRegionOperation() throws IOException {<a name="line.3782"></a> |
| <span class="sourceLineNo">3783</span> region.closeRegionOperation(Operation.BATCH_MUTATE);<a name="line.3783"></a> |
| <span class="sourceLineNo">3784</span> }<a name="line.3784"></a> |
| <span class="sourceLineNo">3785</span><a name="line.3785"></a> |
| <span class="sourceLineNo">3786</span> @Override<a name="line.3786"></a> |
| <span class="sourceLineNo">3787</span> public void checkAndPreparePut(Put p) throws IOException {<a name="line.3787"></a> |
| <span class="sourceLineNo">3788</span> region.checkFamilies(p.getFamilyCellMap().keySet(), p.getDurability());<a name="line.3788"></a> |
| <span class="sourceLineNo">3789</span> }<a name="line.3789"></a> |
| <span class="sourceLineNo">3790</span><a name="line.3790"></a> |
| <span class="sourceLineNo">3791</span> @Override<a name="line.3791"></a> |
| <span class="sourceLineNo">3792</span> public void checkAndPrepare() throws IOException {<a name="line.3792"></a> |
| <span class="sourceLineNo">3793</span> // index 0: puts, index 1: deletes, index 2: increments, index 3: append<a name="line.3793"></a> |
| <span class="sourceLineNo">3794</span> final int[] metrics = {0, 0, 0, 0};<a name="line.3794"></a> |
| <span class="sourceLineNo">3795</span><a name="line.3795"></a> |
| <span class="sourceLineNo">3796</span> visitBatchOperations(true, this.size(), new Visitor() {<a name="line.3796"></a> |
| <span class="sourceLineNo">3797</span> private long now = EnvironmentEdgeManager.currentTime();<a name="line.3797"></a> |
| <span class="sourceLineNo">3798</span> private WALEdit walEdit;<a name="line.3798"></a> |
| <span class="sourceLineNo">3799</span> @Override<a name="line.3799"></a> |
| <span class="sourceLineNo">3800</span> public boolean visit(int index) throws IOException {<a name="line.3800"></a> |
| <span class="sourceLineNo">3801</span> // Run coprocessor pre hook outside of locks to avoid deadlock<a name="line.3801"></a> |
| <span class="sourceLineNo">3802</span> if (region.coprocessorHost != null) {<a name="line.3802"></a> |
| <span class="sourceLineNo">3803</span> if (walEdit == null) {<a name="line.3803"></a> |
| <span class="sourceLineNo">3804</span> walEdit = new WALEdit();<a name="line.3804"></a> |
| <span class="sourceLineNo">3805</span> }<a name="line.3805"></a> |
| <span class="sourceLineNo">3806</span> callPreMutateCPHook(index, walEdit, metrics);<a name="line.3806"></a> |
| <span class="sourceLineNo">3807</span> if (!walEdit.isEmpty()) {<a name="line.3807"></a> |
| <span class="sourceLineNo">3808</span> walEditsFromCoprocessors[index] = walEdit;<a name="line.3808"></a> |
| <span class="sourceLineNo">3809</span> walEdit = null;<a name="line.3809"></a> |
| <span class="sourceLineNo">3810</span> }<a name="line.3810"></a> |
| <span class="sourceLineNo">3811</span> }<a name="line.3811"></a> |
| <span class="sourceLineNo">3812</span> if (isOperationPending(index)) {<a name="line.3812"></a> |
| <span class="sourceLineNo">3813</span> // TODO: Currently validation is done with current time before acquiring locks and<a name="line.3813"></a> |
| <span class="sourceLineNo">3814</span> // updates are done with different timestamps after acquiring locks. This behavior is<a name="line.3814"></a> |
| <span class="sourceLineNo">3815</span> // inherited from the code prior to this change. Can this be changed?<a name="line.3815"></a> |
| <span class="sourceLineNo">3816</span> checkAndPrepareMutation(index, now);<a name="line.3816"></a> |
| <span class="sourceLineNo">3817</span> }<a name="line.3817"></a> |
| <span class="sourceLineNo">3818</span> return true;<a name="line.3818"></a> |
| <span class="sourceLineNo">3819</span> }<a name="line.3819"></a> |
| <span class="sourceLineNo">3820</span> });<a name="line.3820"></a> |
| <span class="sourceLineNo">3821</span><a name="line.3821"></a> |
| <span class="sourceLineNo">3822</span> // FIXME: we may update metrics twice! here for all operations bypassed by CP and later in<a name="line.3822"></a> |
| <span class="sourceLineNo">3823</span> // normal processing.<a name="line.3823"></a> |
| <span class="sourceLineNo">3824</span> // Update metrics in same way as it is done when we go the normal processing route (we now<a name="line.3824"></a> |
| <span class="sourceLineNo">3825</span> // update general metrics though a Coprocessor did the work).<a name="line.3825"></a> |
| <span class="sourceLineNo">3826</span> if (region.metricsRegion != null) {<a name="line.3826"></a> |
| <span class="sourceLineNo">3827</span> if (metrics[0] > 0) {<a name="line.3827"></a> |
| <span class="sourceLineNo">3828</span> // There were some Puts in the batch.<a name="line.3828"></a> |
| <span class="sourceLineNo">3829</span> region.metricsRegion.updatePut();<a name="line.3829"></a> |
| <span class="sourceLineNo">3830</span> }<a name="line.3830"></a> |
| <span class="sourceLineNo">3831</span> if (metrics[1] > 0) {<a name="line.3831"></a> |
| <span class="sourceLineNo">3832</span> // There were some Deletes in the batch.<a name="line.3832"></a> |
| <span class="sourceLineNo">3833</span> region.metricsRegion.updateDelete();<a name="line.3833"></a> |
| <span class="sourceLineNo">3834</span> }<a name="line.3834"></a> |
| <span class="sourceLineNo">3835</span> if (metrics[2] > 0) {<a name="line.3835"></a> |
| <span class="sourceLineNo">3836</span> // There were some Increment in the batch.<a name="line.3836"></a> |
| <span class="sourceLineNo">3837</span> region.metricsRegion.updateIncrement();<a name="line.3837"></a> |
| <span class="sourceLineNo">3838</span> }<a name="line.3838"></a> |
| <span class="sourceLineNo">3839</span> if (metrics[3] > 0) {<a name="line.3839"></a> |
| <span class="sourceLineNo">3840</span> // There were some Append in the batch.<a name="line.3840"></a> |
| <span class="sourceLineNo">3841</span> region.metricsRegion.updateAppend();<a name="line.3841"></a> |
| <span class="sourceLineNo">3842</span> }<a name="line.3842"></a> |
| <span class="sourceLineNo">3843</span> }<a name="line.3843"></a> |
| <span class="sourceLineNo">3844</span> }<a name="line.3844"></a> |
| <span class="sourceLineNo">3845</span><a name="line.3845"></a> |
| <span class="sourceLineNo">3846</span> @Override<a name="line.3846"></a> |
| <span class="sourceLineNo">3847</span> public void prepareMiniBatchOperations(MiniBatchOperationInProgress<Mutation> miniBatchOp,<a name="line.3847"></a> |
| <span class="sourceLineNo">3848</span> long timestamp, final List<RowLock> acquiredRowLocks) throws IOException {<a name="line.3848"></a> |
| <span class="sourceLineNo">3849</span> visitBatchOperations(true, miniBatchOp.getLastIndexExclusive(), (int index) -> {<a name="line.3849"></a> |
| <span class="sourceLineNo">3850</span> Mutation mutation = getMutation(index);<a name="line.3850"></a> |
| <span class="sourceLineNo">3851</span> if (mutation instanceof Put) {<a name="line.3851"></a> |
| <span class="sourceLineNo">3852</span> HRegion.updateCellTimestamps(familyCellMaps[index].values(), Bytes.toBytes(timestamp));<a name="line.3852"></a> |
| <span class="sourceLineNo">3853</span> miniBatchOp.incrementNumOfPuts();<a name="line.3853"></a> |
| <span class="sourceLineNo">3854</span> } else if (mutation instanceof Delete) {<a name="line.3854"></a> |
| <span class="sourceLineNo">3855</span> region.prepareDeleteTimestamps(mutation, familyCellMaps[index],<a name="line.3855"></a> |
| <span class="sourceLineNo">3856</span> Bytes.toBytes(timestamp));<a name="line.3856"></a> |
| <span class="sourceLineNo">3857</span> miniBatchOp.incrementNumOfDeletes();<a name="line.3857"></a> |
| <span class="sourceLineNo">3858</span> } else if (mutation instanceof Increment || mutation instanceof Append) {<a name="line.3858"></a> |
| <span class="sourceLineNo">3859</span> boolean returnResults;<a name="line.3859"></a> |
| <span class="sourceLineNo">3860</span> if (mutation instanceof Increment) {<a name="line.3860"></a> |
| <span class="sourceLineNo">3861</span> returnResults = ((Increment) mutation).isReturnResults();<a name="line.3861"></a> |
| <span class="sourceLineNo">3862</span> } else {<a name="line.3862"></a> |
| <span class="sourceLineNo">3863</span> returnResults = ((Append) mutation).isReturnResults();<a name="line.3863"></a> |
| <span class="sourceLineNo">3864</span> }<a name="line.3864"></a> |
| <span class="sourceLineNo">3865</span><a name="line.3865"></a> |
| <span class="sourceLineNo">3866</span> // For nonce operations<a name="line.3866"></a> |
| <span class="sourceLineNo">3867</span> canProceed[index] = startNonceOperation(nonceGroup, nonce);<a name="line.3867"></a> |
| <span class="sourceLineNo">3868</span> if (!canProceed[index]) {<a name="line.3868"></a> |
| <span class="sourceLineNo">3869</span> Result result;<a name="line.3869"></a> |
| <span class="sourceLineNo">3870</span> if (returnResults) {<a name="line.3870"></a> |
| <span class="sourceLineNo">3871</span> // convert duplicate increment/append to get<a name="line.3871"></a> |
| <span class="sourceLineNo">3872</span> List<Cell> results = region.get(toGet(mutation), false, nonceGroup, nonce);<a name="line.3872"></a> |
| <span class="sourceLineNo">3873</span> result = Result.create(results);<a name="line.3873"></a> |
| <span class="sourceLineNo">3874</span> } else {<a name="line.3874"></a> |
| <span class="sourceLineNo">3875</span> result = Result.EMPTY_RESULT;<a name="line.3875"></a> |
| <span class="sourceLineNo">3876</span> }<a name="line.3876"></a> |
| <span class="sourceLineNo">3877</span> retCodeDetails[index] = new OperationStatus(OperationStatusCode.SUCCESS, result);<a name="line.3877"></a> |
| <span class="sourceLineNo">3878</span> return true;<a name="line.3878"></a> |
| <span class="sourceLineNo">3879</span> }<a name="line.3879"></a> |
| <span class="sourceLineNo">3880</span><a name="line.3880"></a> |
| <span class="sourceLineNo">3881</span> Result result = null;<a name="line.3881"></a> |
| <span class="sourceLineNo">3882</span> if (region.coprocessorHost != null) {<a name="line.3882"></a> |
| <span class="sourceLineNo">3883</span> if (mutation instanceof Increment) {<a name="line.3883"></a> |
| <span class="sourceLineNo">3884</span> result = region.coprocessorHost.preIncrementAfterRowLock((Increment) mutation);<a name="line.3884"></a> |
| <span class="sourceLineNo">3885</span> } else {<a name="line.3885"></a> |
| <span class="sourceLineNo">3886</span> result = region.coprocessorHost.preAppendAfterRowLock((Append) mutation);<a name="line.3886"></a> |
| <span class="sourceLineNo">3887</span> }<a name="line.3887"></a> |
| <span class="sourceLineNo">3888</span> }<a name="line.3888"></a> |
| <span class="sourceLineNo">3889</span> if (result != null) {<a name="line.3889"></a> |
| <span class="sourceLineNo">3890</span> retCodeDetails[index] = new OperationStatus(OperationStatusCode.SUCCESS,<a name="line.3890"></a> |
| <span class="sourceLineNo">3891</span> returnResults ? result : Result.EMPTY_RESULT);<a name="line.3891"></a> |
| <span class="sourceLineNo">3892</span> return true;<a name="line.3892"></a> |
| <span class="sourceLineNo">3893</span> }<a name="line.3893"></a> |
| <span class="sourceLineNo">3894</span><a name="line.3894"></a> |
| <span class="sourceLineNo">3895</span> List<Cell> results = returnResults ? new ArrayList<>(mutation.size()) : null;<a name="line.3895"></a> |
| <span class="sourceLineNo">3896</span> familyCellMaps[index] = reckonDeltas(mutation, results, timestamp);<a name="line.3896"></a> |
| <span class="sourceLineNo">3897</span> this.results[index] = results != null ? Result.create(results) : Result.EMPTY_RESULT;<a name="line.3897"></a> |
| <span class="sourceLineNo">3898</span><a name="line.3898"></a> |
| <span class="sourceLineNo">3899</span> if (mutation instanceof Increment) {<a name="line.3899"></a> |
| <span class="sourceLineNo">3900</span> miniBatchOp.incrementNumOfIncrements();<a name="line.3900"></a> |
| <span class="sourceLineNo">3901</span> } else {<a name="line.3901"></a> |
| <span class="sourceLineNo">3902</span> miniBatchOp.incrementNumOfAppends();<a name="line.3902"></a> |
| <span class="sourceLineNo">3903</span> }<a name="line.3903"></a> |
| <span class="sourceLineNo">3904</span> }<a name="line.3904"></a> |
| <span class="sourceLineNo">3905</span> region.rewriteCellTags(familyCellMaps[index], mutation);<a name="line.3905"></a> |
| <span class="sourceLineNo">3906</span><a name="line.3906"></a> |
| <span class="sourceLineNo">3907</span> // update cell count<a name="line.3907"></a> |
| <span class="sourceLineNo">3908</span> if (region.getEffectiveDurability(mutation.getDurability()) != Durability.SKIP_WAL) {<a name="line.3908"></a> |
| <span class="sourceLineNo">3909</span> for (List<Cell> cells : mutation.getFamilyCellMap().values()) {<a name="line.3909"></a> |
| <span class="sourceLineNo">3910</span> miniBatchOp.addCellCount(cells.size());<a name="line.3910"></a> |
| <span class="sourceLineNo">3911</span> }<a name="line.3911"></a> |
| <span class="sourceLineNo">3912</span> }<a name="line.3912"></a> |
| <span class="sourceLineNo">3913</span><a name="line.3913"></a> |
| <span class="sourceLineNo">3914</span> WALEdit fromCP = walEditsFromCoprocessors[index];<a name="line.3914"></a> |
| <span class="sourceLineNo">3915</span> if (fromCP != null) {<a name="line.3915"></a> |
| <span class="sourceLineNo">3916</span> miniBatchOp.addCellCount(fromCP.size());<a name="line.3916"></a> |
| <span class="sourceLineNo">3917</span> }<a name="line.3917"></a> |
| <span class="sourceLineNo">3918</span> return true;<a name="line.3918"></a> |
| <span class="sourceLineNo">3919</span> });<a name="line.3919"></a> |
| <span class="sourceLineNo">3920</span><a name="line.3920"></a> |
| <span class="sourceLineNo">3921</span> if (region.coprocessorHost != null) {<a name="line.3921"></a> |
| <span class="sourceLineNo">3922</span> // calling the pre CP hook for batch mutation<a name="line.3922"></a> |
| <span class="sourceLineNo">3923</span> region.coprocessorHost.preBatchMutate(miniBatchOp);<a name="line.3923"></a> |
| <span class="sourceLineNo">3924</span> checkAndMergeCPMutations(miniBatchOp, acquiredRowLocks, timestamp);<a name="line.3924"></a> |
| <span class="sourceLineNo">3925</span> }<a name="line.3925"></a> |
| <span class="sourceLineNo">3926</span> }<a name="line.3926"></a> |
| <span class="sourceLineNo">3927</span><a name="line.3927"></a> |
| <span class="sourceLineNo">3928</span> /**<a name="line.3928"></a> |
| <span class="sourceLineNo">3929</span> * Starts the nonce operation for a mutation, if needed.<a name="line.3929"></a> |
| <span class="sourceLineNo">3930</span> * @param nonceGroup Nonce group from the request.<a name="line.3930"></a> |
| <span class="sourceLineNo">3931</span> * @param nonce Nonce.<a name="line.3931"></a> |
| <span class="sourceLineNo">3932</span> * @return whether to proceed this mutation.<a name="line.3932"></a> |
| <span class="sourceLineNo">3933</span> */<a name="line.3933"></a> |
| <span class="sourceLineNo">3934</span> private boolean startNonceOperation(long nonceGroup, long nonce) throws IOException {<a name="line.3934"></a> |
| <span class="sourceLineNo">3935</span> if (region.rsServices == null || region.rsServices.getNonceManager() == null<a name="line.3935"></a> |
| <span class="sourceLineNo">3936</span> || nonce == HConstants.NO_NONCE) {<a name="line.3936"></a> |
| <span class="sourceLineNo">3937</span> return true;<a name="line.3937"></a> |
| <span class="sourceLineNo">3938</span> }<a name="line.3938"></a> |
| <span class="sourceLineNo">3939</span> boolean canProceed;<a name="line.3939"></a> |
| <span class="sourceLineNo">3940</span> try {<a name="line.3940"></a> |
| <span class="sourceLineNo">3941</span> canProceed = region.rsServices.getNonceManager()<a name="line.3941"></a> |
| <span class="sourceLineNo">3942</span> .startOperation(nonceGroup, nonce, region.rsServices);<a name="line.3942"></a> |
| <span class="sourceLineNo">3943</span> } catch (InterruptedException ex) {<a name="line.3943"></a> |
| <span class="sourceLineNo">3944</span> throw new InterruptedIOException("Nonce start operation interrupted");<a name="line.3944"></a> |
| <span class="sourceLineNo">3945</span> }<a name="line.3945"></a> |
| <span class="sourceLineNo">3946</span> return canProceed;<a name="line.3946"></a> |
| <span class="sourceLineNo">3947</span> }<a name="line.3947"></a> |
| <span class="sourceLineNo">3948</span><a name="line.3948"></a> |
| <span class="sourceLineNo">3949</span> /**<a name="line.3949"></a> |
| <span class="sourceLineNo">3950</span> * Ends nonce operation for a mutation, if needed.<a name="line.3950"></a> |
| <span class="sourceLineNo">3951</span> * @param nonceGroup Nonce group from the request. Always 0 in initial implementation.<a name="line.3951"></a> |
| <span class="sourceLineNo">3952</span> * @param nonce Nonce.<a name="line.3952"></a> |
| <span class="sourceLineNo">3953</span> * @param success Whether the operation for this nonce has succeeded.<a name="line.3953"></a> |
| <span class="sourceLineNo">3954</span> */<a name="line.3954"></a> |
| <span class="sourceLineNo">3955</span> private void endNonceOperation(long nonceGroup, long nonce, boolean success) {<a name="line.3955"></a> |
| <span class="sourceLineNo">3956</span> if (region.rsServices != null && region.rsServices.getNonceManager() != null<a name="line.3956"></a> |
| <span class="sourceLineNo">3957</span> && nonce != HConstants.NO_NONCE) {<a name="line.3957"></a> |
| <span class="sourceLineNo">3958</span> region.rsServices.getNonceManager().endOperation(nonceGroup, nonce, success);<a name="line.3958"></a> |
| <span class="sourceLineNo">3959</span> }<a name="line.3959"></a> |
| <span class="sourceLineNo">3960</span> }<a name="line.3960"></a> |
| <span class="sourceLineNo">3961</span><a name="line.3961"></a> |
| <span class="sourceLineNo">3962</span> private static Get toGet(final Mutation mutation) throws IOException {<a name="line.3962"></a> |
| <span class="sourceLineNo">3963</span> assert mutation instanceof Increment || mutation instanceof Append;<a name="line.3963"></a> |
| <span class="sourceLineNo">3964</span> Get get = new Get(mutation.getRow());<a name="line.3964"></a> |
| <span class="sourceLineNo">3965</span> CellScanner cellScanner = mutation.cellScanner();<a name="line.3965"></a> |
| <span class="sourceLineNo">3966</span> while (!cellScanner.advance()) {<a name="line.3966"></a> |
| <span class="sourceLineNo">3967</span> Cell cell = cellScanner.current();<a name="line.3967"></a> |
| <span class="sourceLineNo">3968</span> get.addColumn(CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell));<a name="line.3968"></a> |
| <span class="sourceLineNo">3969</span> }<a name="line.3969"></a> |
| <span class="sourceLineNo">3970</span> if (mutation instanceof Increment) {<a name="line.3970"></a> |
| <span class="sourceLineNo">3971</span> // Increment<a name="line.3971"></a> |
| <span class="sourceLineNo">3972</span> Increment increment = (Increment) mutation;<a name="line.3972"></a> |
| <span class="sourceLineNo">3973</span> get.setTimeRange(increment.getTimeRange().getMin(), increment.getTimeRange().getMax());<a name="line.3973"></a> |
| <span class="sourceLineNo">3974</span> } else {<a name="line.3974"></a> |
| <span class="sourceLineNo">3975</span> // Append<a name="line.3975"></a> |
| <span class="sourceLineNo">3976</span> Append append = (Append) mutation;<a name="line.3976"></a> |
| <span class="sourceLineNo">3977</span> get.setTimeRange(append.getTimeRange().getMin(), append.getTimeRange().getMax());<a name="line.3977"></a> |
| <span class="sourceLineNo">3978</span> }<a name="line.3978"></a> |
| <span class="sourceLineNo">3979</span> for (Entry<String, byte[]> entry : mutation.getAttributesMap().entrySet()) {<a name="line.3979"></a> |
| <span class="sourceLineNo">3980</span> get.setAttribute(entry.getKey(), entry.getValue());<a name="line.3980"></a> |
| <span class="sourceLineNo">3981</span> }<a name="line.3981"></a> |
| <span class="sourceLineNo">3982</span> return get;<a name="line.3982"></a> |
| <span class="sourceLineNo">3983</span> }<a name="line.3983"></a> |
| <span class="sourceLineNo">3984</span><a name="line.3984"></a> |
| <span class="sourceLineNo">3985</span> private Map<byte[], List<Cell>> reckonDeltas(Mutation mutation, List<Cell> results,<a name="line.3985"></a> |
| <span class="sourceLineNo">3986</span> long now) throws IOException {<a name="line.3986"></a> |
| <span class="sourceLineNo">3987</span> assert mutation instanceof Increment || mutation instanceof Append;<a name="line.3987"></a> |
| <span class="sourceLineNo">3988</span> Map<byte[], List<Cell>> ret = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.3988"></a> |
| <span class="sourceLineNo">3989</span> // Process a Store/family at a time.<a name="line.3989"></a> |
| <span class="sourceLineNo">3990</span> for (Map.Entry<byte [], List<Cell>> entry: mutation.getFamilyCellMap().entrySet()) {<a name="line.3990"></a> |
| <span class="sourceLineNo">3991</span> final byte[] columnFamilyName = entry.getKey();<a name="line.3991"></a> |
| <span class="sourceLineNo">3992</span> List<Cell> deltas = entry.getValue();<a name="line.3992"></a> |
| <span class="sourceLineNo">3993</span> // Reckon for the Store what to apply to WAL and MemStore.<a name="line.3993"></a> |
| <span class="sourceLineNo">3994</span> List<Cell> toApply = reckonDeltasByStore(region.stores.get(columnFamilyName), mutation,<a name="line.3994"></a> |
| <span class="sourceLineNo">3995</span> now, deltas, results);<a name="line.3995"></a> |
| <span class="sourceLineNo">3996</span> if (!toApply.isEmpty()) {<a name="line.3996"></a> |
| <span class="sourceLineNo">3997</span> for (Cell cell : toApply) {<a name="line.3997"></a> |
| <span class="sourceLineNo">3998</span> HStore store = region.getStore(cell);<a name="line.3998"></a> |
| <span class="sourceLineNo">3999</span> if (store == null) {<a name="line.3999"></a> |
| <span class="sourceLineNo">4000</span> region.checkFamily(CellUtil.cloneFamily(cell));<a name="line.4000"></a> |
| <span class="sourceLineNo">4001</span> } else {<a name="line.4001"></a> |
| <span class="sourceLineNo">4002</span> ret.computeIfAbsent(store.getColumnFamilyDescriptor().getName(),<a name="line.4002"></a> |
| <span class="sourceLineNo">4003</span> key -> new ArrayList<>()).add(cell);<a name="line.4003"></a> |
| <span class="sourceLineNo">4004</span> }<a name="line.4004"></a> |
| <span class="sourceLineNo">4005</span> }<a name="line.4005"></a> |
| <span class="sourceLineNo">4006</span> }<a name="line.4006"></a> |
| <span class="sourceLineNo">4007</span> }<a name="line.4007"></a> |
| <span class="sourceLineNo">4008</span> return ret;<a name="line.4008"></a> |
| <span class="sourceLineNo">4009</span> }<a name="line.4009"></a> |
| <span class="sourceLineNo">4010</span><a name="line.4010"></a> |
| <span class="sourceLineNo">4011</span> /**<a name="line.4011"></a> |
| <span class="sourceLineNo">4012</span> * Reckon the Cells to apply to WAL, memstore, and to return to the Client in passed<a name="line.4012"></a> |
| <span class="sourceLineNo">4013</span> * column family/Store.<a name="line.4013"></a> |
| <span class="sourceLineNo">4014</span> *<a name="line.4014"></a> |
| <span class="sourceLineNo">4015</span> * Does Get of current value and then adds passed in deltas for this Store returning the<a name="line.4015"></a> |
| <span class="sourceLineNo">4016</span> * result.<a name="line.4016"></a> |
| <span class="sourceLineNo">4017</span> *<a name="line.4017"></a> |
| <span class="sourceLineNo">4018</span> * @param mutation The encompassing Mutation object<a name="line.4018"></a> |
| <span class="sourceLineNo">4019</span> * @param deltas Changes to apply to this Store; either increment amount or data to append<a name="line.4019"></a> |
| <span class="sourceLineNo">4020</span> * @param results In here we accumulate all the Cells we are to return to the client. If null,<a name="line.4020"></a> |
| <span class="sourceLineNo">4021</span> * client doesn't want results returned.<a name="line.4021"></a> |
| <span class="sourceLineNo">4022</span> * @return Resulting Cells after <code>deltas</code> have been applied to current<a name="line.4022"></a> |
| <span class="sourceLineNo">4023</span> * values. Side effect is our filling out of the <code>results</code> List.<a name="line.4023"></a> |
| <span class="sourceLineNo">4024</span> */<a name="line.4024"></a> |
| <span class="sourceLineNo">4025</span> private List<Cell> reckonDeltasByStore(HStore store, Mutation mutation, long now,<a name="line.4025"></a> |
| <span class="sourceLineNo">4026</span> List<Cell> deltas, List<Cell> results) throws IOException {<a name="line.4026"></a> |
| <span class="sourceLineNo">4027</span> assert mutation instanceof Increment || mutation instanceof Append;<a name="line.4027"></a> |
| <span class="sourceLineNo">4028</span> byte[] columnFamily = store.getColumnFamilyDescriptor().getName();<a name="line.4028"></a> |
| <span class="sourceLineNo">4029</span> List<Pair<Cell, Cell>> cellPairs = new ArrayList<>(deltas.size());<a name="line.4029"></a> |
| <span class="sourceLineNo">4030</span><a name="line.4030"></a> |
| <span class="sourceLineNo">4031</span> // Sort the cells so that they match the order that they appear in the Get results.<a name="line.4031"></a> |
| <span class="sourceLineNo">4032</span> // Otherwise, we won't be able to find the existing values if the cells are not specified<a name="line.4032"></a> |
| <span class="sourceLineNo">4033</span> // in order by the client since cells are in an array list.<a name="line.4033"></a> |
| <span class="sourceLineNo">4034</span> deltas.sort(store.getComparator());<a name="line.4034"></a> |
| <span class="sourceLineNo">4035</span><a name="line.4035"></a> |
| <span class="sourceLineNo">4036</span> // Get previous values for all columns in this family.<a name="line.4036"></a> |
| <span class="sourceLineNo">4037</span> Get get = new Get(mutation.getRow());<a name="line.4037"></a> |
| <span class="sourceLineNo">4038</span> for (Cell cell: deltas) {<a name="line.4038"></a> |
| <span class="sourceLineNo">4039</span> get.addColumn(columnFamily, CellUtil.cloneQualifier(cell));<a name="line.4039"></a> |
| <span class="sourceLineNo">4040</span> }<a name="line.4040"></a> |
| <span class="sourceLineNo">4041</span> TimeRange tr;<a name="line.4041"></a> |
| <span class="sourceLineNo">4042</span> if (mutation instanceof Increment) {<a name="line.4042"></a> |
| <span class="sourceLineNo">4043</span> tr = ((Increment) mutation).getTimeRange();<a name="line.4043"></a> |
| <span class="sourceLineNo">4044</span> } else {<a name="line.4044"></a> |
| <span class="sourceLineNo">4045</span> tr = ((Append) mutation).getTimeRange();<a name="line.4045"></a> |
| <span class="sourceLineNo">4046</span> }<a name="line.4046"></a> |
| <span class="sourceLineNo">4047</span><a name="line.4047"></a> |
| <span class="sourceLineNo">4048</span> if (tr != null) {<a name="line.4048"></a> |
| <span class="sourceLineNo">4049</span> get.setTimeRange(tr.getMin(), tr.getMax());<a name="line.4049"></a> |
| <span class="sourceLineNo">4050</span> }<a name="line.4050"></a> |
| <span class="sourceLineNo">4051</span><a name="line.4051"></a> |
| <span class="sourceLineNo">4052</span> List<Cell> currentValues = region.get(get, false);<a name="line.4052"></a> |
| <span class="sourceLineNo">4053</span><a name="line.4053"></a> |
| <span class="sourceLineNo">4054</span> // Iterate the input columns and update existing values if they were found, otherwise<a name="line.4054"></a> |
| <span class="sourceLineNo">4055</span> // add new column initialized to the delta amount<a name="line.4055"></a> |
| <span class="sourceLineNo">4056</span> int currentValuesIndex = 0;<a name="line.4056"></a> |
| <span class="sourceLineNo">4057</span> for (int i = 0; i < deltas.size(); i++) {<a name="line.4057"></a> |
| <span class="sourceLineNo">4058</span> Cell delta = deltas.get(i);<a name="line.4058"></a> |
| <span class="sourceLineNo">4059</span> Cell currentValue = null;<a name="line.4059"></a> |
| <span class="sourceLineNo">4060</span> if (currentValuesIndex < currentValues.size() &&<a name="line.4060"></a> |
| <span class="sourceLineNo">4061</span> CellUtil.matchingQualifier(currentValues.get(currentValuesIndex), delta)) {<a name="line.4061"></a> |
| <span class="sourceLineNo">4062</span> currentValue = currentValues.get(currentValuesIndex);<a name="line.4062"></a> |
| <span class="sourceLineNo">4063</span> if (i < (deltas.size() - 1) && !CellUtil.matchingQualifier(delta, deltas.get(i + 1))) {<a name="line.4063"></a> |
| <span class="sourceLineNo">4064</span> currentValuesIndex++;<a name="line.4064"></a> |
| <span class="sourceLineNo">4065</span> }<a name="line.4065"></a> |
| <span class="sourceLineNo">4066</span> }<a name="line.4066"></a> |
| <span class="sourceLineNo">4067</span> // Switch on whether this an increment or an append building the new Cell to apply.<a name="line.4067"></a> |
| <span class="sourceLineNo">4068</span> Cell newCell;<a name="line.4068"></a> |
| <span class="sourceLineNo">4069</span> if (mutation instanceof Increment) {<a name="line.4069"></a> |
| <span class="sourceLineNo">4070</span> long deltaAmount = getLongValue(delta);<a name="line.4070"></a> |
| <span class="sourceLineNo">4071</span> final long newValue = currentValue == null ?<a name="line.4071"></a> |
| <span class="sourceLineNo">4072</span> deltaAmount : getLongValue(currentValue) + deltaAmount;<a name="line.4072"></a> |
| <span class="sourceLineNo">4073</span> newCell = reckonDelta(delta, currentValue, columnFamily, now, mutation,<a name="line.4073"></a> |
| <span class="sourceLineNo">4074</span> (oldCell) -> Bytes.toBytes(newValue));<a name="line.4074"></a> |
| <span class="sourceLineNo">4075</span> } else {<a name="line.4075"></a> |
| <span class="sourceLineNo">4076</span> newCell = reckonDelta(delta, currentValue, columnFamily, now, mutation,<a name="line.4076"></a> |
| <span class="sourceLineNo">4077</span> (oldCell) -><a name="line.4077"></a> |
| <span class="sourceLineNo">4078</span> ByteBuffer.wrap(new byte[delta.getValueLength() + oldCell.getValueLength()])<a name="line.4078"></a> |
| <span class="sourceLineNo">4079</span> .put(oldCell.getValueArray(), oldCell.getValueOffset(), oldCell.getValueLength())<a name="line.4079"></a> |
| <span class="sourceLineNo">4080</span> .put(delta.getValueArray(), delta.getValueOffset(), delta.getValueLength())<a name="line.4080"></a> |
| <span class="sourceLineNo">4081</span> .array()<a name="line.4081"></a> |
| <span class="sourceLineNo">4082</span> );<a name="line.4082"></a> |
| <span class="sourceLineNo">4083</span> }<a name="line.4083"></a> |
| <span class="sourceLineNo">4084</span> if (region.maxCellSize > 0) {<a name="line.4084"></a> |
| <span class="sourceLineNo">4085</span> int newCellSize = PrivateCellUtil.estimatedSerializedSizeOf(newCell);<a name="line.4085"></a> |
| <span class="sourceLineNo">4086</span> if (newCellSize > region.maxCellSize) {<a name="line.4086"></a> |
| <span class="sourceLineNo">4087</span> String msg = "Cell with size " + newCellSize + " exceeds limit of "<a name="line.4087"></a> |
| <span class="sourceLineNo">4088</span> + region.maxCellSize + " bytes in region " + this;<a name="line.4088"></a> |
| <span class="sourceLineNo">4089</span> LOG.debug(msg);<a name="line.4089"></a> |
| <span class="sourceLineNo">4090</span> throw new DoNotRetryIOException(msg);<a name="line.4090"></a> |
| <span class="sourceLineNo">4091</span> }<a name="line.4091"></a> |
| <span class="sourceLineNo">4092</span> }<a name="line.4092"></a> |
| <span class="sourceLineNo">4093</span> cellPairs.add(new Pair<>(currentValue, newCell));<a name="line.4093"></a> |
| <span class="sourceLineNo">4094</span> // Add to results to get returned to the Client. If null, cilent does not want results.<a name="line.4094"></a> |
| <span class="sourceLineNo">4095</span> if (results != null) {<a name="line.4095"></a> |
| <span class="sourceLineNo">4096</span> results.add(newCell);<a name="line.4096"></a> |
| <span class="sourceLineNo">4097</span> }<a name="line.4097"></a> |
| <span class="sourceLineNo">4098</span> }<a name="line.4098"></a> |
| <span class="sourceLineNo">4099</span> // Give coprocessors a chance to update the new cells before apply to WAL or memstore<a name="line.4099"></a> |
| <span class="sourceLineNo">4100</span> if (region.coprocessorHost != null) {<a name="line.4100"></a> |
| <span class="sourceLineNo">4101</span> // Here the operation must be increment or append.<a name="line.4101"></a> |
| <span class="sourceLineNo">4102</span> cellPairs = mutation instanceof Increment ?<a name="line.4102"></a> |
| <span class="sourceLineNo">4103</span> region.coprocessorHost.postIncrementBeforeWAL(mutation, cellPairs) :<a name="line.4103"></a> |
| <span class="sourceLineNo">4104</span> region.coprocessorHost.postAppendBeforeWAL(mutation, cellPairs);<a name="line.4104"></a> |
| <span class="sourceLineNo">4105</span> }<a name="line.4105"></a> |
| <span class="sourceLineNo">4106</span> return cellPairs.stream().map(Pair::getSecond).collect(Collectors.toList());<a name="line.4106"></a> |
| <span class="sourceLineNo">4107</span> }<a name="line.4107"></a> |
| <span class="sourceLineNo">4108</span><a name="line.4108"></a> |
| <span class="sourceLineNo">4109</span> private static Cell reckonDelta(final Cell delta, final Cell currentCell,<a name="line.4109"></a> |
| <span class="sourceLineNo">4110</span> final byte[] columnFamily, final long now, Mutation mutation,<a name="line.4110"></a> |
| <span class="sourceLineNo">4111</span> Function<Cell, byte[]> supplier) throws IOException {<a name="line.4111"></a> |
| <span class="sourceLineNo">4112</span> // Forward any tags found on the delta.<a name="line.4112"></a> |
| <span class="sourceLineNo">4113</span> List<Tag> tags = TagUtil.carryForwardTags(delta);<a name="line.4113"></a> |
| <span class="sourceLineNo">4114</span> tags = TagUtil.carryForwardTTLTag(tags, mutation.getTTL());<a name="line.4114"></a> |
| <span class="sourceLineNo">4115</span> if (currentCell != null) {<a name="line.4115"></a> |
| <span class="sourceLineNo">4116</span> tags = TagUtil.carryForwardTags(tags, currentCell);<a name="line.4116"></a> |
| <span class="sourceLineNo">4117</span> byte[] newValue = supplier.apply(currentCell);<a name="line.4117"></a> |
| <span class="sourceLineNo">4118</span> return ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)<a name="line.4118"></a> |
| <span class="sourceLineNo">4119</span> .setRow(mutation.getRow(), 0, mutation.getRow().length)<a name="line.4119"></a> |
| <span class="sourceLineNo">4120</span> .setFamily(columnFamily, 0, columnFamily.length)<a name="line.4120"></a> |
| <span class="sourceLineNo">4121</span> // copy the qualifier if the cell is located in shared memory.<a name="line.4121"></a> |
| <span class="sourceLineNo">4122</span> .setQualifier(CellUtil.cloneQualifier(delta))<a name="line.4122"></a> |
| <span class="sourceLineNo">4123</span> .setTimestamp(Math.max(currentCell.getTimestamp() + 1, now))<a name="line.4123"></a> |
| <span class="sourceLineNo">4124</span> .setType(KeyValue.Type.Put.getCode())<a name="line.4124"></a> |
| <span class="sourceLineNo">4125</span> .setValue(newValue, 0, newValue.length)<a name="line.4125"></a> |
| <span class="sourceLineNo">4126</span> .setTags(TagUtil.fromList(tags))<a name="line.4126"></a> |
| <span class="sourceLineNo">4127</span> .build();<a name="line.4127"></a> |
| <span class="sourceLineNo">4128</span> } else {<a name="line.4128"></a> |
| <span class="sourceLineNo">4129</span> PrivateCellUtil.updateLatestStamp(delta, now);<a name="line.4129"></a> |
| <span class="sourceLineNo">4130</span> return CollectionUtils.isEmpty(tags) ? delta : PrivateCellUtil.createCell(delta, tags);<a name="line.4130"></a> |
| <span class="sourceLineNo">4131</span> }<a name="line.4131"></a> |
| <span class="sourceLineNo">4132</span> }<a name="line.4132"></a> |
| <span class="sourceLineNo">4133</span><a name="line.4133"></a> |
| <span class="sourceLineNo">4134</span> /**<a name="line.4134"></a> |
| <span class="sourceLineNo">4135</span> * @return Get the long out of the passed in Cell<a name="line.4135"></a> |
| <span class="sourceLineNo">4136</span> */<a name="line.4136"></a> |
| <span class="sourceLineNo">4137</span> private static long getLongValue(final Cell cell) throws DoNotRetryIOException {<a name="line.4137"></a> |
| <span class="sourceLineNo">4138</span> int len = cell.getValueLength();<a name="line.4138"></a> |
| <span class="sourceLineNo">4139</span> if (len != Bytes.SIZEOF_LONG) {<a name="line.4139"></a> |
| <span class="sourceLineNo">4140</span> // throw DoNotRetryIOException instead of IllegalArgumentException<a name="line.4140"></a> |
| <span class="sourceLineNo">4141</span> throw new DoNotRetryIOException("Field is not a long, it's " + len + " bytes wide");<a name="line.4141"></a> |
| <span class="sourceLineNo">4142</span> }<a name="line.4142"></a> |
| <span class="sourceLineNo">4143</span> return PrivateCellUtil.getValueAsLong(cell);<a name="line.4143"></a> |
| <span class="sourceLineNo">4144</span> }<a name="line.4144"></a> |
| <span class="sourceLineNo">4145</span><a name="line.4145"></a> |
| <span class="sourceLineNo">4146</span> @Override<a name="line.4146"></a> |
| <span class="sourceLineNo">4147</span> public List<Pair<NonceKey, WALEdit>> buildWALEdits(final MiniBatchOperationInProgress<Mutation><a name="line.4147"></a> |
| <span class="sourceLineNo">4148</span> miniBatchOp) throws IOException {<a name="line.4148"></a> |
| <span class="sourceLineNo">4149</span> List<Pair<NonceKey, WALEdit>> walEdits = super.buildWALEdits(miniBatchOp);<a name="line.4149"></a> |
| <span class="sourceLineNo">4150</span> // for MutationBatchOperation, more than one nonce is not allowed<a name="line.4150"></a> |
| <span class="sourceLineNo">4151</span> if (walEdits.size() > 1) {<a name="line.4151"></a> |
| <span class="sourceLineNo">4152</span> throw new IOException("Found multiple nonce keys per batch!");<a name="line.4152"></a> |
| <span class="sourceLineNo">4153</span> }<a name="line.4153"></a> |
| <span class="sourceLineNo">4154</span> return walEdits;<a name="line.4154"></a> |
| <span class="sourceLineNo">4155</span> }<a name="line.4155"></a> |
| <span class="sourceLineNo">4156</span><a name="line.4156"></a> |
| <span class="sourceLineNo">4157</span> @Override<a name="line.4157"></a> |
| <span class="sourceLineNo">4158</span> public WriteEntry writeMiniBatchOperationsToMemStore(<a name="line.4158"></a> |
| <span class="sourceLineNo">4159</span> final MiniBatchOperationInProgress<Mutation> miniBatchOp, @Nullable WriteEntry writeEntry)<a name="line.4159"></a> |
| <span class="sourceLineNo">4160</span> throws IOException {<a name="line.4160"></a> |
| <span class="sourceLineNo">4161</span> if (writeEntry == null) {<a name="line.4161"></a> |
| <span class="sourceLineNo">4162</span> writeEntry = region.mvcc.begin();<a name="line.4162"></a> |
| <span class="sourceLineNo">4163</span> }<a name="line.4163"></a> |
| <span class="sourceLineNo">4164</span> super.writeMiniBatchOperationsToMemStore(miniBatchOp, writeEntry.getWriteNumber());<a name="line.4164"></a> |
| <span class="sourceLineNo">4165</span> return writeEntry;<a name="line.4165"></a> |
| <span class="sourceLineNo">4166</span> }<a name="line.4166"></a> |
| <span class="sourceLineNo">4167</span><a name="line.4167"></a> |
| <span class="sourceLineNo">4168</span> @Override<a name="line.4168"></a> |
| <span class="sourceLineNo">4169</span> public void completeMiniBatchOperations(<a name="line.4169"></a> |
| <span class="sourceLineNo">4170</span> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final WriteEntry writeEntry)<a name="line.4170"></a> |
| <span class="sourceLineNo">4171</span> throws IOException {<a name="line.4171"></a> |
| <span class="sourceLineNo">4172</span> // TODO: can it be done after completing mvcc?<a name="line.4172"></a> |
| <span class="sourceLineNo">4173</span> // calling the post CP hook for batch mutation<a name="line.4173"></a> |
| <span class="sourceLineNo">4174</span> if (region.coprocessorHost != null) {<a name="line.4174"></a> |
| <span class="sourceLineNo">4175</span> region.coprocessorHost.postBatchMutate(miniBatchOp);<a name="line.4175"></a> |
| <span class="sourceLineNo">4176</span> }<a name="line.4176"></a> |
| <span class="sourceLineNo">4177</span> super.completeMiniBatchOperations(miniBatchOp, writeEntry);<a name="line.4177"></a> |
| <span class="sourceLineNo">4178</span><a name="line.4178"></a> |
| <span class="sourceLineNo">4179</span> if (nonce != HConstants.NO_NONCE) {<a name="line.4179"></a> |
| <span class="sourceLineNo">4180</span> if (region.rsServices != null && region.rsServices.getNonceManager() != null) {<a name="line.4180"></a> |
| <span class="sourceLineNo">4181</span> region.rsServices.getNonceManager()<a name="line.4181"></a> |
| <span class="sourceLineNo">4182</span> .addMvccToOperationContext(nonceGroup, nonce, writeEntry.getWriteNumber());<a name="line.4182"></a> |
| <span class="sourceLineNo">4183</span> }<a name="line.4183"></a> |
| <span class="sourceLineNo">4184</span> }<a name="line.4184"></a> |
| <span class="sourceLineNo">4185</span> }<a name="line.4185"></a> |
| <span class="sourceLineNo">4186</span><a name="line.4186"></a> |
| <span class="sourceLineNo">4187</span> @Override<a name="line.4187"></a> |
| <span class="sourceLineNo">4188</span> public void doPostOpCleanupForMiniBatch(MiniBatchOperationInProgress<Mutation> miniBatchOp,<a name="line.4188"></a> |
| <span class="sourceLineNo">4189</span> final WALEdit walEdit, boolean success) throws IOException {<a name="line.4189"></a> |
| <span class="sourceLineNo">4190</span><a name="line.4190"></a> |
| <span class="sourceLineNo">4191</span> super.doPostOpCleanupForMiniBatch(miniBatchOp, walEdit, success);<a name="line.4191"></a> |
| <span class="sourceLineNo">4192</span> if (miniBatchOp != null) {<a name="line.4192"></a> |
| <span class="sourceLineNo">4193</span> // synced so that the coprocessor contract is adhered to.<a name="line.4193"></a> |
| <span class="sourceLineNo">4194</span> if (region.coprocessorHost != null) {<a name="line.4194"></a> |
| <span class="sourceLineNo">4195</span> visitBatchOperations(false, miniBatchOp.getLastIndexExclusive(), (int i) -> {<a name="line.4195"></a> |
| <span class="sourceLineNo">4196</span> // only for successful puts/deletes/increments/appends<a name="line.4196"></a> |
| <span class="sourceLineNo">4197</span> if (retCodeDetails[i].getOperationStatusCode() == OperationStatusCode.SUCCESS) {<a name="line.4197"></a> |
| <span class="sourceLineNo">4198</span> Mutation m = getMutation(i);<a name="line.4198"></a> |
| <span class="sourceLineNo">4199</span> if (m instanceof Put) {<a name="line.4199"></a> |
| <span class="sourceLineNo">4200</span> region.coprocessorHost.postPut((Put) m, walEdit, m.getDurability());<a name="line.4200"></a> |
| <span class="sourceLineNo">4201</span> } else if (m instanceof Delete) {<a name="line.4201"></a> |
| <span class="sourceLineNo">4202</span> region.coprocessorHost.postDelete((Delete) m, walEdit, m.getDurability());<a name="line.4202"></a> |
| <span class="sourceLineNo">4203</span> } else if (m instanceof Increment) {<a name="line.4203"></a> |
| <span class="sourceLineNo">4204</span> Result result = region.getCoprocessorHost().postIncrement((Increment) m,<a name="line.4204"></a> |
| <span class="sourceLineNo">4205</span> results[i]);<a name="line.4205"></a> |
| <span class="sourceLineNo">4206</span> if (result != results[i]) {<a name="line.4206"></a> |
| <span class="sourceLineNo">4207</span> retCodeDetails[i] =<a name="line.4207"></a> |
| <span class="sourceLineNo">4208</span> new OperationStatus(retCodeDetails[i].getOperationStatusCode(), result);<a name="line.4208"></a> |
| <span class="sourceLineNo">4209</span> }<a name="line.4209"></a> |
| <span class="sourceLineNo">4210</span> } else if (m instanceof Append) {<a name="line.4210"></a> |
| <span class="sourceLineNo">4211</span> Result result = region.getCoprocessorHost().postAppend((Append) m, results[i]);<a name="line.4211"></a> |
| <span class="sourceLineNo">4212</span> if (result != results[i]) {<a name="line.4212"></a> |
| <span class="sourceLineNo">4213</span> retCodeDetails[i] =<a name="line.4213"></a> |
| <span class="sourceLineNo">4214</span> new OperationStatus(retCodeDetails[i].getOperationStatusCode(), result);<a name="line.4214"></a> |
| <span class="sourceLineNo">4215</span> }<a name="line.4215"></a> |
| <span class="sourceLineNo">4216</span> }<a name="line.4216"></a> |
| <span class="sourceLineNo">4217</span> }<a name="line.4217"></a> |
| <span class="sourceLineNo">4218</span> return true;<a name="line.4218"></a> |
| <span class="sourceLineNo">4219</span> });<a name="line.4219"></a> |
| <span class="sourceLineNo">4220</span> }<a name="line.4220"></a> |
| <span class="sourceLineNo">4221</span><a name="line.4221"></a> |
| <span class="sourceLineNo">4222</span> // For nonce operations<a name="line.4222"></a> |
| <span class="sourceLineNo">4223</span> visitBatchOperations(false, miniBatchOp.getLastIndexExclusive(), (int i) -> {<a name="line.4223"></a> |
| <span class="sourceLineNo">4224</span> if (canProceed[i]) {<a name="line.4224"></a> |
| <span class="sourceLineNo">4225</span> endNonceOperation(nonceGroup, nonce,<a name="line.4225"></a> |
| <span class="sourceLineNo">4226</span> retCodeDetails[i].getOperationStatusCode() == OperationStatusCode.SUCCESS);<a name="line.4226"></a> |
| <span class="sourceLineNo">4227</span> }<a name="line.4227"></a> |
| <span class="sourceLineNo">4228</span> return true;<a name="line.4228"></a> |
| <span class="sourceLineNo">4229</span> });<a name="line.4229"></a> |
| <span class="sourceLineNo">4230</span><a name="line.4230"></a> |
| <span class="sourceLineNo">4231</span> // See if the column families were consistent through the whole thing.<a name="line.4231"></a> |
| <span class="sourceLineNo">4232</span> // if they were then keep them. If they were not then pass a null.<a name="line.4232"></a> |
| <span class="sourceLineNo">4233</span> // null will be treated as unknown.<a name="line.4233"></a> |
| <span class="sourceLineNo">4234</span> // Total time taken might be involving Puts, Deletes, Increments and Appends.<a name="line.4234"></a> |
| <span class="sourceLineNo">4235</span> // Split the time for puts and deletes based on the total number of Puts, Deletes,<a name="line.4235"></a> |
| <span class="sourceLineNo">4236</span> // Increments and Appends.<a name="line.4236"></a> |
| <span class="sourceLineNo">4237</span> if (region.metricsRegion != null) {<a name="line.4237"></a> |
| <span class="sourceLineNo">4238</span> if (miniBatchOp.getNumOfPuts() > 0) {<a name="line.4238"></a> |
| <span class="sourceLineNo">4239</span> // There were some Puts in the batch.<a name="line.4239"></a> |
| <span class="sourceLineNo">4240</span> region.metricsRegion.updatePut();<a name="line.4240"></a> |
| <span class="sourceLineNo">4241</span> }<a name="line.4241"></a> |
| <span class="sourceLineNo">4242</span> if (miniBatchOp.getNumOfDeletes() > 0) {<a name="line.4242"></a> |
| <span class="sourceLineNo">4243</span> // There were some Deletes in the batch.<a name="line.4243"></a> |
| <span class="sourceLineNo">4244</span> region.metricsRegion.updateDelete();<a name="line.4244"></a> |
| <span class="sourceLineNo">4245</span> }<a name="line.4245"></a> |
| <span class="sourceLineNo">4246</span> if (miniBatchOp.getNumOfIncrements() > 0) {<a name="line.4246"></a> |
| <span class="sourceLineNo">4247</span> // There were some Increments in the batch.<a name="line.4247"></a> |
| <span class="sourceLineNo">4248</span> region.metricsRegion.updateIncrement();<a name="line.4248"></a> |
| <span class="sourceLineNo">4249</span> }<a name="line.4249"></a> |
| <span class="sourceLineNo">4250</span> if (miniBatchOp.getNumOfAppends() > 0) {<a name="line.4250"></a> |
| <span class="sourceLineNo">4251</span> // There were some Appends in the batch.<a name="line.4251"></a> |
| <span class="sourceLineNo">4252</span> region.metricsRegion.updateAppend();<a name="line.4252"></a> |
| <span class="sourceLineNo">4253</span> }<a name="line.4253"></a> |
| <span class="sourceLineNo">4254</span> }<a name="line.4254"></a> |
| <span class="sourceLineNo">4255</span> }<a name="line.4255"></a> |
| <span class="sourceLineNo">4256</span><a name="line.4256"></a> |
| <span class="sourceLineNo">4257</span> if (region.coprocessorHost != null) {<a name="line.4257"></a> |
| <span class="sourceLineNo">4258</span> // call the coprocessor hook to do any finalization steps after the put is done<a name="line.4258"></a> |
| <span class="sourceLineNo">4259</span> region.coprocessorHost.postBatchMutateIndispensably(<a name="line.4259"></a> |
| <span class="sourceLineNo">4260</span> miniBatchOp != null ? miniBatchOp : createMiniBatch(size(), 0), success);<a name="line.4260"></a> |
| <span class="sourceLineNo">4261</span> }<a name="line.4261"></a> |
| <span class="sourceLineNo">4262</span> }<a name="line.4262"></a> |
| <span class="sourceLineNo">4263</span><a name="line.4263"></a> |
| <span class="sourceLineNo">4264</span> /**<a name="line.4264"></a> |
| <span class="sourceLineNo">4265</span> * Runs prePut/preDelete/preIncrement/preAppend coprocessor hook for input mutation in a batch<a name="line.4265"></a> |
| <span class="sourceLineNo">4266</span> * @param metrics Array of 2 ints. index 0: count of puts, index 1: count of deletes, index 2:<a name="line.4266"></a> |
| <span class="sourceLineNo">4267</span> * count of increments and 3: count of appends<a name="line.4267"></a> |
| <span class="sourceLineNo">4268</span> */<a name="line.4268"></a> |
| <span class="sourceLineNo">4269</span> private void callPreMutateCPHook(int index, final WALEdit walEdit, final int[] metrics)<a name="line.4269"></a> |
| <span class="sourceLineNo">4270</span> throws IOException {<a name="line.4270"></a> |
| <span class="sourceLineNo">4271</span> Mutation m = getMutation(index);<a name="line.4271"></a> |
| <span class="sourceLineNo">4272</span> if (m instanceof Put) {<a name="line.4272"></a> |
| <span class="sourceLineNo">4273</span> if (region.coprocessorHost.prePut((Put) m, walEdit, m.getDurability())) {<a name="line.4273"></a> |
| <span class="sourceLineNo">4274</span> // pre hook says skip this Put<a name="line.4274"></a> |
| <span class="sourceLineNo">4275</span> // mark as success and skip in doMiniBatchMutation<a name="line.4275"></a> |
| <span class="sourceLineNo">4276</span> metrics[0]++;<a name="line.4276"></a> |
| <span class="sourceLineNo">4277</span> retCodeDetails[index] = OperationStatus.SUCCESS;<a name="line.4277"></a> |
| <span class="sourceLineNo">4278</span> }<a name="line.4278"></a> |
| <span class="sourceLineNo">4279</span> } else if (m instanceof Delete) {<a name="line.4279"></a> |
| <span class="sourceLineNo">4280</span> Delete curDel = (Delete) m;<a name="line.4280"></a> |
| <span class="sourceLineNo">4281</span> if (curDel.getFamilyCellMap().isEmpty()) {<a name="line.4281"></a> |
| <span class="sourceLineNo">4282</span> // handle deleting a row case<a name="line.4282"></a> |
| <span class="sourceLineNo">4283</span> // TODO: prepareDelete() has been called twice, before and after preDelete() CP hook.<a name="line.4283"></a> |
| <span class="sourceLineNo">4284</span> // Can this be avoided?<a name="line.4284"></a> |
| <span class="sourceLineNo">4285</span> region.prepareDelete(curDel);<a name="line.4285"></a> |
| <span class="sourceLineNo">4286</span> }<a name="line.4286"></a> |
| <span class="sourceLineNo">4287</span> if (region.coprocessorHost.preDelete(curDel, walEdit, m.getDurability())) {<a name="line.4287"></a> |
| <span class="sourceLineNo">4288</span> // pre hook says skip this Delete<a name="line.4288"></a> |
| <span class="sourceLineNo">4289</span> // mark as success and skip in doMiniBatchMutation<a name="line.4289"></a> |
| <span class="sourceLineNo">4290</span> metrics[1]++;<a name="line.4290"></a> |
| <span class="sourceLineNo">4291</span> retCodeDetails[index] = OperationStatus.SUCCESS;<a name="line.4291"></a> |
| <span class="sourceLineNo">4292</span> }<a name="line.4292"></a> |
| <span class="sourceLineNo">4293</span> } else if (m instanceof Increment) {<a name="line.4293"></a> |
| <span class="sourceLineNo">4294</span> Increment increment = (Increment) m;<a name="line.4294"></a> |
| <span class="sourceLineNo">4295</span> Result result = region.coprocessorHost.preIncrement(increment);<a name="line.4295"></a> |
| <span class="sourceLineNo">4296</span> if (result != null) {<a name="line.4296"></a> |
| <span class="sourceLineNo">4297</span> // pre hook says skip this Increment<a name="line.4297"></a> |
| <span class="sourceLineNo">4298</span> // mark as success and skip in doMiniBatchMutation<a name="line.4298"></a> |
| <span class="sourceLineNo">4299</span> metrics[2]++;<a name="line.4299"></a> |
| <span class="sourceLineNo">4300</span> retCodeDetails[index] = new OperationStatus(OperationStatusCode.SUCCESS, result);<a name="line.4300"></a> |
| <span class="sourceLineNo">4301</span> }<a name="line.4301"></a> |
| <span class="sourceLineNo">4302</span> } else if (m instanceof Append) {<a name="line.4302"></a> |
| <span class="sourceLineNo">4303</span> Append append = (Append) m;<a name="line.4303"></a> |
| <span class="sourceLineNo">4304</span> Result result = region.coprocessorHost.preAppend(append);<a name="line.4304"></a> |
| <span class="sourceLineNo">4305</span> if (result != null) {<a name="line.4305"></a> |
| <span class="sourceLineNo">4306</span> // pre hook says skip this Append<a name="line.4306"></a> |
| <span class="sourceLineNo">4307</span> // mark as success and skip in doMiniBatchMutation<a name="line.4307"></a> |
| <span class="sourceLineNo">4308</span> metrics[3]++;<a name="line.4308"></a> |
| <span class="sourceLineNo">4309</span> retCodeDetails[index] = new OperationStatus(OperationStatusCode.SUCCESS, result);<a name="line.4309"></a> |
| <span class="sourceLineNo">4310</span> }<a name="line.4310"></a> |
| <span class="sourceLineNo">4311</span> } else {<a name="line.4311"></a> |
| <span class="sourceLineNo">4312</span> String msg = "Put/Delete/Increment/Append mutations only supported in a batch";<a name="line.4312"></a> |
| <span class="sourceLineNo">4313</span> retCodeDetails[index] = new OperationStatus(OperationStatusCode.FAILURE, msg);<a name="line.4313"></a> |
| <span class="sourceLineNo">4314</span> if (isAtomic()) { // fail, atomic means all or none<a name="line.4314"></a> |
| <span class="sourceLineNo">4315</span> throw new IOException(msg);<a name="line.4315"></a> |
| <span class="sourceLineNo">4316</span> }<a name="line.4316"></a> |
| <span class="sourceLineNo">4317</span> }<a name="line.4317"></a> |
| <span class="sourceLineNo">4318</span> }<a name="line.4318"></a> |
| <span class="sourceLineNo">4319</span><a name="line.4319"></a> |
| <span class="sourceLineNo">4320</span> // TODO Support Increment/Append operations<a name="line.4320"></a> |
| <span class="sourceLineNo">4321</span> private void checkAndMergeCPMutations(final MiniBatchOperationInProgress<Mutation> miniBatchOp,<a name="line.4321"></a> |
| <span class="sourceLineNo">4322</span> final List<RowLock> acquiredRowLocks, final long timestamp) throws IOException {<a name="line.4322"></a> |
| <span class="sourceLineNo">4323</span> visitBatchOperations(true, nextIndexToProcess + miniBatchOp.size(), (int i) -> {<a name="line.4323"></a> |
| <span class="sourceLineNo">4324</span> // we pass (i - firstIndex) below since the call expects a relative index<a name="line.4324"></a> |
| <span class="sourceLineNo">4325</span> Mutation[] cpMutations = miniBatchOp.getOperationsFromCoprocessors(i - nextIndexToProcess);<a name="line.4325"></a> |
| <span class="sourceLineNo">4326</span> if (cpMutations == null) {<a name="line.4326"></a> |
| <span class="sourceLineNo">4327</span> return true;<a name="line.4327"></a> |
| <span class="sourceLineNo">4328</span> }<a name="line.4328"></a> |
| <span class="sourceLineNo">4329</span> // Else Coprocessor added more Mutations corresponding to the Mutation at this index.<a name="line.4329"></a> |
| <span class="sourceLineNo">4330</span> Mutation mutation = getMutation(i);<a name="line.4330"></a> |
| <span class="sourceLineNo">4331</span> for (Mutation cpMutation : cpMutations) {<a name="line.4331"></a> |
| <span class="sourceLineNo">4332</span> this.checkAndPrepareMutation(cpMutation, timestamp);<a name="line.4332"></a> |
| <span class="sourceLineNo">4333</span><a name="line.4333"></a> |
| <span class="sourceLineNo">4334</span> // Acquire row locks. If not, the whole batch will fail.<a name="line.4334"></a> |
| <span class="sourceLineNo">4335</span> acquiredRowLocks.add(region.getRowLockInternal(cpMutation.getRow(), true, null));<a name="line.4335"></a> |
| <span class="sourceLineNo">4336</span><a name="line.4336"></a> |
| <span class="sourceLineNo">4337</span> // Returned mutations from coprocessor correspond to the Mutation at index i. We can<a name="line.4337"></a> |
| <span class="sourceLineNo">4338</span> // directly add the cells from those mutations to the familyMaps of this mutation.<a name="line.4338"></a> |
| <span class="sourceLineNo">4339</span> Map<byte[], List<Cell>> cpFamilyMap = cpMutation.getFamilyCellMap();<a name="line.4339"></a> |
| <span class="sourceLineNo">4340</span> region.rewriteCellTags(cpFamilyMap, mutation);<a name="line.4340"></a> |
| <span class="sourceLineNo">4341</span> // will get added to the memStore later<a name="line.4341"></a> |
| <span class="sourceLineNo">4342</span> mergeFamilyMaps(familyCellMaps[i], cpFamilyMap);<a name="line.4342"></a> |
| <span class="sourceLineNo">4343</span><a name="line.4343"></a> |
| <span class="sourceLineNo">4344</span> // The durability of returned mutation is replaced by the corresponding mutation.<a name="line.4344"></a> |
| <span class="sourceLineNo">4345</span> // If the corresponding mutation contains the SKIP_WAL, we shouldn't count the<a name="line.4345"></a> |
| <span class="sourceLineNo">4346</span> // cells of returned mutation.<a name="line.4346"></a> |
| <span class="sourceLineNo">4347</span> if (region.getEffectiveDurability(mutation.getDurability()) != Durability.SKIP_WAL) {<a name="line.4347"></a> |
| <span class="sourceLineNo">4348</span> for (List<Cell> cells : cpFamilyMap.values()) {<a name="line.4348"></a> |
| <span class="sourceLineNo">4349</span> miniBatchOp.addCellCount(cells.size());<a name="line.4349"></a> |
| <span class="sourceLineNo">4350</span> }<a name="line.4350"></a> |
| <span class="sourceLineNo">4351</span> }<a name="line.4351"></a> |
| <span class="sourceLineNo">4352</span> }<a name="line.4352"></a> |
| <span class="sourceLineNo">4353</span> return true;<a name="line.4353"></a> |
| <span class="sourceLineNo">4354</span> });<a name="line.4354"></a> |
| <span class="sourceLineNo">4355</span> }<a name="line.4355"></a> |
| <span class="sourceLineNo">4356</span><a name="line.4356"></a> |
| <span class="sourceLineNo">4357</span> private void mergeFamilyMaps(Map<byte[], List<Cell>> familyMap,<a name="line.4357"></a> |
| <span class="sourceLineNo">4358</span> Map<byte[], List<Cell>> toBeMerged) {<a name="line.4358"></a> |
| <span class="sourceLineNo">4359</span> for (Map.Entry<byte[], List<Cell>> entry : toBeMerged.entrySet()) {<a name="line.4359"></a> |
| <span class="sourceLineNo">4360</span> List<Cell> cells = familyMap.get(entry.getKey());<a name="line.4360"></a> |
| <span class="sourceLineNo">4361</span> if (cells == null) {<a name="line.4361"></a> |
| <span class="sourceLineNo">4362</span> familyMap.put(entry.getKey(), entry.getValue());<a name="line.4362"></a> |
| <span class="sourceLineNo">4363</span> } else {<a name="line.4363"></a> |
| <span class="sourceLineNo">4364</span> cells.addAll(entry.getValue());<a name="line.4364"></a> |
| <span class="sourceLineNo">4365</span> }<a name="line.4365"></a> |
| <span class="sourceLineNo">4366</span> }<a name="line.4366"></a> |
| <span class="sourceLineNo">4367</span> }<a name="line.4367"></a> |
| <span class="sourceLineNo">4368</span> }<a name="line.4368"></a> |
| <span class="sourceLineNo">4369</span><a name="line.4369"></a> |
| <span class="sourceLineNo">4370</span> /**<a name="line.4370"></a> |
| <span class="sourceLineNo">4371</span> * Batch of mutations for replay. Base class is shared with {@link MutationBatchOperation} as most<a name="line.4371"></a> |
| <span class="sourceLineNo">4372</span> * of the logic is same.<a name="line.4372"></a> |
| <span class="sourceLineNo">4373</span> */<a name="line.4373"></a> |
| <span class="sourceLineNo">4374</span> private static final class ReplayBatchOperation extends BatchOperation<MutationReplay> {<a name="line.4374"></a> |
| <span class="sourceLineNo">4375</span><a name="line.4375"></a> |
| <span class="sourceLineNo">4376</span> private long origLogSeqNum = 0;<a name="line.4376"></a> |
| <span class="sourceLineNo">4377</span><a name="line.4377"></a> |
| <span class="sourceLineNo">4378</span> public ReplayBatchOperation(final HRegion region, MutationReplay[] operations,<a name="line.4378"></a> |
| <span class="sourceLineNo">4379</span> long origLogSeqNum) {<a name="line.4379"></a> |
| <span class="sourceLineNo">4380</span> super(region, operations);<a name="line.4380"></a> |
| <span class="sourceLineNo">4381</span> this.origLogSeqNum = origLogSeqNum;<a name="line.4381"></a> |
| <span class="sourceLineNo">4382</span> }<a name="line.4382"></a> |
| <span class="sourceLineNo">4383</span><a name="line.4383"></a> |
| <span class="sourceLineNo">4384</span> @Override<a name="line.4384"></a> |
| <span class="sourceLineNo">4385</span> public Mutation getMutation(int index) {<a name="line.4385"></a> |
| <span class="sourceLineNo">4386</span> return this.operations[index].mutation;<a name="line.4386"></a> |
| <span class="sourceLineNo">4387</span> }<a name="line.4387"></a> |
| <span class="sourceLineNo">4388</span><a name="line.4388"></a> |
| <span class="sourceLineNo">4389</span> @Override<a name="line.4389"></a> |
| <span class="sourceLineNo">4390</span> public long getNonceGroup(int index) {<a name="line.4390"></a> |
| <span class="sourceLineNo">4391</span> return this.operations[index].nonceGroup;<a name="line.4391"></a> |
| <span class="sourceLineNo">4392</span> }<a name="line.4392"></a> |
| <span class="sourceLineNo">4393</span><a name="line.4393"></a> |
| <span class="sourceLineNo">4394</span> @Override<a name="line.4394"></a> |
| <span class="sourceLineNo">4395</span> public long getNonce(int index) {<a name="line.4395"></a> |
| <span class="sourceLineNo">4396</span> return this.operations[index].nonce;<a name="line.4396"></a> |
| <span class="sourceLineNo">4397</span> }<a name="line.4397"></a> |
| <span class="sourceLineNo">4398</span><a name="line.4398"></a> |
| <span class="sourceLineNo">4399</span> @Override<a name="line.4399"></a> |
| <span class="sourceLineNo">4400</span> public Mutation[] getMutationsForCoprocs() {<a name="line.4400"></a> |
| <span class="sourceLineNo">4401</span> return null;<a name="line.4401"></a> |
| <span class="sourceLineNo">4402</span> }<a name="line.4402"></a> |
| <span class="sourceLineNo">4403</span><a name="line.4403"></a> |
| <span class="sourceLineNo">4404</span> @Override<a name="line.4404"></a> |
| <span class="sourceLineNo">4405</span> public boolean isInReplay() {<a name="line.4405"></a> |
| <span class="sourceLineNo">4406</span> return true;<a name="line.4406"></a> |
| <span class="sourceLineNo">4407</span> }<a name="line.4407"></a> |
| <span class="sourceLineNo">4408</span><a name="line.4408"></a> |
| <span class="sourceLineNo">4409</span> @Override<a name="line.4409"></a> |
| <span class="sourceLineNo">4410</span> public long getOrigLogSeqNum() {<a name="line.4410"></a> |
| <span class="sourceLineNo">4411</span> return this.origLogSeqNum;<a name="line.4411"></a> |
| <span class="sourceLineNo">4412</span> }<a name="line.4412"></a> |
| <span class="sourceLineNo">4413</span><a name="line.4413"></a> |
| <span class="sourceLineNo">4414</span> @Override<a name="line.4414"></a> |
| <span class="sourceLineNo">4415</span> public void startRegionOperation() throws IOException {<a name="line.4415"></a> |
| <span class="sourceLineNo">4416</span> region.startRegionOperation(Operation.REPLAY_BATCH_MUTATE);<a name="line.4416"></a> |
| <span class="sourceLineNo">4417</span> }<a name="line.4417"></a> |
| <span class="sourceLineNo">4418</span><a name="line.4418"></a> |
| <span class="sourceLineNo">4419</span> @Override<a name="line.4419"></a> |
| <span class="sourceLineNo">4420</span> public void closeRegionOperation() throws IOException {<a name="line.4420"></a> |
| <span class="sourceLineNo">4421</span> region.closeRegionOperation(Operation.REPLAY_BATCH_MUTATE);<a name="line.4421"></a> |
| <span class="sourceLineNo">4422</span> }<a name="line.4422"></a> |
| <span class="sourceLineNo">4423</span><a name="line.4423"></a> |
| <span class="sourceLineNo">4424</span> /**<a name="line.4424"></a> |
| <span class="sourceLineNo">4425</span> * During replay, there could exist column families which are removed between region server<a name="line.4425"></a> |
| <span class="sourceLineNo">4426</span> * failure and replay<a name="line.4426"></a> |
| <span class="sourceLineNo">4427</span> */<a name="line.4427"></a> |
| <span class="sourceLineNo">4428</span> @Override<a name="line.4428"></a> |
| <span class="sourceLineNo">4429</span> protected void checkAndPreparePut(Put p) throws IOException {<a name="line.4429"></a> |
| <span class="sourceLineNo">4430</span> Map<byte[], List<Cell>> familyCellMap = p.getFamilyCellMap();<a name="line.4430"></a> |
| <span class="sourceLineNo">4431</span> List<byte[]> nonExistentList = null;<a name="line.4431"></a> |
| <span class="sourceLineNo">4432</span> for (byte[] family : familyCellMap.keySet()) {<a name="line.4432"></a> |
| <span class="sourceLineNo">4433</span> if (!region.htableDescriptor.hasColumnFamily(family)) {<a name="line.4433"></a> |
| <span class="sourceLineNo">4434</span> if (nonExistentList == null) {<a name="line.4434"></a> |
| <span class="sourceLineNo">4435</span> nonExistentList = new ArrayList<>();<a name="line.4435"></a> |
| <span class="sourceLineNo">4436</span> }<a name="line.4436"></a> |
| <span class="sourceLineNo">4437</span> nonExistentList.add(family);<a name="line.4437"></a> |
| <span class="sourceLineNo">4438</span> }<a name="line.4438"></a> |
| <span class="sourceLineNo">4439</span> }<a name="line.4439"></a> |
| <span class="sourceLineNo">4440</span> if (nonExistentList != null) {<a name="line.4440"></a> |
| <span class="sourceLineNo">4441</span> for (byte[] family : nonExistentList) {<a name="line.4441"></a> |
| <span class="sourceLineNo">4442</span> // Perhaps schema was changed between crash and replay<a name="line.4442"></a> |
| <span class="sourceLineNo">4443</span> LOG.info("No family for {} omit from reply in region {}.", Bytes.toString(family), this);<a name="line.4443"></a> |
| <span class="sourceLineNo">4444</span> familyCellMap.remove(family);<a name="line.4444"></a> |
| <span class="sourceLineNo">4445</span> }<a name="line.4445"></a> |
| <span class="sourceLineNo">4446</span> }<a name="line.4446"></a> |
| <span class="sourceLineNo">4447</span> }<a name="line.4447"></a> |
| <span class="sourceLineNo">4448</span><a name="line.4448"></a> |
| <span class="sourceLineNo">4449</span> @Override<a name="line.4449"></a> |
| <span class="sourceLineNo">4450</span> public void checkAndPrepare() throws IOException {<a name="line.4450"></a> |
| <span class="sourceLineNo">4451</span> long now = EnvironmentEdgeManager.currentTime();<a name="line.4451"></a> |
| <span class="sourceLineNo">4452</span> visitBatchOperations(true, this.size(), (int index) -> {<a name="line.4452"></a> |
| <span class="sourceLineNo">4453</span> checkAndPrepareMutation(index, now);<a name="line.4453"></a> |
| <span class="sourceLineNo">4454</span> return true;<a name="line.4454"></a> |
| <span class="sourceLineNo">4455</span> });<a name="line.4455"></a> |
| <span class="sourceLineNo">4456</span> }<a name="line.4456"></a> |
| <span class="sourceLineNo">4457</span><a name="line.4457"></a> |
| <span class="sourceLineNo">4458</span> @Override<a name="line.4458"></a> |
| <span class="sourceLineNo">4459</span> public void prepareMiniBatchOperations(MiniBatchOperationInProgress<Mutation> miniBatchOp,<a name="line.4459"></a> |
| <span class="sourceLineNo">4460</span> long timestamp, final List<RowLock> acquiredRowLocks) throws IOException {<a name="line.4460"></a> |
| <span class="sourceLineNo">4461</span> visitBatchOperations(true, miniBatchOp.getLastIndexExclusive(), (int index) -> {<a name="line.4461"></a> |
| <span class="sourceLineNo">4462</span> // update cell count<a name="line.4462"></a> |
| <span class="sourceLineNo">4463</span> for (List<Cell> cells : getMutation(index).getFamilyCellMap().values()) {<a name="line.4463"></a> |
| <span class="sourceLineNo">4464</span> miniBatchOp.addCellCount(cells.size());<a name="line.4464"></a> |
| <span class="sourceLineNo">4465</span> }<a name="line.4465"></a> |
| <span class="sourceLineNo">4466</span> return true;<a name="line.4466"></a> |
| <span class="sourceLineNo">4467</span> });<a name="line.4467"></a> |
| <span class="sourceLineNo">4468</span> }<a name="line.4468"></a> |
| <span class="sourceLineNo">4469</span><a name="line.4469"></a> |
| <span class="sourceLineNo">4470</span> @Override<a name="line.4470"></a> |
| <span class="sourceLineNo">4471</span> public WriteEntry writeMiniBatchOperationsToMemStore(<a name="line.4471"></a> |
| <span class="sourceLineNo">4472</span> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final WriteEntry writeEntry)<a name="line.4472"></a> |
| <span class="sourceLineNo">4473</span> throws IOException {<a name="line.4473"></a> |
| <span class="sourceLineNo">4474</span> super.writeMiniBatchOperationsToMemStore(miniBatchOp, getOrigLogSeqNum());<a name="line.4474"></a> |
| <span class="sourceLineNo">4475</span> return writeEntry;<a name="line.4475"></a> |
| <span class="sourceLineNo">4476</span> }<a name="line.4476"></a> |
| <span class="sourceLineNo">4477</span><a name="line.4477"></a> |
| <span class="sourceLineNo">4478</span> @Override<a name="line.4478"></a> |
| <span class="sourceLineNo">4479</span> public void completeMiniBatchOperations(<a name="line.4479"></a> |
| <span class="sourceLineNo">4480</span> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final WriteEntry writeEntry)<a name="line.4480"></a> |
| <span class="sourceLineNo">4481</span> throws IOException {<a name="line.4481"></a> |
| <span class="sourceLineNo">4482</span> super.completeMiniBatchOperations(miniBatchOp, writeEntry);<a name="line.4482"></a> |
| <span class="sourceLineNo">4483</span> region.mvcc.advanceTo(getOrigLogSeqNum());<a name="line.4483"></a> |
| <span class="sourceLineNo">4484</span> }<a name="line.4484"></a> |
| <span class="sourceLineNo">4485</span> }<a name="line.4485"></a> |
| <span class="sourceLineNo">4486</span><a name="line.4486"></a> |
| <span class="sourceLineNo">4487</span> private OperationStatus[] batchMutate(Mutation[] mutations, boolean atomic, long nonceGroup,<a name="line.4487"></a> |
| <span class="sourceLineNo">4488</span> long nonce) throws IOException {<a name="line.4488"></a> |
| <span class="sourceLineNo">4489</span> // As it stands, this is used for 3 things<a name="line.4489"></a> |
| <span class="sourceLineNo">4490</span> // * batchMutate with single mutation - put/delete/increment/append, separate or from<a name="line.4490"></a> |
| <span class="sourceLineNo">4491</span> // checkAndMutate.<a name="line.4491"></a> |
| <span class="sourceLineNo">4492</span> // * coprocessor calls (see ex. BulkDeleteEndpoint).<a name="line.4492"></a> |
| <span class="sourceLineNo">4493</span> // So nonces are not really ever used by HBase. They could be by coprocs, and checkAnd...<a name="line.4493"></a> |
| <span class="sourceLineNo">4494</span> return batchMutate(new MutationBatchOperation(this, mutations, atomic, nonceGroup, nonce));<a name="line.4494"></a> |
| <span class="sourceLineNo">4495</span> }<a name="line.4495"></a> |
| <span class="sourceLineNo">4496</span><a name="line.4496"></a> |
| <span class="sourceLineNo">4497</span> @Override<a name="line.4497"></a> |
| <span class="sourceLineNo">4498</span> public OperationStatus[] batchMutate(Mutation[] mutations) throws IOException {<a name="line.4498"></a> |
| <span class="sourceLineNo">4499</span> // If the mutations has any Increment/Append operations, we need to do batchMutate atomically<a name="line.4499"></a> |
| <span class="sourceLineNo">4500</span> boolean atomic =<a name="line.4500"></a> |
| <span class="sourceLineNo">4501</span> Arrays.stream(mutations).anyMatch(m -> m instanceof Increment || m instanceof Append);<a name="line.4501"></a> |
| <span class="sourceLineNo">4502</span> return batchMutate(mutations, atomic);<a name="line.4502"></a> |
| <span class="sourceLineNo">4503</span> }<a name="line.4503"></a> |
| <span class="sourceLineNo">4504</span><a name="line.4504"></a> |
| <span class="sourceLineNo">4505</span> OperationStatus[] batchMutate(Mutation[] mutations, boolean atomic) throws IOException {<a name="line.4505"></a> |
| <span class="sourceLineNo">4506</span> return batchMutate(mutations, atomic, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.4506"></a> |
| <span class="sourceLineNo">4507</span> }<a name="line.4507"></a> |
| <span class="sourceLineNo">4508</span><a name="line.4508"></a> |
| <span class="sourceLineNo">4509</span> public OperationStatus[] batchReplay(MutationReplay[] mutations, long replaySeqId)<a name="line.4509"></a> |
| <span class="sourceLineNo">4510</span> throws IOException {<a name="line.4510"></a> |
| <span class="sourceLineNo">4511</span> if (!RegionReplicaUtil.isDefaultReplica(getRegionInfo())<a name="line.4511"></a> |
| <span class="sourceLineNo">4512</span> && replaySeqId < lastReplayedOpenRegionSeqId) {<a name="line.4512"></a> |
| <span class="sourceLineNo">4513</span> // if it is a secondary replica we should ignore these entries silently<a name="line.4513"></a> |
| <span class="sourceLineNo">4514</span> // since they are coming out of order<a name="line.4514"></a> |
| <span class="sourceLineNo">4515</span> if (LOG.isTraceEnabled()) {<a name="line.4515"></a> |
| <span class="sourceLineNo">4516</span> LOG.trace(getRegionInfo().getEncodedName() + " : "<a name="line.4516"></a> |
| <span class="sourceLineNo">4517</span> + "Skipping " + mutations.length + " mutations with replaySeqId=" + replaySeqId<a name="line.4517"></a> |
| <span class="sourceLineNo">4518</span> + " which is < than lastReplayedOpenRegionSeqId=" + lastReplayedOpenRegionSeqId);<a name="line.4518"></a> |
| <span class="sourceLineNo">4519</span> for (MutationReplay mut : mutations) {<a name="line.4519"></a> |
| <span class="sourceLineNo">4520</span> LOG.trace(getRegionInfo().getEncodedName() + " : Skipping : " + mut.mutation);<a name="line.4520"></a> |
| <span class="sourceLineNo">4521</span> }<a name="line.4521"></a> |
| <span class="sourceLineNo">4522</span> }<a name="line.4522"></a> |
| <span class="sourceLineNo">4523</span><a name="line.4523"></a> |
| <span class="sourceLineNo">4524</span> OperationStatus[] statuses = new OperationStatus[mutations.length];<a name="line.4524"></a> |
| <span class="sourceLineNo">4525</span> for (int i = 0; i < statuses.length; i++) {<a name="line.4525"></a> |
| <span class="sourceLineNo">4526</span> statuses[i] = OperationStatus.SUCCESS;<a name="line.4526"></a> |
| <span class="sourceLineNo">4527</span> }<a name="line.4527"></a> |
| <span class="sourceLineNo">4528</span> return statuses;<a name="line.4528"></a> |
| <span class="sourceLineNo">4529</span> }<a name="line.4529"></a> |
| <span class="sourceLineNo">4530</span> return batchMutate(new ReplayBatchOperation(this, mutations, replaySeqId));<a name="line.4530"></a> |
| <span class="sourceLineNo">4531</span> }<a name="line.4531"></a> |
| <span class="sourceLineNo">4532</span><a name="line.4532"></a> |
| <span class="sourceLineNo">4533</span> /**<a name="line.4533"></a> |
| <span class="sourceLineNo">4534</span> * Perform a batch of mutations.<a name="line.4534"></a> |
| <span class="sourceLineNo">4535</span> * <p/><a name="line.4535"></a> |
| <span class="sourceLineNo">4536</span> * Operations in a batch are stored with highest durability specified of for all operations in a<a name="line.4536"></a> |
| <span class="sourceLineNo">4537</span> * batch, except for {@link Durability#SKIP_WAL}.<a name="line.4537"></a> |
| <span class="sourceLineNo">4538</span> * <p/><a name="line.4538"></a> |
| <span class="sourceLineNo">4539</span> * This function is called from {@link #batchReplay(WALSplitUtil.MutationReplay[], long)} with<a name="line.4539"></a> |
| <span class="sourceLineNo">4540</span> * {@link ReplayBatchOperation} instance and {@link #batchMutate(Mutation[])} with<a name="line.4540"></a> |
| <span class="sourceLineNo">4541</span> * {@link MutationBatchOperation} instance as an argument. As the processing of replay batch and<a name="line.4541"></a> |
| <span class="sourceLineNo">4542</span> * mutation batch is very similar, lot of code is shared by providing generic methods in base<a name="line.4542"></a> |
| <span class="sourceLineNo">4543</span> * class {@link BatchOperation}. The logic for this method and<a name="line.4543"></a> |
| <span class="sourceLineNo">4544</span> * {@link #doMiniBatchMutate(BatchOperation)} is implemented using methods in base class which are<a name="line.4544"></a> |
| <span class="sourceLineNo">4545</span> * overridden by derived classes to implement special behavior.<a name="line.4545"></a> |
| <span class="sourceLineNo">4546</span> * @param batchOp contains the list of mutations<a name="line.4546"></a> |
| <span class="sourceLineNo">4547</span> * @return an array of OperationStatus which internally contains the OperationStatusCode and the<a name="line.4547"></a> |
| <span class="sourceLineNo">4548</span> * exceptionMessage if any.<a name="line.4548"></a> |
| <span class="sourceLineNo">4549</span> * @throws IOException if an IO problem is encountered<a name="line.4549"></a> |
| <span class="sourceLineNo">4550</span> */<a name="line.4550"></a> |
| <span class="sourceLineNo">4551</span> private OperationStatus[] batchMutate(BatchOperation<?> batchOp) throws IOException {<a name="line.4551"></a> |
| <span class="sourceLineNo">4552</span> boolean initialized = false;<a name="line.4552"></a> |
| <span class="sourceLineNo">4553</span> batchOp.startRegionOperation();<a name="line.4553"></a> |
| <span class="sourceLineNo">4554</span> try {<a name="line.4554"></a> |
| <span class="sourceLineNo">4555</span> while (!batchOp.isDone()) {<a name="line.4555"></a> |
| <span class="sourceLineNo">4556</span> if (!batchOp.isInReplay()) {<a name="line.4556"></a> |
| <span class="sourceLineNo">4557</span> checkReadOnly();<a name="line.4557"></a> |
| <span class="sourceLineNo">4558</span> }<a name="line.4558"></a> |
| <span class="sourceLineNo">4559</span> checkResources();<a name="line.4559"></a> |
| <span class="sourceLineNo">4560</span><a name="line.4560"></a> |
| <span class="sourceLineNo">4561</span> if (!initialized) {<a name="line.4561"></a> |
| <span class="sourceLineNo">4562</span> this.writeRequestsCount.add(batchOp.size());<a name="line.4562"></a> |
| <span class="sourceLineNo">4563</span> // validate and prepare batch for write, for MutationBatchOperation it also calls CP<a name="line.4563"></a> |
| <span class="sourceLineNo">4564</span> // prePut()/preDelete()/preIncrement()/preAppend() hooks<a name="line.4564"></a> |
| <span class="sourceLineNo">4565</span> batchOp.checkAndPrepare();<a name="line.4565"></a> |
| <span class="sourceLineNo">4566</span> initialized = true;<a name="line.4566"></a> |
| <span class="sourceLineNo">4567</span> }<a name="line.4567"></a> |
| <span class="sourceLineNo">4568</span> doMiniBatchMutate(batchOp);<a name="line.4568"></a> |
| <span class="sourceLineNo">4569</span> requestFlushIfNeeded();<a name="line.4569"></a> |
| <span class="sourceLineNo">4570</span> }<a name="line.4570"></a> |
| <span class="sourceLineNo">4571</span> } finally {<a name="line.4571"></a> |
| <span class="sourceLineNo">4572</span> if (rsServices != null && rsServices.getMetrics() != null) {<a name="line.4572"></a> |
| <span class="sourceLineNo">4573</span> rsServices.getMetrics().updateWriteQueryMeter(this.htableDescriptor.<a name="line.4573"></a> |
| <span class="sourceLineNo">4574</span> getTableName(), batchOp.size());<a name="line.4574"></a> |
| <span class="sourceLineNo">4575</span> }<a name="line.4575"></a> |
| <span class="sourceLineNo">4576</span> batchOp.closeRegionOperation();<a name="line.4576"></a> |
| <span class="sourceLineNo">4577</span> }<a name="line.4577"></a> |
| <span class="sourceLineNo">4578</span> return batchOp.retCodeDetails;<a name="line.4578"></a> |
| <span class="sourceLineNo">4579</span> }<a name="line.4579"></a> |
| <span class="sourceLineNo">4580</span><a name="line.4580"></a> |
| <span class="sourceLineNo">4581</span> /**<a name="line.4581"></a> |
| <span class="sourceLineNo">4582</span> * Called to do a piece of the batch that came in to {@link #batchMutate(Mutation[])}<a name="line.4582"></a> |
| <span class="sourceLineNo">4583</span> * In here we also handle replay of edits on region recover. Also gets change in size brought<a name="line.4583"></a> |
| <span class="sourceLineNo">4584</span> * about by applying {@code batchOp}.<a name="line.4584"></a> |
| <span class="sourceLineNo">4585</span> */<a name="line.4585"></a> |
| <span class="sourceLineNo">4586</span> private void doMiniBatchMutate(BatchOperation<?> batchOp) throws IOException {<a name="line.4586"></a> |
| <span class="sourceLineNo">4587</span> boolean success = false;<a name="line.4587"></a> |
| <span class="sourceLineNo">4588</span> WALEdit walEdit = null;<a name="line.4588"></a> |
| <span class="sourceLineNo">4589</span> WriteEntry writeEntry = null;<a name="line.4589"></a> |
| <span class="sourceLineNo">4590</span> boolean locked = false;<a name="line.4590"></a> |
| <span class="sourceLineNo">4591</span> // We try to set up a batch in the range [batchOp.nextIndexToProcess,lastIndexExclusive)<a name="line.4591"></a> |
| <span class="sourceLineNo">4592</span> MiniBatchOperationInProgress<Mutation> miniBatchOp = null;<a name="line.4592"></a> |
| <span class="sourceLineNo">4593</span> /** Keep track of the locks we hold so we can release them in finally clause */<a name="line.4593"></a> |
| <span class="sourceLineNo">4594</span> List<RowLock> acquiredRowLocks = Lists.newArrayListWithCapacity(batchOp.size());<a name="line.4594"></a> |
| <span class="sourceLineNo">4595</span><a name="line.4595"></a> |
| <span class="sourceLineNo">4596</span> // Check for thread interrupt status in case we have been signaled from<a name="line.4596"></a> |
| <span class="sourceLineNo">4597</span> // #interruptRegionOperation.<a name="line.4597"></a> |
| <span class="sourceLineNo">4598</span> checkInterrupt();<a name="line.4598"></a> |
| <span class="sourceLineNo">4599</span><a name="line.4599"></a> |
| <span class="sourceLineNo">4600</span> try {<a name="line.4600"></a> |
| <span class="sourceLineNo">4601</span> // STEP 1. Try to acquire as many locks as we can and build mini-batch of operations with<a name="line.4601"></a> |
| <span class="sourceLineNo">4602</span> // locked rows<a name="line.4602"></a> |
| <span class="sourceLineNo">4603</span> miniBatchOp = batchOp.lockRowsAndBuildMiniBatch(acquiredRowLocks);<a name="line.4603"></a> |
| <span class="sourceLineNo">4604</span><a name="line.4604"></a> |
| <span class="sourceLineNo">4605</span> // We've now grabbed as many mutations off the list as we can<a name="line.4605"></a> |
| <span class="sourceLineNo">4606</span> // Ensure we acquire at least one.<a name="line.4606"></a> |
| <span class="sourceLineNo">4607</span> if (miniBatchOp.getReadyToWriteCount() <= 0) {<a name="line.4607"></a> |
| <span class="sourceLineNo">4608</span> // Nothing to put/delete/increment/append -- an exception in the above such as<a name="line.4608"></a> |
| <span class="sourceLineNo">4609</span> // NoSuchColumnFamily?<a name="line.4609"></a> |
| <span class="sourceLineNo">4610</span> return;<a name="line.4610"></a> |
| <span class="sourceLineNo">4611</span> }<a name="line.4611"></a> |
| <span class="sourceLineNo">4612</span><a name="line.4612"></a> |
| <span class="sourceLineNo">4613</span> // Check for thread interrupt status in case we have been signaled from<a name="line.4613"></a> |
| <span class="sourceLineNo">4614</span> // #interruptRegionOperation. Do it before we take the lock and disable interrupts for<a name="line.4614"></a> |
| <span class="sourceLineNo">4615</span> // the WAL append.<a name="line.4615"></a> |
| <span class="sourceLineNo">4616</span> checkInterrupt();<a name="line.4616"></a> |
| <span class="sourceLineNo">4617</span><a name="line.4617"></a> |
| <span class="sourceLineNo">4618</span> lock(this.updatesLock.readLock(), miniBatchOp.getReadyToWriteCount());<a name="line.4618"></a> |
| <span class="sourceLineNo">4619</span> locked = true;<a name="line.4619"></a> |
| <span class="sourceLineNo">4620</span><a name="line.4620"></a> |
| <span class="sourceLineNo">4621</span> // From this point until memstore update this operation should not be interrupted.<a name="line.4621"></a> |
| <span class="sourceLineNo">4622</span> disableInterrupts();<a name="line.4622"></a> |
| <span class="sourceLineNo">4623</span><a name="line.4623"></a> |
| <span class="sourceLineNo">4624</span> // STEP 2. Update mini batch of all operations in progress with LATEST_TIMESTAMP timestamp<a name="line.4624"></a> |
| <span class="sourceLineNo">4625</span> // We should record the timestamp only after we have acquired the rowLock,<a name="line.4625"></a> |
| <span class="sourceLineNo">4626</span> // otherwise, newer puts/deletes/increment/append are not guaranteed to have a newer<a name="line.4626"></a> |
| <span class="sourceLineNo">4627</span> // timestamp<a name="line.4627"></a> |
| <span class="sourceLineNo">4628</span><a name="line.4628"></a> |
| <span class="sourceLineNo">4629</span> long now = EnvironmentEdgeManager.currentTime();<a name="line.4629"></a> |
| <span class="sourceLineNo">4630</span> batchOp.prepareMiniBatchOperations(miniBatchOp, now, acquiredRowLocks);<a name="line.4630"></a> |
| <span class="sourceLineNo">4631</span><a name="line.4631"></a> |
| <span class="sourceLineNo">4632</span> // STEP 3. Build WAL edit<a name="line.4632"></a> |
| <span class="sourceLineNo">4633</span><a name="line.4633"></a> |
| <span class="sourceLineNo">4634</span> List<Pair<NonceKey, WALEdit>> walEdits = batchOp.buildWALEdits(miniBatchOp);<a name="line.4634"></a> |
| <span class="sourceLineNo">4635</span><a name="line.4635"></a> |
| <span class="sourceLineNo">4636</span> // STEP 4. Append the WALEdits to WAL and sync.<a name="line.4636"></a> |
| <span class="sourceLineNo">4637</span><a name="line.4637"></a> |
| <span class="sourceLineNo">4638</span> for(Iterator<Pair<NonceKey, WALEdit>> it = walEdits.iterator(); it.hasNext();) {<a name="line.4638"></a> |
| <span class="sourceLineNo">4639</span> Pair<NonceKey, WALEdit> nonceKeyWALEditPair = it.next();<a name="line.4639"></a> |
| <span class="sourceLineNo">4640</span> walEdit = nonceKeyWALEditPair.getSecond();<a name="line.4640"></a> |
| <span class="sourceLineNo">4641</span> NonceKey nonceKey = nonceKeyWALEditPair.getFirst();<a name="line.4641"></a> |
| <span class="sourceLineNo">4642</span><a name="line.4642"></a> |
| <span class="sourceLineNo">4643</span> if (walEdit != null && !walEdit.isEmpty()) {<a name="line.4643"></a> |
| <span class="sourceLineNo">4644</span> writeEntry = doWALAppend(walEdit, batchOp.durability, batchOp.getClusterIds(), now,<a name="line.4644"></a> |
| <span class="sourceLineNo">4645</span> nonceKey.getNonceGroup(), nonceKey.getNonce(), batchOp.getOrigLogSeqNum());<a name="line.4645"></a> |
| <span class="sourceLineNo">4646</span> }<a name="line.4646"></a> |
| <span class="sourceLineNo">4647</span><a name="line.4647"></a> |
| <span class="sourceLineNo">4648</span> // Complete mvcc for all but last writeEntry (for replay case)<a name="line.4648"></a> |
| <span class="sourceLineNo">4649</span> if (it.hasNext() && writeEntry != null) {<a name="line.4649"></a> |
| <span class="sourceLineNo">4650</span> mvcc.complete(writeEntry);<a name="line.4650"></a> |
| <span class="sourceLineNo">4651</span> writeEntry = null;<a name="line.4651"></a> |
| <span class="sourceLineNo">4652</span> }<a name="line.4652"></a> |
| <span class="sourceLineNo">4653</span> }<a name="line.4653"></a> |
| <span class="sourceLineNo">4654</span><a name="line.4654"></a> |
| <span class="sourceLineNo">4655</span> // STEP 5. Write back to memStore<a name="line.4655"></a> |
| <span class="sourceLineNo">4656</span> // NOTE: writeEntry can be null here<a name="line.4656"></a> |
| <span class="sourceLineNo">4657</span> writeEntry = batchOp.writeMiniBatchOperationsToMemStore(miniBatchOp, writeEntry);<a name="line.4657"></a> |
| <span class="sourceLineNo">4658</span><a name="line.4658"></a> |
| <span class="sourceLineNo">4659</span> // STEP 6. Complete MiniBatchOperations: If required calls postBatchMutate() CP hook and<a name="line.4659"></a> |
| <span class="sourceLineNo">4660</span> // complete mvcc for last writeEntry<a name="line.4660"></a> |
| <span class="sourceLineNo">4661</span> batchOp.completeMiniBatchOperations(miniBatchOp, writeEntry);<a name="line.4661"></a> |
| <span class="sourceLineNo">4662</span> writeEntry = null;<a name="line.4662"></a> |
| <span class="sourceLineNo">4663</span> success = true;<a name="line.4663"></a> |
| <span class="sourceLineNo">4664</span> } finally {<a name="line.4664"></a> |
| <span class="sourceLineNo">4665</span> // Call complete rather than completeAndWait because we probably had error if walKey != null<a name="line.4665"></a> |
| <span class="sourceLineNo">4666</span> if (writeEntry != null) mvcc.complete(writeEntry);<a name="line.4666"></a> |
| <span class="sourceLineNo">4667</span><a name="line.4667"></a> |
| <span class="sourceLineNo">4668</span> if (locked) {<a name="line.4668"></a> |
| <span class="sourceLineNo">4669</span> this.updatesLock.readLock().unlock();<a name="line.4669"></a> |
| <span class="sourceLineNo">4670</span> }<a name="line.4670"></a> |
| <span class="sourceLineNo">4671</span> releaseRowLocks(acquiredRowLocks);<a name="line.4671"></a> |
| <span class="sourceLineNo">4672</span><a name="line.4672"></a> |
| <span class="sourceLineNo">4673</span> enableInterrupts();<a name="line.4673"></a> |
| <span class="sourceLineNo">4674</span><a name="line.4674"></a> |
| <span class="sourceLineNo">4675</span> final int finalLastIndexExclusive =<a name="line.4675"></a> |
| <span class="sourceLineNo">4676</span> miniBatchOp != null ? miniBatchOp.getLastIndexExclusive() : batchOp.size();<a name="line.4676"></a> |
| <span class="sourceLineNo">4677</span> final boolean finalSuccess = success;<a name="line.4677"></a> |
| <span class="sourceLineNo">4678</span> batchOp.visitBatchOperations(true, finalLastIndexExclusive,<a name="line.4678"></a> |
| <span class="sourceLineNo">4679</span> (int i) -> {<a name="line.4679"></a> |
| <span class="sourceLineNo">4680</span> Mutation mutation = batchOp.getMutation(i);<a name="line.4680"></a> |
| <span class="sourceLineNo">4681</span> if (mutation instanceof Increment || mutation instanceof Append) {<a name="line.4681"></a> |
| <span class="sourceLineNo">4682</span> if (finalSuccess) {<a name="line.4682"></a> |
| <span class="sourceLineNo">4683</span> batchOp.retCodeDetails[i] = new OperationStatus(OperationStatusCode.SUCCESS,<a name="line.4683"></a> |
| <span class="sourceLineNo">4684</span> batchOp.results[i]);<a name="line.4684"></a> |
| <span class="sourceLineNo">4685</span> } else {<a name="line.4685"></a> |
| <span class="sourceLineNo">4686</span> batchOp.retCodeDetails[i] = OperationStatus.FAILURE;<a name="line.4686"></a> |
| <span class="sourceLineNo">4687</span> }<a name="line.4687"></a> |
| <span class="sourceLineNo">4688</span> } else {<a name="line.4688"></a> |
| <span class="sourceLineNo">4689</span> batchOp.retCodeDetails[i] =<a name="line.4689"></a> |
| <span class="sourceLineNo">4690</span> finalSuccess ? OperationStatus.SUCCESS : OperationStatus.FAILURE;<a name="line.4690"></a> |
| <span class="sourceLineNo">4691</span> }<a name="line.4691"></a> |
| <span class="sourceLineNo">4692</span> return true;<a name="line.4692"></a> |
| <span class="sourceLineNo">4693</span> });<a name="line.4693"></a> |
| <span class="sourceLineNo">4694</span><a name="line.4694"></a> |
| <span class="sourceLineNo">4695</span> batchOp.doPostOpCleanupForMiniBatch(miniBatchOp, walEdit, finalSuccess);<a name="line.4695"></a> |
| <span class="sourceLineNo">4696</span><a name="line.4696"></a> |
| <span class="sourceLineNo">4697</span> batchOp.nextIndexToProcess = finalLastIndexExclusive;<a name="line.4697"></a> |
| <span class="sourceLineNo">4698</span> }<a name="line.4698"></a> |
| <span class="sourceLineNo">4699</span> }<a name="line.4699"></a> |
| <span class="sourceLineNo">4700</span><a name="line.4700"></a> |
| <span class="sourceLineNo">4701</span> /**<a name="line.4701"></a> |
| <span class="sourceLineNo">4702</span> * Returns effective durability from the passed durability and<a name="line.4702"></a> |
| <span class="sourceLineNo">4703</span> * the table descriptor.<a name="line.4703"></a> |
| <span class="sourceLineNo">4704</span> */<a name="line.4704"></a> |
| <span class="sourceLineNo">4705</span> private Durability getEffectiveDurability(Durability d) {<a name="line.4705"></a> |
| <span class="sourceLineNo">4706</span> return d == Durability.USE_DEFAULT ? this.regionDurability : d;<a name="line.4706"></a> |
| <span class="sourceLineNo">4707</span> }<a name="line.4707"></a> |
| <span class="sourceLineNo">4708</span><a name="line.4708"></a> |
| <span class="sourceLineNo">4709</span> @Override<a name="line.4709"></a> |
| <span class="sourceLineNo">4710</span> @Deprecated<a name="line.4710"></a> |
| <span class="sourceLineNo">4711</span> public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareOperator op,<a name="line.4711"></a> |
| <span class="sourceLineNo">4712</span> ByteArrayComparable comparator, TimeRange timeRange, Mutation mutation) throws IOException {<a name="line.4712"></a> |
| <span class="sourceLineNo">4713</span> CheckAndMutate checkAndMutate;<a name="line.4713"></a> |
| <span class="sourceLineNo">4714</span> try {<a name="line.4714"></a> |
| <span class="sourceLineNo">4715</span> CheckAndMutate.Builder builder = CheckAndMutate.newBuilder(row)<a name="line.4715"></a> |
| <span class="sourceLineNo">4716</span> .ifMatches(family, qualifier, op, comparator.getValue()).timeRange(timeRange);<a name="line.4716"></a> |
| <span class="sourceLineNo">4717</span> if (mutation instanceof Put) {<a name="line.4717"></a> |
| <span class="sourceLineNo">4718</span> checkAndMutate = builder.build((Put) mutation);<a name="line.4718"></a> |
| <span class="sourceLineNo">4719</span> } else if (mutation instanceof Delete) {<a name="line.4719"></a> |
| <span class="sourceLineNo">4720</span> checkAndMutate = builder.build((Delete) mutation);<a name="line.4720"></a> |
| <span class="sourceLineNo">4721</span> } else {<a name="line.4721"></a> |
| <span class="sourceLineNo">4722</span> throw new DoNotRetryIOException("Unsupported mutate type: " + mutation.getClass()<a name="line.4722"></a> |
| <span class="sourceLineNo">4723</span> .getSimpleName().toUpperCase());<a name="line.4723"></a> |
| <span class="sourceLineNo">4724</span> }<a name="line.4724"></a> |
| <span class="sourceLineNo">4725</span> } catch (IllegalArgumentException e) {<a name="line.4725"></a> |
| <span class="sourceLineNo">4726</span> throw new DoNotRetryIOException(e.getMessage());<a name="line.4726"></a> |
| <span class="sourceLineNo">4727</span> }<a name="line.4727"></a> |
| <span class="sourceLineNo">4728</span> return checkAndMutate(checkAndMutate).isSuccess();<a name="line.4728"></a> |
| <span class="sourceLineNo">4729</span> }<a name="line.4729"></a> |
| <span class="sourceLineNo">4730</span><a name="line.4730"></a> |
| <span class="sourceLineNo">4731</span> @Override<a name="line.4731"></a> |
| <span class="sourceLineNo">4732</span> @Deprecated<a name="line.4732"></a> |
| <span class="sourceLineNo">4733</span> public boolean checkAndMutate(byte[] row, Filter filter, TimeRange timeRange, Mutation mutation)<a name="line.4733"></a> |
| <span class="sourceLineNo">4734</span> throws IOException {<a name="line.4734"></a> |
| <span class="sourceLineNo">4735</span> CheckAndMutate checkAndMutate;<a name="line.4735"></a> |
| <span class="sourceLineNo">4736</span> try {<a name="line.4736"></a> |
| <span class="sourceLineNo">4737</span> CheckAndMutate.Builder builder = CheckAndMutate.newBuilder(row).ifMatches(filter)<a name="line.4737"></a> |
| <span class="sourceLineNo">4738</span> .timeRange(timeRange);<a name="line.4738"></a> |
| <span class="sourceLineNo">4739</span> if (mutation instanceof Put) {<a name="line.4739"></a> |
| <span class="sourceLineNo">4740</span> checkAndMutate = builder.build((Put) mutation);<a name="line.4740"></a> |
| <span class="sourceLineNo">4741</span> } else if (mutation instanceof Delete) {<a name="line.4741"></a> |
| <span class="sourceLineNo">4742</span> checkAndMutate = builder.build((Delete) mutation);<a name="line.4742"></a> |
| <span class="sourceLineNo">4743</span> } else {<a name="line.4743"></a> |
| <span class="sourceLineNo">4744</span> throw new DoNotRetryIOException("Unsupported mutate type: " + mutation.getClass()<a name="line.4744"></a> |
| <span class="sourceLineNo">4745</span> .getSimpleName().toUpperCase());<a name="line.4745"></a> |
| <span class="sourceLineNo">4746</span> }<a name="line.4746"></a> |
| <span class="sourceLineNo">4747</span> } catch (IllegalArgumentException e) {<a name="line.4747"></a> |
| <span class="sourceLineNo">4748</span> throw new DoNotRetryIOException(e.getMessage());<a name="line.4748"></a> |
| <span class="sourceLineNo">4749</span> }<a name="line.4749"></a> |
| <span class="sourceLineNo">4750</span> return checkAndMutate(checkAndMutate).isSuccess();<a name="line.4750"></a> |
| <span class="sourceLineNo">4751</span> }<a name="line.4751"></a> |
| <span class="sourceLineNo">4752</span><a name="line.4752"></a> |
| <span class="sourceLineNo">4753</span> @Override<a name="line.4753"></a> |
| <span class="sourceLineNo">4754</span> @Deprecated<a name="line.4754"></a> |
| <span class="sourceLineNo">4755</span> public boolean checkAndRowMutate(byte[] row, byte[] family, byte[] qualifier, CompareOperator op,<a name="line.4755"></a> |
| <span class="sourceLineNo">4756</span> ByteArrayComparable comparator, TimeRange timeRange, RowMutations rm) throws IOException {<a name="line.4756"></a> |
| <span class="sourceLineNo">4757</span> CheckAndMutate checkAndMutate;<a name="line.4757"></a> |
| <span class="sourceLineNo">4758</span> try {<a name="line.4758"></a> |
| <span class="sourceLineNo">4759</span> checkAndMutate = CheckAndMutate.newBuilder(row)<a name="line.4759"></a> |
| <span class="sourceLineNo">4760</span> .ifMatches(family, qualifier, op, comparator.getValue()).timeRange(timeRange).build(rm);<a name="line.4760"></a> |
| <span class="sourceLineNo">4761</span> } catch (IllegalArgumentException e) {<a name="line.4761"></a> |
| <span class="sourceLineNo">4762</span> throw new DoNotRetryIOException(e.getMessage());<a name="line.4762"></a> |
| <span class="sourceLineNo">4763</span> }<a name="line.4763"></a> |
| <span class="sourceLineNo">4764</span> return checkAndMutate(checkAndMutate).isSuccess();<a name="line.4764"></a> |
| <span class="sourceLineNo">4765</span> }<a name="line.4765"></a> |
| <span class="sourceLineNo">4766</span><a name="line.4766"></a> |
| <span class="sourceLineNo">4767</span> @Override<a name="line.4767"></a> |
| <span class="sourceLineNo">4768</span> @Deprecated<a name="line.4768"></a> |
| <span class="sourceLineNo">4769</span> public boolean checkAndRowMutate(byte[] row, Filter filter, TimeRange timeRange, RowMutations rm)<a name="line.4769"></a> |
| <span class="sourceLineNo">4770</span> throws IOException {<a name="line.4770"></a> |
| <span class="sourceLineNo">4771</span> CheckAndMutate checkAndMutate;<a name="line.4771"></a> |
| <span class="sourceLineNo">4772</span> try {<a name="line.4772"></a> |
| <span class="sourceLineNo">4773</span> checkAndMutate = CheckAndMutate.newBuilder(row).ifMatches(filter).timeRange(timeRange)<a name="line.4773"></a> |
| <span class="sourceLineNo">4774</span> .build(rm);<a name="line.4774"></a> |
| <span class="sourceLineNo">4775</span> } catch (IllegalArgumentException e) {<a name="line.4775"></a> |
| <span class="sourceLineNo">4776</span> throw new DoNotRetryIOException(e.getMessage());<a name="line.4776"></a> |
| <span class="sourceLineNo">4777</span> }<a name="line.4777"></a> |
| <span class="sourceLineNo">4778</span> return checkAndMutate(checkAndMutate).isSuccess();<a name="line.4778"></a> |
| <span class="sourceLineNo">4779</span> }<a name="line.4779"></a> |
| <span class="sourceLineNo">4780</span><a name="line.4780"></a> |
| <span class="sourceLineNo">4781</span> @Override<a name="line.4781"></a> |
| <span class="sourceLineNo">4782</span> public CheckAndMutateResult checkAndMutate(CheckAndMutate checkAndMutate) throws IOException {<a name="line.4782"></a> |
| <span class="sourceLineNo">4783</span> byte[] row = checkAndMutate.getRow();<a name="line.4783"></a> |
| <span class="sourceLineNo">4784</span> Filter filter = null;<a name="line.4784"></a> |
| <span class="sourceLineNo">4785</span> byte[] family = null;<a name="line.4785"></a> |
| <span class="sourceLineNo">4786</span> byte[] qualifier = null;<a name="line.4786"></a> |
| <span class="sourceLineNo">4787</span> CompareOperator op = null;<a name="line.4787"></a> |
| <span class="sourceLineNo">4788</span> ByteArrayComparable comparator = null;<a name="line.4788"></a> |
| <span class="sourceLineNo">4789</span> if (checkAndMutate.hasFilter()) {<a name="line.4789"></a> |
| <span class="sourceLineNo">4790</span> filter = checkAndMutate.getFilter();<a name="line.4790"></a> |
| <span class="sourceLineNo">4791</span> } else {<a name="line.4791"></a> |
| <span class="sourceLineNo">4792</span> family = checkAndMutate.getFamily();<a name="line.4792"></a> |
| <span class="sourceLineNo">4793</span> qualifier = checkAndMutate.getQualifier();<a name="line.4793"></a> |
| <span class="sourceLineNo">4794</span> op = checkAndMutate.getCompareOp();<a name="line.4794"></a> |
| <span class="sourceLineNo">4795</span> comparator = new BinaryComparator(checkAndMutate.getValue());<a name="line.4795"></a> |
| <span class="sourceLineNo">4796</span> }<a name="line.4796"></a> |
| <span class="sourceLineNo">4797</span> TimeRange timeRange = checkAndMutate.getTimeRange();<a name="line.4797"></a> |
| <span class="sourceLineNo">4798</span><a name="line.4798"></a> |
| <span class="sourceLineNo">4799</span> Mutation mutation = null;<a name="line.4799"></a> |
| <span class="sourceLineNo">4800</span> RowMutations rowMutations = null;<a name="line.4800"></a> |
| <span class="sourceLineNo">4801</span> if (checkAndMutate.getAction() instanceof Mutation) {<a name="line.4801"></a> |
| <span class="sourceLineNo">4802</span> mutation = (Mutation) checkAndMutate.getAction();<a name="line.4802"></a> |
| <span class="sourceLineNo">4803</span> } else {<a name="line.4803"></a> |
| <span class="sourceLineNo">4804</span> rowMutations = (RowMutations) checkAndMutate.getAction();<a name="line.4804"></a> |
| <span class="sourceLineNo">4805</span> }<a name="line.4805"></a> |
| <span class="sourceLineNo">4806</span><a name="line.4806"></a> |
| <span class="sourceLineNo">4807</span> if (mutation != null) {<a name="line.4807"></a> |
| <span class="sourceLineNo">4808</span> checkMutationType(mutation);<a name="line.4808"></a> |
| <span class="sourceLineNo">4809</span> checkRow(mutation, row);<a name="line.4809"></a> |
| <span class="sourceLineNo">4810</span> } else {<a name="line.4810"></a> |
| <span class="sourceLineNo">4811</span> checkRow(rowMutations, row);<a name="line.4811"></a> |
| <span class="sourceLineNo">4812</span> }<a name="line.4812"></a> |
| <span class="sourceLineNo">4813</span> checkReadOnly();<a name="line.4813"></a> |
| <span class="sourceLineNo">4814</span> // TODO, add check for value length also move this check to the client<a name="line.4814"></a> |
| <span class="sourceLineNo">4815</span> checkResources();<a name="line.4815"></a> |
| <span class="sourceLineNo">4816</span> startRegionOperation();<a name="line.4816"></a> |
| <span class="sourceLineNo">4817</span> try {<a name="line.4817"></a> |
| <span class="sourceLineNo">4818</span> Get get = new Get(row);<a name="line.4818"></a> |
| <span class="sourceLineNo">4819</span> if (family != null) {<a name="line.4819"></a> |
| <span class="sourceLineNo">4820</span> checkFamily(family);<a name="line.4820"></a> |
| <span class="sourceLineNo">4821</span> get.addColumn(family, qualifier);<a name="line.4821"></a> |
| <span class="sourceLineNo">4822</span> }<a name="line.4822"></a> |
| <span class="sourceLineNo">4823</span> if (filter != null) {<a name="line.4823"></a> |
| <span class="sourceLineNo">4824</span> get.setFilter(filter);<a name="line.4824"></a> |
| <span class="sourceLineNo">4825</span> }<a name="line.4825"></a> |
| <span class="sourceLineNo">4826</span> if (timeRange != null) {<a name="line.4826"></a> |
| <span class="sourceLineNo">4827</span> get.setTimeRange(timeRange.getMin(), timeRange.getMax());<a name="line.4827"></a> |
| <span class="sourceLineNo">4828</span> }<a name="line.4828"></a> |
| <span class="sourceLineNo">4829</span> // Lock row - note that doBatchMutate will relock this row if called<a name="line.4829"></a> |
| <span class="sourceLineNo">4830</span> checkRow(row, "doCheckAndRowMutate");<a name="line.4830"></a> |
| <span class="sourceLineNo">4831</span> RowLock rowLock = getRowLockInternal(get.getRow(), false, null);<a name="line.4831"></a> |
| <span class="sourceLineNo">4832</span> try {<a name="line.4832"></a> |
| <span class="sourceLineNo">4833</span> if (this.getCoprocessorHost() != null) {<a name="line.4833"></a> |
| <span class="sourceLineNo">4834</span> CheckAndMutateResult result =<a name="line.4834"></a> |
| <span class="sourceLineNo">4835</span> getCoprocessorHost().preCheckAndMutateAfterRowLock(checkAndMutate);<a name="line.4835"></a> |
| <span class="sourceLineNo">4836</span> if (result != null) {<a name="line.4836"></a> |
| <span class="sourceLineNo">4837</span> return result;<a name="line.4837"></a> |
| <span class="sourceLineNo">4838</span> }<a name="line.4838"></a> |
| <span class="sourceLineNo">4839</span> }<a name="line.4839"></a> |
| <span class="sourceLineNo">4840</span><a name="line.4840"></a> |
| <span class="sourceLineNo">4841</span> // NOTE: We used to wait here until mvcc caught up: mvcc.await();<a name="line.4841"></a> |
| <span class="sourceLineNo">4842</span> // Supposition is that now all changes are done under row locks, then when we go to read,<a name="line.4842"></a> |
| <span class="sourceLineNo">4843</span> // we'll get the latest on this row.<a name="line.4843"></a> |
| <span class="sourceLineNo">4844</span> List<Cell> result = get(get, false);<a name="line.4844"></a> |
| <span class="sourceLineNo">4845</span> boolean matches = false;<a name="line.4845"></a> |
| <span class="sourceLineNo">4846</span> long cellTs = 0;<a name="line.4846"></a> |
| <span class="sourceLineNo">4847</span> if (filter != null) {<a name="line.4847"></a> |
| <span class="sourceLineNo">4848</span> if (!result.isEmpty()) {<a name="line.4848"></a> |
| <span class="sourceLineNo">4849</span> matches = true;<a name="line.4849"></a> |
| <span class="sourceLineNo">4850</span> cellTs = result.get(0).getTimestamp();<a name="line.4850"></a> |
| <span class="sourceLineNo">4851</span> }<a name="line.4851"></a> |
| <span class="sourceLineNo">4852</span> } else {<a name="line.4852"></a> |
| <span class="sourceLineNo">4853</span> boolean valueIsNull = comparator.getValue() == null || comparator.getValue().length == 0;<a name="line.4853"></a> |
| <span class="sourceLineNo">4854</span> if (result.isEmpty() && valueIsNull) {<a name="line.4854"></a> |
| <span class="sourceLineNo">4855</span> matches = true;<a name="line.4855"></a> |
| <span class="sourceLineNo">4856</span> } else if (result.size() > 0 && result.get(0).getValueLength() == 0 && valueIsNull) {<a name="line.4856"></a> |
| <span class="sourceLineNo">4857</span> matches = true;<a name="line.4857"></a> |
| <span class="sourceLineNo">4858</span> cellTs = result.get(0).getTimestamp();<a name="line.4858"></a> |
| <span class="sourceLineNo">4859</span> } else if (result.size() == 1 && !valueIsNull) {<a name="line.4859"></a> |
| <span class="sourceLineNo">4860</span> Cell kv = result.get(0);<a name="line.4860"></a> |
| <span class="sourceLineNo">4861</span> cellTs = kv.getTimestamp();<a name="line.4861"></a> |
| <span class="sourceLineNo">4862</span> int compareResult = PrivateCellUtil.compareValue(kv, comparator);<a name="line.4862"></a> |
| <span class="sourceLineNo">4863</span> matches = matches(op, compareResult);<a name="line.4863"></a> |
| <span class="sourceLineNo">4864</span> }<a name="line.4864"></a> |
| <span class="sourceLineNo">4865</span> }<a name="line.4865"></a> |
| <span class="sourceLineNo">4866</span><a name="line.4866"></a> |
| <span class="sourceLineNo">4867</span> // If matches, perform the mutation or the rowMutations<a name="line.4867"></a> |
| <span class="sourceLineNo">4868</span> if (matches) {<a name="line.4868"></a> |
| <span class="sourceLineNo">4869</span> // We have acquired the row lock already. If the system clock is NOT monotonically<a name="line.4869"></a> |
| <span class="sourceLineNo">4870</span> // non-decreasing (see HBASE-14070) we should make sure that the mutation has a<a name="line.4870"></a> |
| <span class="sourceLineNo">4871</span> // larger timestamp than what was observed via Get. doBatchMutate already does this, but<a name="line.4871"></a> |
| <span class="sourceLineNo">4872</span> // there is no way to pass the cellTs. See HBASE-14054.<a name="line.4872"></a> |
| <span class="sourceLineNo">4873</span> long now = EnvironmentEdgeManager.currentTime();<a name="line.4873"></a> |
| <span class="sourceLineNo">4874</span> long ts = Math.max(now, cellTs); // ensure write is not eclipsed<a name="line.4874"></a> |
| <span class="sourceLineNo">4875</span> byte[] byteTs = Bytes.toBytes(ts);<a name="line.4875"></a> |
| <span class="sourceLineNo">4876</span> if (mutation != null) {<a name="line.4876"></a> |
| <span class="sourceLineNo">4877</span> if (mutation instanceof Put) {<a name="line.4877"></a> |
| <span class="sourceLineNo">4878</span> updateCellTimestamps(mutation.getFamilyCellMap().values(), byteTs);<a name="line.4878"></a> |
| <span class="sourceLineNo">4879</span> }<a name="line.4879"></a> |
| <span class="sourceLineNo">4880</span> // And else 'delete' is not needed since it already does a second get, and sets the<a name="line.4880"></a> |
| <span class="sourceLineNo">4881</span> // timestamp from get (see prepareDeleteTimestamps).<a name="line.4881"></a> |
| <span class="sourceLineNo">4882</span> } else {<a name="line.4882"></a> |
| <span class="sourceLineNo">4883</span> for (Mutation m: rowMutations.getMutations()) {<a name="line.4883"></a> |
| <span class="sourceLineNo">4884</span> if (m instanceof Put) {<a name="line.4884"></a> |
| <span class="sourceLineNo">4885</span> updateCellTimestamps(m.getFamilyCellMap().values(), byteTs);<a name="line.4885"></a> |
| <span class="sourceLineNo">4886</span> }<a name="line.4886"></a> |
| <span class="sourceLineNo">4887</span> }<a name="line.4887"></a> |
| <span class="sourceLineNo">4888</span> // And else 'delete' is not needed since it already does a second get, and sets the<a name="line.4888"></a> |
| <span class="sourceLineNo">4889</span> // timestamp from get (see prepareDeleteTimestamps).<a name="line.4889"></a> |
| <span class="sourceLineNo">4890</span> }<a name="line.4890"></a> |
| <span class="sourceLineNo">4891</span> // All edits for the given row (across all column families) must happen atomically.<a name="line.4891"></a> |
| <span class="sourceLineNo">4892</span> Result r;<a name="line.4892"></a> |
| <span class="sourceLineNo">4893</span> if (mutation != null) {<a name="line.4893"></a> |
| <span class="sourceLineNo">4894</span> r = mutate(mutation, true).getResult();<a name="line.4894"></a> |
| <span class="sourceLineNo">4895</span> } else {<a name="line.4895"></a> |
| <span class="sourceLineNo">4896</span> r = mutateRow(rowMutations);<a name="line.4896"></a> |
| <span class="sourceLineNo">4897</span> }<a name="line.4897"></a> |
| <span class="sourceLineNo">4898</span> this.checkAndMutateChecksPassed.increment();<a name="line.4898"></a> |
| <span class="sourceLineNo">4899</span> return new CheckAndMutateResult(true, r);<a name="line.4899"></a> |
| <span class="sourceLineNo">4900</span> }<a name="line.4900"></a> |
| <span class="sourceLineNo">4901</span> this.checkAndMutateChecksFailed.increment();<a name="line.4901"></a> |
| <span class="sourceLineNo">4902</span> return new CheckAndMutateResult(false, null);<a name="line.4902"></a> |
| <span class="sourceLineNo">4903</span> } finally {<a name="line.4903"></a> |
| <span class="sourceLineNo">4904</span> rowLock.release();<a name="line.4904"></a> |
| <span class="sourceLineNo">4905</span> }<a name="line.4905"></a> |
| <span class="sourceLineNo">4906</span> } finally {<a name="line.4906"></a> |
| <span class="sourceLineNo">4907</span> closeRegionOperation();<a name="line.4907"></a> |
| <span class="sourceLineNo">4908</span> }<a name="line.4908"></a> |
| <span class="sourceLineNo">4909</span> }<a name="line.4909"></a> |
| <span class="sourceLineNo">4910</span><a name="line.4910"></a> |
| <span class="sourceLineNo">4911</span> private void checkMutationType(final Mutation mutation)<a name="line.4911"></a> |
| <span class="sourceLineNo">4912</span> throws DoNotRetryIOException {<a name="line.4912"></a> |
| <span class="sourceLineNo">4913</span> if (!(mutation instanceof Put) && !(mutation instanceof Delete) &&<a name="line.4913"></a> |
| <span class="sourceLineNo">4914</span> !(mutation instanceof Increment) && !(mutation instanceof Append)) {<a name="line.4914"></a> |
| <span class="sourceLineNo">4915</span> throw new org.apache.hadoop.hbase.DoNotRetryIOException(<a name="line.4915"></a> |
| <span class="sourceLineNo">4916</span> "Action must be Put or Delete or Increment or Delete");<a name="line.4916"></a> |
| <span class="sourceLineNo">4917</span> }<a name="line.4917"></a> |
| <span class="sourceLineNo">4918</span> }<a name="line.4918"></a> |
| <span class="sourceLineNo">4919</span><a name="line.4919"></a> |
| <span class="sourceLineNo">4920</span> private void checkRow(final Row action, final byte[] row)<a name="line.4920"></a> |
| <span class="sourceLineNo">4921</span> throws DoNotRetryIOException {<a name="line.4921"></a> |
| <span class="sourceLineNo">4922</span> if (!Bytes.equals(row, action.getRow())) {<a name="line.4922"></a> |
| <span class="sourceLineNo">4923</span> throw new org.apache.hadoop.hbase.DoNotRetryIOException("Action's getRow must match");<a name="line.4923"></a> |
| <span class="sourceLineNo">4924</span> }<a name="line.4924"></a> |
| <span class="sourceLineNo">4925</span> }<a name="line.4925"></a> |
| <span class="sourceLineNo">4926</span><a name="line.4926"></a> |
| <span class="sourceLineNo">4927</span> private boolean matches(final CompareOperator op, final int compareResult) {<a name="line.4927"></a> |
| <span class="sourceLineNo">4928</span> boolean matches = false;<a name="line.4928"></a> |
| <span class="sourceLineNo">4929</span> switch (op) {<a name="line.4929"></a> |
| <span class="sourceLineNo">4930</span> case LESS:<a name="line.4930"></a> |
| <span class="sourceLineNo">4931</span> matches = compareResult < 0;<a name="line.4931"></a> |
| <span class="sourceLineNo">4932</span> break;<a name="line.4932"></a> |
| <span class="sourceLineNo">4933</span> case LESS_OR_EQUAL:<a name="line.4933"></a> |
| <span class="sourceLineNo">4934</span> matches = compareResult <= 0;<a name="line.4934"></a> |
| <span class="sourceLineNo">4935</span> break;<a name="line.4935"></a> |
| <span class="sourceLineNo">4936</span> case EQUAL:<a name="line.4936"></a> |
| <span class="sourceLineNo">4937</span> matches = compareResult == 0;<a name="line.4937"></a> |
| <span class="sourceLineNo">4938</span> break;<a name="line.4938"></a> |
| <span class="sourceLineNo">4939</span> case NOT_EQUAL:<a name="line.4939"></a> |
| <span class="sourceLineNo">4940</span> matches = compareResult != 0;<a name="line.4940"></a> |
| <span class="sourceLineNo">4941</span> break;<a name="line.4941"></a> |
| <span class="sourceLineNo">4942</span> case GREATER_OR_EQUAL:<a name="line.4942"></a> |
| <span class="sourceLineNo">4943</span> matches = compareResult >= 0;<a name="line.4943"></a> |
| <span class="sourceLineNo">4944</span> break;<a name="line.4944"></a> |
| <span class="sourceLineNo">4945</span> case GREATER:<a name="line.4945"></a> |
| <span class="sourceLineNo">4946</span> matches = compareResult > 0;<a name="line.4946"></a> |
| <span class="sourceLineNo">4947</span> break;<a name="line.4947"></a> |
| <span class="sourceLineNo">4948</span> default:<a name="line.4948"></a> |
| <span class="sourceLineNo">4949</span> throw new RuntimeException("Unknown Compare op " + op.name());<a name="line.4949"></a> |
| <span class="sourceLineNo">4950</span> }<a name="line.4950"></a> |
| <span class="sourceLineNo">4951</span> return matches;<a name="line.4951"></a> |
| <span class="sourceLineNo">4952</span> }<a name="line.4952"></a> |
| <span class="sourceLineNo">4953</span><a name="line.4953"></a> |
| <span class="sourceLineNo">4954</span> private OperationStatus mutate(Mutation mutation) throws IOException {<a name="line.4954"></a> |
| <span class="sourceLineNo">4955</span> return mutate(mutation, false);<a name="line.4955"></a> |
| <span class="sourceLineNo">4956</span> }<a name="line.4956"></a> |
| <span class="sourceLineNo">4957</span><a name="line.4957"></a> |
| <span class="sourceLineNo">4958</span> private OperationStatus mutate(Mutation mutation, boolean atomic) throws IOException {<a name="line.4958"></a> |
| <span class="sourceLineNo">4959</span> return mutate(mutation, atomic, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.4959"></a> |
| <span class="sourceLineNo">4960</span> }<a name="line.4960"></a> |
| <span class="sourceLineNo">4961</span><a name="line.4961"></a> |
| <span class="sourceLineNo">4962</span> private OperationStatus mutate(Mutation mutation, boolean atomic, long nonceGroup, long nonce)<a name="line.4962"></a> |
| <span class="sourceLineNo">4963</span> throws IOException {<a name="line.4963"></a> |
| <span class="sourceLineNo">4964</span> OperationStatus[] status =<a name="line.4964"></a> |
| <span class="sourceLineNo">4965</span> this.batchMutate(new Mutation[] { mutation }, atomic, nonceGroup, nonce);<a name="line.4965"></a> |
| <span class="sourceLineNo">4966</span> if (status[0].getOperationStatusCode().equals(OperationStatusCode.SANITY_CHECK_FAILURE)) {<a name="line.4966"></a> |
| <span class="sourceLineNo">4967</span> throw new FailedSanityCheckException(status[0].getExceptionMsg());<a name="line.4967"></a> |
| <span class="sourceLineNo">4968</span> } else if (status[0].getOperationStatusCode().equals(OperationStatusCode.BAD_FAMILY)) {<a name="line.4968"></a> |
| <span class="sourceLineNo">4969</span> throw new NoSuchColumnFamilyException(status[0].getExceptionMsg());<a name="line.4969"></a> |
| <span class="sourceLineNo">4970</span> } else if (status[0].getOperationStatusCode().equals(OperationStatusCode.STORE_TOO_BUSY)) {<a name="line.4970"></a> |
| <span class="sourceLineNo">4971</span> throw new RegionTooBusyException(status[0].getExceptionMsg());<a name="line.4971"></a> |
| <span class="sourceLineNo">4972</span> }<a name="line.4972"></a> |
| <span class="sourceLineNo">4973</span> return status[0];<a name="line.4973"></a> |
| <span class="sourceLineNo">4974</span> }<a name="line.4974"></a> |
| <span class="sourceLineNo">4975</span><a name="line.4975"></a> |
| <span class="sourceLineNo">4976</span> /**<a name="line.4976"></a> |
| <span class="sourceLineNo">4977</span> * Complete taking the snapshot on the region. Writes the region info and adds references to the<a name="line.4977"></a> |
| <span class="sourceLineNo">4978</span> * working snapshot directory.<a name="line.4978"></a> |
| <span class="sourceLineNo">4979</span> *<a name="line.4979"></a> |
| <span class="sourceLineNo">4980</span> * TODO for api consistency, consider adding another version with no {@link ForeignExceptionSnare}<a name="line.4980"></a> |
| <span class="sourceLineNo">4981</span> * arg. (In the future other cancellable HRegion methods could eventually add a<a name="line.4981"></a> |
| <span class="sourceLineNo">4982</span> * {@link ForeignExceptionSnare}, or we could do something fancier).<a name="line.4982"></a> |
| <span class="sourceLineNo">4983</span> *<a name="line.4983"></a> |
| <span class="sourceLineNo">4984</span> * @param desc snapshot description object<a name="line.4984"></a> |
| <span class="sourceLineNo">4985</span> * @param exnSnare ForeignExceptionSnare that captures external exceptions in case we need to<a name="line.4985"></a> |
| <span class="sourceLineNo">4986</span> * bail out. This is allowed to be null and will just be ignored in that case.<a name="line.4986"></a> |
| <span class="sourceLineNo">4987</span> * @throws IOException if there is an external or internal error causing the snapshot to fail<a name="line.4987"></a> |
| <span class="sourceLineNo">4988</span> */<a name="line.4988"></a> |
| <span class="sourceLineNo">4989</span> public void addRegionToSnapshot(SnapshotDescription desc,<a name="line.4989"></a> |
| <span class="sourceLineNo">4990</span> ForeignExceptionSnare exnSnare) throws IOException {<a name="line.4990"></a> |
| <span class="sourceLineNo">4991</span> Path rootDir = CommonFSUtils.getRootDir(conf);<a name="line.4991"></a> |
| <span class="sourceLineNo">4992</span> Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);<a name="line.4992"></a> |
| <span class="sourceLineNo">4993</span><a name="line.4993"></a> |
| <span class="sourceLineNo">4994</span> SnapshotManifest manifest = SnapshotManifest.create(conf, getFilesystem(),<a name="line.4994"></a> |
| <span class="sourceLineNo">4995</span> snapshotDir, desc, exnSnare);<a name="line.4995"></a> |
| <span class="sourceLineNo">4996</span> manifest.addRegion(this);<a name="line.4996"></a> |
| <span class="sourceLineNo">4997</span> }<a name="line.4997"></a> |
| <span class="sourceLineNo">4998</span><a name="line.4998"></a> |
| <span class="sourceLineNo">4999</span> private void updateSequenceId(final Iterable<List<Cell>> cellItr, final long sequenceId)<a name="line.4999"></a> |
| <span class="sourceLineNo">5000</span> throws IOException {<a name="line.5000"></a> |
| <span class="sourceLineNo">5001</span> for (List<Cell> cells: cellItr) {<a name="line.5001"></a> |
| <span class="sourceLineNo">5002</span> if (cells == null) return;<a name="line.5002"></a> |
| <span class="sourceLineNo">5003</span> for (Cell cell : cells) {<a name="line.5003"></a> |
| <span class="sourceLineNo">5004</span> PrivateCellUtil.setSequenceId(cell, sequenceId);<a name="line.5004"></a> |
| <span class="sourceLineNo">5005</span> }<a name="line.5005"></a> |
| <span class="sourceLineNo">5006</span> }<a name="line.5006"></a> |
| <span class="sourceLineNo">5007</span> }<a name="line.5007"></a> |
| <span class="sourceLineNo">5008</span><a name="line.5008"></a> |
| <span class="sourceLineNo">5009</span> /**<a name="line.5009"></a> |
| <span class="sourceLineNo">5010</span> * Replace any cell timestamps set to {@link org.apache.hadoop.hbase.HConstants#LATEST_TIMESTAMP}<a name="line.5010"></a> |
| <span class="sourceLineNo">5011</span> * provided current timestamp.<a name="line.5011"></a> |
| <span class="sourceLineNo">5012</span> * @param cellItr<a name="line.5012"></a> |
| <span class="sourceLineNo">5013</span> * @param now<a name="line.5013"></a> |
| <span class="sourceLineNo">5014</span> */<a name="line.5014"></a> |
| <span class="sourceLineNo">5015</span> private static void updateCellTimestamps(final Iterable<List<Cell>> cellItr, final byte[] now)<a name="line.5015"></a> |
| <span class="sourceLineNo">5016</span> throws IOException {<a name="line.5016"></a> |
| <span class="sourceLineNo">5017</span> for (List<Cell> cells: cellItr) {<a name="line.5017"></a> |
| <span class="sourceLineNo">5018</span> if (cells == null) continue;<a name="line.5018"></a> |
| <span class="sourceLineNo">5019</span> // Optimization: 'foreach' loop is not used. See:<a name="line.5019"></a> |
| <span class="sourceLineNo">5020</span> // HBASE-12023 HRegion.applyFamilyMapToMemstore creates too many iterator objects<a name="line.5020"></a> |
| <span class="sourceLineNo">5021</span> assert cells instanceof RandomAccess;<a name="line.5021"></a> |
| <span class="sourceLineNo">5022</span> int listSize = cells.size();<a name="line.5022"></a> |
| <span class="sourceLineNo">5023</span> for (int i = 0; i < listSize; i++) {<a name="line.5023"></a> |
| <span class="sourceLineNo">5024</span> PrivateCellUtil.updateLatestStamp(cells.get(i), now);<a name="line.5024"></a> |
| <span class="sourceLineNo">5025</span> }<a name="line.5025"></a> |
| <span class="sourceLineNo">5026</span> }<a name="line.5026"></a> |
| <span class="sourceLineNo">5027</span> }<a name="line.5027"></a> |
| <span class="sourceLineNo">5028</span><a name="line.5028"></a> |
| <span class="sourceLineNo">5029</span> /**<a name="line.5029"></a> |
| <span class="sourceLineNo">5030</span> * Possibly rewrite incoming cell tags.<a name="line.5030"></a> |
| <span class="sourceLineNo">5031</span> */<a name="line.5031"></a> |
| <span class="sourceLineNo">5032</span> private void rewriteCellTags(Map<byte[], List<Cell>> familyMap, final Mutation m) {<a name="line.5032"></a> |
| <span class="sourceLineNo">5033</span> // Check if we have any work to do and early out otherwise<a name="line.5033"></a> |
| <span class="sourceLineNo">5034</span> // Update these checks as more logic is added here<a name="line.5034"></a> |
| <span class="sourceLineNo">5035</span> if (m.getTTL() == Long.MAX_VALUE) {<a name="line.5035"></a> |
| <span class="sourceLineNo">5036</span> return;<a name="line.5036"></a> |
| <span class="sourceLineNo">5037</span> }<a name="line.5037"></a> |
| <span class="sourceLineNo">5038</span><a name="line.5038"></a> |
| <span class="sourceLineNo">5039</span> // From this point we know we have some work to do<a name="line.5039"></a> |
| <span class="sourceLineNo">5040</span> for (Map.Entry<byte[], List<Cell>> e: familyMap.entrySet()) {<a name="line.5040"></a> |
| <span class="sourceLineNo">5041</span> List<Cell> cells = e.getValue();<a name="line.5041"></a> |
| <span class="sourceLineNo">5042</span> assert cells instanceof RandomAccess;<a name="line.5042"></a> |
| <span class="sourceLineNo">5043</span> int listSize = cells.size();<a name="line.5043"></a> |
| <span class="sourceLineNo">5044</span> for (int i = 0; i < listSize; i++) {<a name="line.5044"></a> |
| <span class="sourceLineNo">5045</span> Cell cell = cells.get(i);<a name="line.5045"></a> |
| <span class="sourceLineNo">5046</span> List<Tag> newTags = TagUtil.carryForwardTags(null, cell);<a name="line.5046"></a> |
| <span class="sourceLineNo">5047</span> newTags = TagUtil.carryForwardTTLTag(newTags, m.getTTL());<a name="line.5047"></a> |
| <span class="sourceLineNo">5048</span> // Rewrite the cell with the updated set of tags<a name="line.5048"></a> |
| <span class="sourceLineNo">5049</span> cells.set(i, PrivateCellUtil.createCell(cell, newTags));<a name="line.5049"></a> |
| <span class="sourceLineNo">5050</span> }<a name="line.5050"></a> |
| <span class="sourceLineNo">5051</span> }<a name="line.5051"></a> |
| <span class="sourceLineNo">5052</span> }<a name="line.5052"></a> |
| <span class="sourceLineNo">5053</span><a name="line.5053"></a> |
| <span class="sourceLineNo">5054</span> /**<a name="line.5054"></a> |
| <span class="sourceLineNo">5055</span> * Check if resources to support an update.<a name="line.5055"></a> |
| <span class="sourceLineNo">5056</span> * <p/><a name="line.5056"></a> |
| <span class="sourceLineNo">5057</span> * We throw RegionTooBusyException if above memstore limit and expect client to retry using some<a name="line.5057"></a> |
| <span class="sourceLineNo">5058</span> * kind of backoff<a name="line.5058"></a> |
| <span class="sourceLineNo">5059</span> */<a name="line.5059"></a> |
| <span class="sourceLineNo">5060</span> private void checkResources() throws RegionTooBusyException {<a name="line.5060"></a> |
| <span class="sourceLineNo">5061</span> // If catalog region, do not impose resource constraints or block updates.<a name="line.5061"></a> |
| <span class="sourceLineNo">5062</span> if (this.getRegionInfo().isMetaRegion()) {<a name="line.5062"></a> |
| <span class="sourceLineNo">5063</span> return;<a name="line.5063"></a> |
| <span class="sourceLineNo">5064</span> }<a name="line.5064"></a> |
| <span class="sourceLineNo">5065</span><a name="line.5065"></a> |
| <span class="sourceLineNo">5066</span> MemStoreSize mss = this.memStoreSizing.getMemStoreSize();<a name="line.5066"></a> |
| <span class="sourceLineNo">5067</span> if (mss.getHeapSize() + mss.getOffHeapSize() > this.blockingMemStoreSize) {<a name="line.5067"></a> |
| <span class="sourceLineNo">5068</span> blockedRequestsCount.increment();<a name="line.5068"></a> |
| <span class="sourceLineNo">5069</span> requestFlush();<a name="line.5069"></a> |
| <span class="sourceLineNo">5070</span> // Don't print current limit because it will vary too much. The message is used as a key<a name="line.5070"></a> |
| <span class="sourceLineNo">5071</span> // over in RetriesExhaustedWithDetailsException processing.<a name="line.5071"></a> |
| <span class="sourceLineNo">5072</span> final String regionName =<a name="line.5072"></a> |
| <span class="sourceLineNo">5073</span> this.getRegionInfo() == null ? "unknown" : this.getRegionInfo().getEncodedName();<a name="line.5073"></a> |
| <span class="sourceLineNo">5074</span> final String serverName = this.getRegionServerServices() == null ?<a name="line.5074"></a> |
| <span class="sourceLineNo">5075</span> "unknown" : (this.getRegionServerServices().getServerName() == null ? "unknown" :<a name="line.5075"></a> |
| <span class="sourceLineNo">5076</span> this.getRegionServerServices().getServerName().toString());<a name="line.5076"></a> |
| <span class="sourceLineNo">5077</span> RegionTooBusyException rtbe = new RegionTooBusyException(<a name="line.5077"></a> |
| <span class="sourceLineNo">5078</span> "Over memstore limit=" + org.apache.hadoop.hbase.procedure2.util.StringUtils<a name="line.5078"></a> |
| <span class="sourceLineNo">5079</span> .humanSize(this.blockingMemStoreSize) + ", regionName=" + regionName + ", server="<a name="line.5079"></a> |
| <span class="sourceLineNo">5080</span> + serverName);<a name="line.5080"></a> |
| <span class="sourceLineNo">5081</span> LOG.warn("Region is too busy due to exceeding memstore size limit.", rtbe);<a name="line.5081"></a> |
| <span class="sourceLineNo">5082</span> throw rtbe;<a name="line.5082"></a> |
| <span class="sourceLineNo">5083</span> }<a name="line.5083"></a> |
| <span class="sourceLineNo">5084</span> }<a name="line.5084"></a> |
| <span class="sourceLineNo">5085</span><a name="line.5085"></a> |
| <span class="sourceLineNo">5086</span> /**<a name="line.5086"></a> |
| <span class="sourceLineNo">5087</span> * @throws IOException Throws exception if region is in read-only mode.<a name="line.5087"></a> |
| <span class="sourceLineNo">5088</span> */<a name="line.5088"></a> |
| <span class="sourceLineNo">5089</span> private void checkReadOnly() throws IOException {<a name="line.5089"></a> |
| <span class="sourceLineNo">5090</span> if (isReadOnly()) {<a name="line.5090"></a> |
| <span class="sourceLineNo">5091</span> throw new DoNotRetryIOException("region is read only");<a name="line.5091"></a> |
| <span class="sourceLineNo">5092</span> }<a name="line.5092"></a> |
| <span class="sourceLineNo">5093</span> }<a name="line.5093"></a> |
| <span class="sourceLineNo">5094</span><a name="line.5094"></a> |
| <span class="sourceLineNo">5095</span> private void checkReadsEnabled() throws IOException {<a name="line.5095"></a> |
| <span class="sourceLineNo">5096</span> if (!this.writestate.readsEnabled) {<a name="line.5096"></a> |
| <span class="sourceLineNo">5097</span> throw new IOException(getRegionInfo().getEncodedName()<a name="line.5097"></a> |
| <span class="sourceLineNo">5098</span> + ": The region's reads are disabled. Cannot serve the request");<a name="line.5098"></a> |
| <span class="sourceLineNo">5099</span> }<a name="line.5099"></a> |
| <span class="sourceLineNo">5100</span> }<a name="line.5100"></a> |
| <span class="sourceLineNo">5101</span><a name="line.5101"></a> |
| <span class="sourceLineNo">5102</span> public void setReadsEnabled(boolean readsEnabled) {<a name="line.5102"></a> |
| <span class="sourceLineNo">5103</span> if (readsEnabled && !this.writestate.readsEnabled) {<a name="line.5103"></a> |
| <span class="sourceLineNo">5104</span> LOG.info("Enabling reads for {}", getRegionInfo().getEncodedName());<a name="line.5104"></a> |
| <span class="sourceLineNo">5105</span> }<a name="line.5105"></a> |
| <span class="sourceLineNo">5106</span> this.writestate.setReadsEnabled(readsEnabled);<a name="line.5106"></a> |
| <span class="sourceLineNo">5107</span> }<a name="line.5107"></a> |
| <span class="sourceLineNo">5108</span><a name="line.5108"></a> |
| <span class="sourceLineNo">5109</span> /**<a name="line.5109"></a> |
| <span class="sourceLineNo">5110</span> * @param delta If we are doing delta changes -- e.g. increment/append -- then this flag will be<a name="line.5110"></a> |
| <span class="sourceLineNo">5111</span> * set; when set we will run operations that make sense in the increment/append scenario<a name="line.5111"></a> |
| <span class="sourceLineNo">5112</span> * but that do not make sense otherwise.<a name="line.5112"></a> |
| <span class="sourceLineNo">5113</span> * @see #applyToMemStore(HStore, Cell, MemStoreSizing)<a name="line.5113"></a> |
| <span class="sourceLineNo">5114</span> */<a name="line.5114"></a> |
| <span class="sourceLineNo">5115</span> private void applyToMemStore(HStore store, List<Cell> cells, boolean delta,<a name="line.5115"></a> |
| <span class="sourceLineNo">5116</span> MemStoreSizing memstoreAccounting) throws IOException {<a name="line.5116"></a> |
| <span class="sourceLineNo">5117</span> // Any change in how we update Store/MemStore needs to also be done in other applyToMemStore!!!!<a name="line.5117"></a> |
| <span class="sourceLineNo">5118</span> boolean upsert = delta && store.getColumnFamilyDescriptor().getMaxVersions() == 1;<a name="line.5118"></a> |
| <span class="sourceLineNo">5119</span> if (upsert) {<a name="line.5119"></a> |
| <span class="sourceLineNo">5120</span> store.upsert(cells, getSmallestReadPoint(), memstoreAccounting);<a name="line.5120"></a> |
| <span class="sourceLineNo">5121</span> } else {<a name="line.5121"></a> |
| <span class="sourceLineNo">5122</span> store.add(cells, memstoreAccounting);<a name="line.5122"></a> |
| <span class="sourceLineNo">5123</span> }<a name="line.5123"></a> |
| <span class="sourceLineNo">5124</span> }<a name="line.5124"></a> |
| <span class="sourceLineNo">5125</span><a name="line.5125"></a> |
| <span class="sourceLineNo">5126</span> /**<a name="line.5126"></a> |
| <span class="sourceLineNo">5127</span> * @see #applyToMemStore(HStore, List, boolean, MemStoreSizing)<a name="line.5127"></a> |
| <span class="sourceLineNo">5128</span> */<a name="line.5128"></a> |
| <span class="sourceLineNo">5129</span> private void applyToMemStore(HStore store, Cell cell, MemStoreSizing memstoreAccounting)<a name="line.5129"></a> |
| <span class="sourceLineNo">5130</span> throws IOException {<a name="line.5130"></a> |
| <span class="sourceLineNo">5131</span> // Any change in how we update Store/MemStore needs to also be done in other applyToMemStore!!!!<a name="line.5131"></a> |
| <span class="sourceLineNo">5132</span> if (store == null) {<a name="line.5132"></a> |
| <span class="sourceLineNo">5133</span> checkFamily(CellUtil.cloneFamily(cell));<a name="line.5133"></a> |
| <span class="sourceLineNo">5134</span> // Unreachable because checkFamily will throw exception<a name="line.5134"></a> |
| <span class="sourceLineNo">5135</span> }<a name="line.5135"></a> |
| <span class="sourceLineNo">5136</span> store.add(cell, memstoreAccounting);<a name="line.5136"></a> |
| <span class="sourceLineNo">5137</span> }<a name="line.5137"></a> |
| <span class="sourceLineNo">5138</span><a name="line.5138"></a> |
| <span class="sourceLineNo">5139</span> private void checkFamilies(Collection<byte[]> families, Durability durability)<a name="line.5139"></a> |
| <span class="sourceLineNo">5140</span> throws NoSuchColumnFamilyException, InvalidMutationDurabilityException {<a name="line.5140"></a> |
| <span class="sourceLineNo">5141</span> for (byte[] family : families) {<a name="line.5141"></a> |
| <span class="sourceLineNo">5142</span> checkFamily(family, durability);<a name="line.5142"></a> |
| <span class="sourceLineNo">5143</span> }<a name="line.5143"></a> |
| <span class="sourceLineNo">5144</span> }<a name="line.5144"></a> |
| <span class="sourceLineNo">5145</span><a name="line.5145"></a> |
| <span class="sourceLineNo">5146</span> private void checkFamily(final byte[] family, Durability durability)<a name="line.5146"></a> |
| <span class="sourceLineNo">5147</span> throws NoSuchColumnFamilyException, InvalidMutationDurabilityException {<a name="line.5147"></a> |
| <span class="sourceLineNo">5148</span> checkFamily(family);<a name="line.5148"></a> |
| <span class="sourceLineNo">5149</span> if (durability.equals(Durability.SKIP_WAL)<a name="line.5149"></a> |
| <span class="sourceLineNo">5150</span> && htableDescriptor.getColumnFamily(family).getScope()<a name="line.5150"></a> |
| <span class="sourceLineNo">5151</span> != HConstants.REPLICATION_SCOPE_LOCAL) {<a name="line.5151"></a> |
| <span class="sourceLineNo">5152</span> throw new InvalidMutationDurabilityException(<a name="line.5152"></a> |
| <span class="sourceLineNo">5153</span> "Mutation's durability is SKIP_WAL but table's column family " + Bytes.toString(family)<a name="line.5153"></a> |
| <span class="sourceLineNo">5154</span> + " need replication");<a name="line.5154"></a> |
| <span class="sourceLineNo">5155</span> }<a name="line.5155"></a> |
| <span class="sourceLineNo">5156</span> }<a name="line.5156"></a> |
| <span class="sourceLineNo">5157</span><a name="line.5157"></a> |
| <span class="sourceLineNo">5158</span> private void checkFamily(final byte[] family) throws NoSuchColumnFamilyException {<a name="line.5158"></a> |
| <span class="sourceLineNo">5159</span> if (!this.htableDescriptor.hasColumnFamily(family)) {<a name="line.5159"></a> |
| <span class="sourceLineNo">5160</span> throw new NoSuchColumnFamilyException(<a name="line.5160"></a> |
| <span class="sourceLineNo">5161</span> "Column family " + Bytes.toString(family) + " does not exist in region " + this<a name="line.5161"></a> |
| <span class="sourceLineNo">5162</span> + " in table " + this.htableDescriptor);<a name="line.5162"></a> |
| <span class="sourceLineNo">5163</span> }<a name="line.5163"></a> |
| <span class="sourceLineNo">5164</span> }<a name="line.5164"></a> |
| <span class="sourceLineNo">5165</span><a name="line.5165"></a> |
| <span class="sourceLineNo">5166</span> /**<a name="line.5166"></a> |
| <span class="sourceLineNo">5167</span> * Check the collection of families for valid timestamps<a name="line.5167"></a> |
| <span class="sourceLineNo">5168</span> * @param familyMap<a name="line.5168"></a> |
| <span class="sourceLineNo">5169</span> * @param now current timestamp<a name="line.5169"></a> |
| <span class="sourceLineNo">5170</span> * @throws FailedSanityCheckException<a name="line.5170"></a> |
| <span class="sourceLineNo">5171</span> */<a name="line.5171"></a> |
| <span class="sourceLineNo">5172</span> public void checkTimestamps(final Map<byte[], List<Cell>> familyMap, long now)<a name="line.5172"></a> |
| <span class="sourceLineNo">5173</span> throws FailedSanityCheckException {<a name="line.5173"></a> |
| <span class="sourceLineNo">5174</span> if (timestampSlop == HConstants.LATEST_TIMESTAMP) {<a name="line.5174"></a> |
| <span class="sourceLineNo">5175</span> return;<a name="line.5175"></a> |
| <span class="sourceLineNo">5176</span> }<a name="line.5176"></a> |
| <span class="sourceLineNo">5177</span> long maxTs = now + timestampSlop;<a name="line.5177"></a> |
| <span class="sourceLineNo">5178</span> for (List<Cell> kvs : familyMap.values()) {<a name="line.5178"></a> |
| <span class="sourceLineNo">5179</span> // Optimization: 'foreach' loop is not used. See:<a name="line.5179"></a> |
| <span class="sourceLineNo">5180</span> // HBASE-12023 HRegion.applyFamilyMapToMemstore creates too many iterator objects<a name="line.5180"></a> |
| <span class="sourceLineNo">5181</span> assert kvs instanceof RandomAccess;<a name="line.5181"></a> |
| <span class="sourceLineNo">5182</span> int listSize = kvs.size();<a name="line.5182"></a> |
| <span class="sourceLineNo">5183</span> for (int i=0; i < listSize; i++) {<a name="line.5183"></a> |
| <span class="sourceLineNo">5184</span> Cell cell = kvs.get(i);<a name="line.5184"></a> |
| <span class="sourceLineNo">5185</span> // see if the user-side TS is out of range. latest = server-side<a name="line.5185"></a> |
| <span class="sourceLineNo">5186</span> long ts = cell.getTimestamp();<a name="line.5186"></a> |
| <span class="sourceLineNo">5187</span> if (ts != HConstants.LATEST_TIMESTAMP && ts > maxTs) {<a name="line.5187"></a> |
| <span class="sourceLineNo">5188</span> throw new FailedSanityCheckException("Timestamp for KV out of range "<a name="line.5188"></a> |
| <span class="sourceLineNo">5189</span> + cell + " (too.new=" + timestampSlop + ")");<a name="line.5189"></a> |
| <span class="sourceLineNo">5190</span> }<a name="line.5190"></a> |
| <span class="sourceLineNo">5191</span> }<a name="line.5191"></a> |
| <span class="sourceLineNo">5192</span> }<a name="line.5192"></a> |
| <span class="sourceLineNo">5193</span> }<a name="line.5193"></a> |
| <span class="sourceLineNo">5194</span><a name="line.5194"></a> |
| <span class="sourceLineNo">5195</span> /*<a name="line.5195"></a> |
| <span class="sourceLineNo">5196</span> * @param size<a name="line.5196"></a> |
| <span class="sourceLineNo">5197</span> * @return True if size is over the flush threshold<a name="line.5197"></a> |
| <span class="sourceLineNo">5198</span> */<a name="line.5198"></a> |
| <span class="sourceLineNo">5199</span> private boolean isFlushSize(MemStoreSize size) {<a name="line.5199"></a> |
| <span class="sourceLineNo">5200</span> return size.getHeapSize() + size.getOffHeapSize() > getMemStoreFlushSize();<a name="line.5200"></a> |
| <span class="sourceLineNo">5201</span> }<a name="line.5201"></a> |
| <span class="sourceLineNo">5202</span><a name="line.5202"></a> |
| <span class="sourceLineNo">5203</span> private void deleteRecoveredEdits(FileSystem fs, Iterable<Path> files) throws IOException {<a name="line.5203"></a> |
| <span class="sourceLineNo">5204</span> for (Path file : files) {<a name="line.5204"></a> |
| <span class="sourceLineNo">5205</span> if (!fs.delete(file, false)) {<a name="line.5205"></a> |
| <span class="sourceLineNo">5206</span> LOG.error("Failed delete of {}", file);<a name="line.5206"></a> |
| <span class="sourceLineNo">5207</span> } else {<a name="line.5207"></a> |
| <span class="sourceLineNo">5208</span> LOG.debug("Deleted recovered.edits file={}", file);<a name="line.5208"></a> |
| <span class="sourceLineNo">5209</span> }<a name="line.5209"></a> |
| <span class="sourceLineNo">5210</span> }<a name="line.5210"></a> |
| <span class="sourceLineNo">5211</span> }<a name="line.5211"></a> |
| <span class="sourceLineNo">5212</span><a name="line.5212"></a> |
| <span class="sourceLineNo">5213</span> /**<a name="line.5213"></a> |
| <span class="sourceLineNo">5214</span> * Read the edits put under this region by wal splitting process. Put<a name="line.5214"></a> |
| <span class="sourceLineNo">5215</span> * the recovered edits back up into this region.<a name="line.5215"></a> |
| <span class="sourceLineNo">5216</span> *<a name="line.5216"></a> |
| <span class="sourceLineNo">5217</span> * <p>We can ignore any wal message that has a sequence ID that's equal to or<a name="line.5217"></a> |
| <span class="sourceLineNo">5218</span> * lower than minSeqId. (Because we know such messages are already<a name="line.5218"></a> |
| <span class="sourceLineNo">5219</span> * reflected in the HFiles.)<a name="line.5219"></a> |
| <span class="sourceLineNo">5220</span> *<a name="line.5220"></a> |
| <span class="sourceLineNo">5221</span> * <p>While this is running we are putting pressure on memory yet we are<a name="line.5221"></a> |
| <span class="sourceLineNo">5222</span> * outside of our usual accounting because we are not yet an onlined region<a name="line.5222"></a> |
| <span class="sourceLineNo">5223</span> * (this stuff is being run as part of Region initialization). This means<a name="line.5223"></a> |
| <span class="sourceLineNo">5224</span> * that if we're up against global memory limits, we'll not be flagged to flush<a name="line.5224"></a> |
| <span class="sourceLineNo">5225</span> * because we are not online. We can't be flushed by usual mechanisms anyways;<a name="line.5225"></a> |
| <span class="sourceLineNo">5226</span> * we're not yet online so our relative sequenceids are not yet aligned with<a name="line.5226"></a> |
| <span class="sourceLineNo">5227</span> * WAL sequenceids -- not till we come up online, post processing of split<a name="line.5227"></a> |
| <span class="sourceLineNo">5228</span> * edits.<a name="line.5228"></a> |
| <span class="sourceLineNo">5229</span> *<a name="line.5229"></a> |
| <span class="sourceLineNo">5230</span> * <p>But to help relieve memory pressure, at least manage our own heap size<a name="line.5230"></a> |
| <span class="sourceLineNo">5231</span> * flushing if are in excess of per-region limits. Flushing, though, we have<a name="line.5231"></a> |
| <span class="sourceLineNo">5232</span> * to be careful and avoid using the regionserver/wal sequenceid. Its running<a name="line.5232"></a> |
| <span class="sourceLineNo">5233</span> * on a different line to whats going on in here in this region context so if we<a name="line.5233"></a> |
| <span class="sourceLineNo">5234</span> * crashed replaying these edits, but in the midst had a flush that used the<a name="line.5234"></a> |
| <span class="sourceLineNo">5235</span> * regionserver wal with a sequenceid in excess of whats going on in here<a name="line.5235"></a> |
| <span class="sourceLineNo">5236</span> * in this region and with its split editlogs, then we could miss edits the<a name="line.5236"></a> |
| <span class="sourceLineNo">5237</span> * next time we go to recover. So, we have to flush inline, using seqids that<a name="line.5237"></a> |
| <span class="sourceLineNo">5238</span> * make sense in a this single region context only -- until we online.<a name="line.5238"></a> |
| <span class="sourceLineNo">5239</span> *<a name="line.5239"></a> |
| <span class="sourceLineNo">5240</span> * @param maxSeqIdInStores Any edit found in split editlogs needs to be in excess of<a name="line.5240"></a> |
| <span class="sourceLineNo">5241</span> * the maxSeqId for the store to be applied, else its skipped.<a name="line.5241"></a> |
| <span class="sourceLineNo">5242</span> * @return the sequence id of the last edit added to this region out of the<a name="line.5242"></a> |
| <span class="sourceLineNo">5243</span> * recovered edits log or <code>minSeqId</code> if nothing added from editlogs.<a name="line.5243"></a> |
| <span class="sourceLineNo">5244</span> */<a name="line.5244"></a> |
| <span class="sourceLineNo">5245</span> long replayRecoveredEditsIfAny(Map<byte[], Long> maxSeqIdInStores,<a name="line.5245"></a> |
| <span class="sourceLineNo">5246</span> final CancelableProgressable reporter, final MonitoredTask status) throws IOException {<a name="line.5246"></a> |
| <span class="sourceLineNo">5247</span> long minSeqIdForTheRegion = -1;<a name="line.5247"></a> |
| <span class="sourceLineNo">5248</span> for (Long maxSeqIdInStore : maxSeqIdInStores.values()) {<a name="line.5248"></a> |
| <span class="sourceLineNo">5249</span> if (maxSeqIdInStore < minSeqIdForTheRegion || minSeqIdForTheRegion == -1) {<a name="line.5249"></a> |
| <span class="sourceLineNo">5250</span> minSeqIdForTheRegion = maxSeqIdInStore;<a name="line.5250"></a> |
| <span class="sourceLineNo">5251</span> }<a name="line.5251"></a> |
| <span class="sourceLineNo">5252</span> }<a name="line.5252"></a> |
| <span class="sourceLineNo">5253</span> long seqId = minSeqIdForTheRegion;<a name="line.5253"></a> |
| <span class="sourceLineNo">5254</span> String specialRecoveredEditsDirStr = conf.get(SPECIAL_RECOVERED_EDITS_DIR);<a name="line.5254"></a> |
| <span class="sourceLineNo">5255</span> if (org.apache.commons.lang3.StringUtils.isBlank(specialRecoveredEditsDirStr)) {<a name="line.5255"></a> |
| <span class="sourceLineNo">5256</span> FileSystem walFS = getWalFileSystem();<a name="line.5256"></a> |
| <span class="sourceLineNo">5257</span> FileSystem rootFS = getFilesystem();<a name="line.5257"></a> |
| <span class="sourceLineNo">5258</span> Path wrongRegionWALDir = CommonFSUtils.getWrongWALRegionDir(conf, getRegionInfo().getTable(),<a name="line.5258"></a> |
| <span class="sourceLineNo">5259</span> getRegionInfo().getEncodedName());<a name="line.5259"></a> |
| <span class="sourceLineNo">5260</span> Path regionWALDir = getWALRegionDir();<a name="line.5260"></a> |
| <span class="sourceLineNo">5261</span> Path regionDir =<a name="line.5261"></a> |
| <span class="sourceLineNo">5262</span> FSUtils.getRegionDirFromRootDir(CommonFSUtils.getRootDir(conf), getRegionInfo());<a name="line.5262"></a> |
| <span class="sourceLineNo">5263</span><a name="line.5263"></a> |
| <span class="sourceLineNo">5264</span> // We made a mistake in HBASE-20734 so we need to do this dirty hack...<a name="line.5264"></a> |
| <span class="sourceLineNo">5265</span> NavigableSet<Path> filesUnderWrongRegionWALDir =<a name="line.5265"></a> |
| <span class="sourceLineNo">5266</span> WALSplitUtil.getSplitEditFilesSorted(walFS, wrongRegionWALDir);<a name="line.5266"></a> |
| <span class="sourceLineNo">5267</span> seqId = Math.max(seqId, replayRecoveredEditsForPaths(minSeqIdForTheRegion, walFS,<a name="line.5267"></a> |
| <span class="sourceLineNo">5268</span> filesUnderWrongRegionWALDir, reporter, regionDir));<a name="line.5268"></a> |
| <span class="sourceLineNo">5269</span> // This is to ensure backwards compatability with HBASE-20723 where recovered edits can appear<a name="line.5269"></a> |
| <span class="sourceLineNo">5270</span> // under the root dir even if walDir is set.<a name="line.5270"></a> |
| <span class="sourceLineNo">5271</span> NavigableSet<Path> filesUnderRootDir = Collections.emptyNavigableSet();<a name="line.5271"></a> |
| <span class="sourceLineNo">5272</span> if (!regionWALDir.equals(regionDir)) {<a name="line.5272"></a> |
| <span class="sourceLineNo">5273</span> filesUnderRootDir = WALSplitUtil.getSplitEditFilesSorted(rootFS, regionDir);<a name="line.5273"></a> |
| <span class="sourceLineNo">5274</span> seqId = Math.max(seqId, replayRecoveredEditsForPaths(minSeqIdForTheRegion, rootFS,<a name="line.5274"></a> |
| <span class="sourceLineNo">5275</span> filesUnderRootDir, reporter, regionDir));<a name="line.5275"></a> |
| <span class="sourceLineNo">5276</span> }<a name="line.5276"></a> |
| <span class="sourceLineNo">5277</span><a name="line.5277"></a> |
| <span class="sourceLineNo">5278</span> NavigableSet<Path> files = WALSplitUtil.getSplitEditFilesSorted(walFS, regionWALDir);<a name="line.5278"></a> |
| <span class="sourceLineNo">5279</span> seqId = Math.max(seqId,<a name="line.5279"></a> |
| <span class="sourceLineNo">5280</span> replayRecoveredEditsForPaths(minSeqIdForTheRegion, walFS, files, reporter, regionWALDir));<a name="line.5280"></a> |
| <span class="sourceLineNo">5281</span> if (seqId > minSeqIdForTheRegion) {<a name="line.5281"></a> |
| <span class="sourceLineNo">5282</span> // Then we added some edits to memory. Flush and cleanup split edit files.<a name="line.5282"></a> |
| <span class="sourceLineNo">5283</span> internalFlushcache(null, seqId, stores.values(), status, false,<a name="line.5283"></a> |
| <span class="sourceLineNo">5284</span> FlushLifeCycleTracker.DUMMY);<a name="line.5284"></a> |
| <span class="sourceLineNo">5285</span> }<a name="line.5285"></a> |
| <span class="sourceLineNo">5286</span> // Now delete the content of recovered edits. We're done w/ them.<a name="line.5286"></a> |
| <span class="sourceLineNo">5287</span> if (files.size() > 0 && this.conf.getBoolean("hbase.region.archive.recovered.edits", false)) {<a name="line.5287"></a> |
| <span class="sourceLineNo">5288</span> // For debugging data loss issues!<a name="line.5288"></a> |
| <span class="sourceLineNo">5289</span> // If this flag is set, make use of the hfile archiving by making recovered.edits a fake<a name="line.5289"></a> |
| <span class="sourceLineNo">5290</span> // column family. Have to fake out file type too by casting our recovered.edits as<a name="line.5290"></a> |
| <span class="sourceLineNo">5291</span> // storefiles<a name="line.5291"></a> |
| <span class="sourceLineNo">5292</span> String fakeFamilyName = WALSplitUtil.getRegionDirRecoveredEditsDir(regionWALDir).getName();<a name="line.5292"></a> |
| <span class="sourceLineNo">5293</span> Set<HStoreFile> fakeStoreFiles = new HashSet<>(files.size());<a name="line.5293"></a> |
| <span class="sourceLineNo">5294</span> for (Path file : files) {<a name="line.5294"></a> |
| <span class="sourceLineNo">5295</span> fakeStoreFiles.add(new HStoreFile(walFS, file, this.conf, null, null, true));<a name="line.5295"></a> |
| <span class="sourceLineNo">5296</span> }<a name="line.5296"></a> |
| <span class="sourceLineNo">5297</span> getRegionWALFileSystem().archiveRecoveredEdits(fakeFamilyName, fakeStoreFiles);<a name="line.5297"></a> |
| <span class="sourceLineNo">5298</span> } else {<a name="line.5298"></a> |
| <span class="sourceLineNo">5299</span> deleteRecoveredEdits(walFS, Iterables.concat(files, filesUnderWrongRegionWALDir));<a name="line.5299"></a> |
| <span class="sourceLineNo">5300</span> deleteRecoveredEdits(rootFS, filesUnderRootDir);<a name="line.5300"></a> |
| <span class="sourceLineNo">5301</span> }<a name="line.5301"></a> |
| <span class="sourceLineNo">5302</span> } else {<a name="line.5302"></a> |
| <span class="sourceLineNo">5303</span> Path recoveredEditsDir = new Path(specialRecoveredEditsDirStr);<a name="line.5303"></a> |
| <span class="sourceLineNo">5304</span> FileSystem fs = recoveredEditsDir.getFileSystem(conf);<a name="line.5304"></a> |
| <span class="sourceLineNo">5305</span> FileStatus[] files = fs.listStatus(recoveredEditsDir);<a name="line.5305"></a> |
| <span class="sourceLineNo">5306</span> LOG.debug("Found {} recovered edits file(s) under {}", files == null ? 0 : files.length,<a name="line.5306"></a> |
| <span class="sourceLineNo">5307</span> recoveredEditsDir);<a name="line.5307"></a> |
| <span class="sourceLineNo">5308</span> if (files != null) {<a name="line.5308"></a> |
| <span class="sourceLineNo">5309</span> for (FileStatus file : files) {<a name="line.5309"></a> |
| <span class="sourceLineNo">5310</span> // it is safe to trust the zero-length in this case because we've been through rename and<a name="line.5310"></a> |
| <span class="sourceLineNo">5311</span> // lease recovery in the above.<a name="line.5311"></a> |
| <span class="sourceLineNo">5312</span> if (isZeroLengthThenDelete(fs, file, file.getPath())) {<a name="line.5312"></a> |
| <span class="sourceLineNo">5313</span> continue;<a name="line.5313"></a> |
| <span class="sourceLineNo">5314</span> }<a name="line.5314"></a> |
| <span class="sourceLineNo">5315</span> seqId =<a name="line.5315"></a> |
| <span class="sourceLineNo">5316</span> Math.max(seqId, replayRecoveredEdits(file.getPath(), maxSeqIdInStores, reporter, fs));<a name="line.5316"></a> |
| <span class="sourceLineNo">5317</span> }<a name="line.5317"></a> |
| <span class="sourceLineNo">5318</span> }<a name="line.5318"></a> |
| <span class="sourceLineNo">5319</span> if (seqId > minSeqIdForTheRegion) {<a name="line.5319"></a> |
| <span class="sourceLineNo">5320</span> // Then we added some edits to memory. Flush and cleanup split edit files.<a name="line.5320"></a> |
| <span class="sourceLineNo">5321</span> internalFlushcache(null, seqId, stores.values(), status, false,<a name="line.5321"></a> |
| <span class="sourceLineNo">5322</span> FlushLifeCycleTracker.DUMMY);<a name="line.5322"></a> |
| <span class="sourceLineNo">5323</span> }<a name="line.5323"></a> |
| <span class="sourceLineNo">5324</span> deleteRecoveredEdits(fs,<a name="line.5324"></a> |
| <span class="sourceLineNo">5325</span> Stream.of(files).map(FileStatus::getPath).collect(Collectors.toList()));<a name="line.5325"></a> |
| <span class="sourceLineNo">5326</span> }<a name="line.5326"></a> |
| <span class="sourceLineNo">5327</span><a name="line.5327"></a> |
| <span class="sourceLineNo">5328</span> return seqId;<a name="line.5328"></a> |
| <span class="sourceLineNo">5329</span> }<a name="line.5329"></a> |
| <span class="sourceLineNo">5330</span><a name="line.5330"></a> |
| <span class="sourceLineNo">5331</span> private long replayRecoveredEditsForPaths(long minSeqIdForTheRegion, FileSystem fs,<a name="line.5331"></a> |
| <span class="sourceLineNo">5332</span> final NavigableSet<Path> files, final CancelableProgressable reporter, final Path regionDir)<a name="line.5332"></a> |
| <span class="sourceLineNo">5333</span> throws IOException {<a name="line.5333"></a> |
| <span class="sourceLineNo">5334</span> long seqid = minSeqIdForTheRegion;<a name="line.5334"></a> |
| <span class="sourceLineNo">5335</span> if (LOG.isDebugEnabled()) {<a name="line.5335"></a> |
| <span class="sourceLineNo">5336</span> LOG.debug("Found " + (files == null ? 0 : files.size())<a name="line.5336"></a> |
| <span class="sourceLineNo">5337</span> + " recovered edits file(s) under " + regionDir);<a name="line.5337"></a> |
| <span class="sourceLineNo">5338</span> }<a name="line.5338"></a> |
| <span class="sourceLineNo">5339</span><a name="line.5339"></a> |
| <span class="sourceLineNo">5340</span> if (files == null || files.isEmpty()) {<a name="line.5340"></a> |
| <span class="sourceLineNo">5341</span> return minSeqIdForTheRegion;<a name="line.5341"></a> |
| <span class="sourceLineNo">5342</span> }<a name="line.5342"></a> |
| <span class="sourceLineNo">5343</span><a name="line.5343"></a> |
| <span class="sourceLineNo">5344</span> for (Path edits: files) {<a name="line.5344"></a> |
| <span class="sourceLineNo">5345</span> if (edits == null || !fs.exists(edits)) {<a name="line.5345"></a> |
| <span class="sourceLineNo">5346</span> LOG.warn("Null or non-existent edits file: " + edits);<a name="line.5346"></a> |
| <span class="sourceLineNo">5347</span> continue;<a name="line.5347"></a> |
| <span class="sourceLineNo">5348</span> }<a name="line.5348"></a> |
| <span class="sourceLineNo">5349</span> if (isZeroLengthThenDelete(fs, fs.getFileStatus(edits), edits)) {<a name="line.5349"></a> |
| <span class="sourceLineNo">5350</span> continue;<a name="line.5350"></a> |
| <span class="sourceLineNo">5351</span> }<a name="line.5351"></a> |
| <span class="sourceLineNo">5352</span><a name="line.5352"></a> |
| <span class="sourceLineNo">5353</span> long maxSeqId;<a name="line.5353"></a> |
| <span class="sourceLineNo">5354</span> String fileName = edits.getName();<a name="line.5354"></a> |
| <span class="sourceLineNo">5355</span> maxSeqId = Math.abs(Long.parseLong(fileName));<a name="line.5355"></a> |
| <span class="sourceLineNo">5356</span> if (maxSeqId <= minSeqIdForTheRegion) {<a name="line.5356"></a> |
| <span class="sourceLineNo">5357</span> if (LOG.isDebugEnabled()) {<a name="line.5357"></a> |
| <span class="sourceLineNo">5358</span> String msg = "Maximum sequenceid for this wal is " + maxSeqId<a name="line.5358"></a> |
| <span class="sourceLineNo">5359</span> + " and minimum sequenceid for the region " + this + " is " + minSeqIdForTheRegion<a name="line.5359"></a> |
| <span class="sourceLineNo">5360</span> + ", skipped the whole file, path=" + edits;<a name="line.5360"></a> |
| <span class="sourceLineNo">5361</span> LOG.debug(msg);<a name="line.5361"></a> |
| <span class="sourceLineNo">5362</span> }<a name="line.5362"></a> |
| <span class="sourceLineNo">5363</span> continue;<a name="line.5363"></a> |
| <span class="sourceLineNo">5364</span> }<a name="line.5364"></a> |
| <span class="sourceLineNo">5365</span><a name="line.5365"></a> |
| <span class="sourceLineNo">5366</span> try {<a name="line.5366"></a> |
| <span class="sourceLineNo">5367</span> // replay the edits. Replay can return -1 if everything is skipped, only update<a name="line.5367"></a> |
| <span class="sourceLineNo">5368</span> // if seqId is greater<a name="line.5368"></a> |
| <span class="sourceLineNo">5369</span> seqid = Math.max(seqid, replayRecoveredEdits(edits, maxSeqIdInStores, reporter, fs));<a name="line.5369"></a> |
| <span class="sourceLineNo">5370</span> } catch (IOException e) {<a name="line.5370"></a> |
| <span class="sourceLineNo">5371</span> handleException(fs, edits, e);<a name="line.5371"></a> |
| <span class="sourceLineNo">5372</span> }<a name="line.5372"></a> |
| <span class="sourceLineNo">5373</span> }<a name="line.5373"></a> |
| <span class="sourceLineNo">5374</span> return seqid;<a name="line.5374"></a> |
| <span class="sourceLineNo">5375</span> }<a name="line.5375"></a> |
| <span class="sourceLineNo">5376</span><a name="line.5376"></a> |
| <span class="sourceLineNo">5377</span> private void handleException(FileSystem fs, Path edits, IOException e) throws IOException {<a name="line.5377"></a> |
| <span class="sourceLineNo">5378</span> boolean skipErrors = conf.getBoolean(HConstants.HREGION_EDITS_REPLAY_SKIP_ERRORS,<a name="line.5378"></a> |
| <span class="sourceLineNo">5379</span> conf.getBoolean("hbase.skip.errors", HConstants.DEFAULT_HREGION_EDITS_REPLAY_SKIP_ERRORS));<a name="line.5379"></a> |
| <span class="sourceLineNo">5380</span> if (conf.get("hbase.skip.errors") != null) {<a name="line.5380"></a> |
| <span class="sourceLineNo">5381</span> LOG.warn("The property 'hbase.skip.errors' has been deprecated. Please use "<a name="line.5381"></a> |
| <span class="sourceLineNo">5382</span> + HConstants.HREGION_EDITS_REPLAY_SKIP_ERRORS + " instead.");<a name="line.5382"></a> |
| <span class="sourceLineNo">5383</span> }<a name="line.5383"></a> |
| <span class="sourceLineNo">5384</span> if (skipErrors) {<a name="line.5384"></a> |
| <span class="sourceLineNo">5385</span> Path p = WALSplitUtil.moveAsideBadEditsFile(fs, edits);<a name="line.5385"></a> |
| <span class="sourceLineNo">5386</span> LOG.error(HConstants.HREGION_EDITS_REPLAY_SKIP_ERRORS + "=true so continuing. Renamed "<a name="line.5386"></a> |
| <span class="sourceLineNo">5387</span> + edits + " as " + p,<a name="line.5387"></a> |
| <span class="sourceLineNo">5388</span> e);<a name="line.5388"></a> |
| <span class="sourceLineNo">5389</span> } else {<a name="line.5389"></a> |
| <span class="sourceLineNo">5390</span> throw e;<a name="line.5390"></a> |
| <span class="sourceLineNo">5391</span> }<a name="line.5391"></a> |
| <span class="sourceLineNo">5392</span> }<a name="line.5392"></a> |
| <span class="sourceLineNo">5393</span><a name="line.5393"></a> |
| <span class="sourceLineNo">5394</span> /**<a name="line.5394"></a> |
| <span class="sourceLineNo">5395</span> * @param edits File of recovered edits.<a name="line.5395"></a> |
| <span class="sourceLineNo">5396</span> * @param maxSeqIdInStores Maximum sequenceid found in each store. Edits in wal must be larger<a name="line.5396"></a> |
| <span class="sourceLineNo">5397</span> * than this to be replayed for each store.<a name="line.5397"></a> |
| <span class="sourceLineNo">5398</span> * @return the sequence id of the last edit added to this region out of the recovered edits log or<a name="line.5398"></a> |
| <span class="sourceLineNo">5399</span> * <code>minSeqId</code> if nothing added from editlogs.<a name="line.5399"></a> |
| <span class="sourceLineNo">5400</span> */<a name="line.5400"></a> |
| <span class="sourceLineNo">5401</span> private long replayRecoveredEdits(final Path edits, Map<byte[], Long> maxSeqIdInStores,<a name="line.5401"></a> |
| <span class="sourceLineNo">5402</span> final CancelableProgressable reporter, FileSystem fs) throws IOException {<a name="line.5402"></a> |
| <span class="sourceLineNo">5403</span> String msg = "Replaying edits from " + edits;<a name="line.5403"></a> |
| <span class="sourceLineNo">5404</span> LOG.info(msg);<a name="line.5404"></a> |
| <span class="sourceLineNo">5405</span> MonitoredTask status = TaskMonitor.get().createStatus(msg);<a name="line.5405"></a> |
| <span class="sourceLineNo">5406</span><a name="line.5406"></a> |
| <span class="sourceLineNo">5407</span> status.setStatus("Opening recovered edits");<a name="line.5407"></a> |
| <span class="sourceLineNo">5408</span> WAL.Reader reader = null;<a name="line.5408"></a> |
| <span class="sourceLineNo">5409</span> try {<a name="line.5409"></a> |
| <span class="sourceLineNo">5410</span> reader = WALFactory.createReader(fs, edits, conf);<a name="line.5410"></a> |
| <span class="sourceLineNo">5411</span> long currentEditSeqId = -1;<a name="line.5411"></a> |
| <span class="sourceLineNo">5412</span> long currentReplaySeqId = -1;<a name="line.5412"></a> |
| <span class="sourceLineNo">5413</span> long firstSeqIdInLog = -1;<a name="line.5413"></a> |
| <span class="sourceLineNo">5414</span> long skippedEdits = 0;<a name="line.5414"></a> |
| <span class="sourceLineNo">5415</span> long editsCount = 0;<a name="line.5415"></a> |
| <span class="sourceLineNo">5416</span> long intervalEdits = 0;<a name="line.5416"></a> |
| <span class="sourceLineNo">5417</span> WAL.Entry entry;<a name="line.5417"></a> |
| <span class="sourceLineNo">5418</span> HStore store = null;<a name="line.5418"></a> |
| <span class="sourceLineNo">5419</span> boolean reported_once = false;<a name="line.5419"></a> |
| <span class="sourceLineNo">5420</span> ServerNonceManager ng = this.rsServices == null ? null : this.rsServices.getNonceManager();<a name="line.5420"></a> |
| <span class="sourceLineNo">5421</span><a name="line.5421"></a> |
| <span class="sourceLineNo">5422</span> try {<a name="line.5422"></a> |
| <span class="sourceLineNo">5423</span> // How many edits seen before we check elapsed time<a name="line.5423"></a> |
| <span class="sourceLineNo">5424</span> int interval = this.conf.getInt("hbase.hstore.report.interval.edits", 2000);<a name="line.5424"></a> |
| <span class="sourceLineNo">5425</span> // How often to send a progress report (default 1/2 master timeout)<a name="line.5425"></a> |
| <span class="sourceLineNo">5426</span> int period = this.conf.getInt("hbase.hstore.report.period", 300000);<a name="line.5426"></a> |
| <span class="sourceLineNo">5427</span> long lastReport = EnvironmentEdgeManager.currentTime();<a name="line.5427"></a> |
| <span class="sourceLineNo">5428</span><a name="line.5428"></a> |
| <span class="sourceLineNo">5429</span> if (coprocessorHost != null) {<a name="line.5429"></a> |
| <span class="sourceLineNo">5430</span> coprocessorHost.preReplayWALs(this.getRegionInfo(), edits);<a name="line.5430"></a> |
| <span class="sourceLineNo">5431</span> }<a name="line.5431"></a> |
| <span class="sourceLineNo">5432</span><a name="line.5432"></a> |
| <span class="sourceLineNo">5433</span> while ((entry = reader.next()) != null) {<a name="line.5433"></a> |
| <span class="sourceLineNo">5434</span> WALKey key = entry.getKey();<a name="line.5434"></a> |
| <span class="sourceLineNo">5435</span> WALEdit val = entry.getEdit();<a name="line.5435"></a> |
| <span class="sourceLineNo">5436</span><a name="line.5436"></a> |
| <span class="sourceLineNo">5437</span> if (ng != null) { // some test, or nonces disabled<a name="line.5437"></a> |
| <span class="sourceLineNo">5438</span> ng.reportOperationFromWal(key.getNonceGroup(), key.getNonce(), key.getWriteTime());<a name="line.5438"></a> |
| <span class="sourceLineNo">5439</span> }<a name="line.5439"></a> |
| <span class="sourceLineNo">5440</span><a name="line.5440"></a> |
| <span class="sourceLineNo">5441</span> if (reporter != null) {<a name="line.5441"></a> |
| <span class="sourceLineNo">5442</span> intervalEdits += val.size();<a name="line.5442"></a> |
| <span class="sourceLineNo">5443</span> if (intervalEdits >= interval) {<a name="line.5443"></a> |
| <span class="sourceLineNo">5444</span> // Number of edits interval reached<a name="line.5444"></a> |
| <span class="sourceLineNo">5445</span> intervalEdits = 0;<a name="line.5445"></a> |
| <span class="sourceLineNo">5446</span> long cur = EnvironmentEdgeManager.currentTime();<a name="line.5446"></a> |
| <span class="sourceLineNo">5447</span> if (lastReport + period <= cur) {<a name="line.5447"></a> |
| <span class="sourceLineNo">5448</span> status.setStatus("Replaying edits..." +<a name="line.5448"></a> |
| <span class="sourceLineNo">5449</span> " skipped=" + skippedEdits +<a name="line.5449"></a> |
| <span class="sourceLineNo">5450</span> " edits=" + editsCount);<a name="line.5450"></a> |
| <span class="sourceLineNo">5451</span> // Timeout reached<a name="line.5451"></a> |
| <span class="sourceLineNo">5452</span> if(!reporter.progress()) {<a name="line.5452"></a> |
| <span class="sourceLineNo">5453</span> msg = "Progressable reporter failed, stopping replay for region " + this;<a name="line.5453"></a> |
| <span class="sourceLineNo">5454</span> LOG.warn(msg);<a name="line.5454"></a> |
| <span class="sourceLineNo">5455</span> status.abort(msg);<a name="line.5455"></a> |
| <span class="sourceLineNo">5456</span> throw new IOException(msg);<a name="line.5456"></a> |
| <span class="sourceLineNo">5457</span> }<a name="line.5457"></a> |
| <span class="sourceLineNo">5458</span> reported_once = true;<a name="line.5458"></a> |
| <span class="sourceLineNo">5459</span> lastReport = cur;<a name="line.5459"></a> |
| <span class="sourceLineNo">5460</span> }<a name="line.5460"></a> |
| <span class="sourceLineNo">5461</span> }<a name="line.5461"></a> |
| <span class="sourceLineNo">5462</span> }<a name="line.5462"></a> |
| <span class="sourceLineNo">5463</span><a name="line.5463"></a> |
| <span class="sourceLineNo">5464</span> if (firstSeqIdInLog == -1) {<a name="line.5464"></a> |
| <span class="sourceLineNo">5465</span> firstSeqIdInLog = key.getSequenceId();<a name="line.5465"></a> |
| <span class="sourceLineNo">5466</span> }<a name="line.5466"></a> |
| <span class="sourceLineNo">5467</span> if (currentEditSeqId > key.getSequenceId()) {<a name="line.5467"></a> |
| <span class="sourceLineNo">5468</span> // when this condition is true, it means we have a serious defect because we need to<a name="line.5468"></a> |
| <span class="sourceLineNo">5469</span> // maintain increasing SeqId for WAL edits per region<a name="line.5469"></a> |
| <span class="sourceLineNo">5470</span> LOG.error(getRegionInfo().getEncodedName() + " : "<a name="line.5470"></a> |
| <span class="sourceLineNo">5471</span> + "Found decreasing SeqId. PreId=" + currentEditSeqId + " key=" + key<a name="line.5471"></a> |
| <span class="sourceLineNo">5472</span> + "; edit=" + val);<a name="line.5472"></a> |
| <span class="sourceLineNo">5473</span> } else {<a name="line.5473"></a> |
| <span class="sourceLineNo">5474</span> currentEditSeqId = key.getSequenceId();<a name="line.5474"></a> |
| <span class="sourceLineNo">5475</span> }<a name="line.5475"></a> |
| <span class="sourceLineNo">5476</span> currentReplaySeqId = (key.getOrigLogSeqNum() > 0) ?<a name="line.5476"></a> |
| <span class="sourceLineNo">5477</span> key.getOrigLogSeqNum() : currentEditSeqId;<a name="line.5477"></a> |
| <span class="sourceLineNo">5478</span><a name="line.5478"></a> |
| <span class="sourceLineNo">5479</span> // Start coprocessor replay here. The coprocessor is for each WALEdit<a name="line.5479"></a> |
| <span class="sourceLineNo">5480</span> // instead of a KeyValue.<a name="line.5480"></a> |
| <span class="sourceLineNo">5481</span> if (coprocessorHost != null) {<a name="line.5481"></a> |
| <span class="sourceLineNo">5482</span> status.setStatus("Running pre-WAL-restore hook in coprocessors");<a name="line.5482"></a> |
| <span class="sourceLineNo">5483</span> if (coprocessorHost.preWALRestore(this.getRegionInfo(), key, val)) {<a name="line.5483"></a> |
| <span class="sourceLineNo">5484</span> // if bypass this wal entry, ignore it ...<a name="line.5484"></a> |
| <span class="sourceLineNo">5485</span> continue;<a name="line.5485"></a> |
| <span class="sourceLineNo">5486</span> }<a name="line.5486"></a> |
| <span class="sourceLineNo">5487</span> }<a name="line.5487"></a> |
| <span class="sourceLineNo">5488</span> boolean checkRowWithinBoundary = false;<a name="line.5488"></a> |
| <span class="sourceLineNo">5489</span> // Check this edit is for this region.<a name="line.5489"></a> |
| <span class="sourceLineNo">5490</span> if (!Bytes.equals(key.getEncodedRegionName(),<a name="line.5490"></a> |
| <span class="sourceLineNo">5491</span> this.getRegionInfo().getEncodedNameAsBytes())) {<a name="line.5491"></a> |
| <span class="sourceLineNo">5492</span> checkRowWithinBoundary = true;<a name="line.5492"></a> |
| <span class="sourceLineNo">5493</span> }<a name="line.5493"></a> |
| <span class="sourceLineNo">5494</span><a name="line.5494"></a> |
| <span class="sourceLineNo">5495</span> boolean flush = false;<a name="line.5495"></a> |
| <span class="sourceLineNo">5496</span> MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();<a name="line.5496"></a> |
| <span class="sourceLineNo">5497</span> for (Cell cell: val.getCells()) {<a name="line.5497"></a> |
| <span class="sourceLineNo">5498</span> // Check this edit is for me. Also, guard against writing the special<a name="line.5498"></a> |
| <span class="sourceLineNo">5499</span> // METACOLUMN info such as HBASE::CACHEFLUSH entries<a name="line.5499"></a> |
| <span class="sourceLineNo">5500</span> if (WALEdit.isMetaEditFamily(cell)) {<a name="line.5500"></a> |
| <span class="sourceLineNo">5501</span> // if region names don't match, skipp replaying compaction marker<a name="line.5501"></a> |
| <span class="sourceLineNo">5502</span> if (!checkRowWithinBoundary) {<a name="line.5502"></a> |
| <span class="sourceLineNo">5503</span> //this is a special edit, we should handle it<a name="line.5503"></a> |
| <span class="sourceLineNo">5504</span> CompactionDescriptor compaction = WALEdit.getCompaction(cell);<a name="line.5504"></a> |
| <span class="sourceLineNo">5505</span> if (compaction != null) {<a name="line.5505"></a> |
| <span class="sourceLineNo">5506</span> //replay the compaction<a name="line.5506"></a> |
| <span class="sourceLineNo">5507</span> replayWALCompactionMarker(compaction, false, true, Long.MAX_VALUE);<a name="line.5507"></a> |
| <span class="sourceLineNo">5508</span> }<a name="line.5508"></a> |
| <span class="sourceLineNo">5509</span> }<a name="line.5509"></a> |
| <span class="sourceLineNo">5510</span> skippedEdits++;<a name="line.5510"></a> |
| <span class="sourceLineNo">5511</span> continue;<a name="line.5511"></a> |
| <span class="sourceLineNo">5512</span> }<a name="line.5512"></a> |
| <span class="sourceLineNo">5513</span> // Figure which store the edit is meant for.<a name="line.5513"></a> |
| <span class="sourceLineNo">5514</span> if (store == null || !CellUtil.matchingFamily(cell,<a name="line.5514"></a> |
| <span class="sourceLineNo">5515</span> store.getColumnFamilyDescriptor().getName())) {<a name="line.5515"></a> |
| <span class="sourceLineNo">5516</span> store = getStore(cell);<a name="line.5516"></a> |
| <span class="sourceLineNo">5517</span> }<a name="line.5517"></a> |
| <span class="sourceLineNo">5518</span> if (store == null) {<a name="line.5518"></a> |
| <span class="sourceLineNo">5519</span> // This should never happen. Perhaps schema was changed between<a name="line.5519"></a> |
| <span class="sourceLineNo">5520</span> // crash and redeploy?<a name="line.5520"></a> |
| <span class="sourceLineNo">5521</span> LOG.warn("No family for cell {} in region {}", cell, this);<a name="line.5521"></a> |
| <span class="sourceLineNo">5522</span> skippedEdits++;<a name="line.5522"></a> |
| <span class="sourceLineNo">5523</span> continue;<a name="line.5523"></a> |
| <span class="sourceLineNo">5524</span> }<a name="line.5524"></a> |
| <span class="sourceLineNo">5525</span> if (checkRowWithinBoundary && !rowIsInRange(this.getRegionInfo(),<a name="line.5525"></a> |
| <span class="sourceLineNo">5526</span> cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())) {<a name="line.5526"></a> |
| <span class="sourceLineNo">5527</span> LOG.warn("Row of {} is not within region boundary for region {}", cell, this);<a name="line.5527"></a> |
| <span class="sourceLineNo">5528</span> skippedEdits++;<a name="line.5528"></a> |
| <span class="sourceLineNo">5529</span> continue;<a name="line.5529"></a> |
| <span class="sourceLineNo">5530</span> }<a name="line.5530"></a> |
| <span class="sourceLineNo">5531</span> // Now, figure if we should skip this edit.<a name="line.5531"></a> |
| <span class="sourceLineNo">5532</span> if (key.getSequenceId() <= maxSeqIdInStores.get(store.getColumnFamilyDescriptor()<a name="line.5532"></a> |
| <span class="sourceLineNo">5533</span> .getName())) {<a name="line.5533"></a> |
| <span class="sourceLineNo">5534</span> skippedEdits++;<a name="line.5534"></a> |
| <span class="sourceLineNo">5535</span> continue;<a name="line.5535"></a> |
| <span class="sourceLineNo">5536</span> }<a name="line.5536"></a> |
| <span class="sourceLineNo">5537</span> PrivateCellUtil.setSequenceId(cell, currentReplaySeqId);<a name="line.5537"></a> |
| <span class="sourceLineNo">5538</span><a name="line.5538"></a> |
| <span class="sourceLineNo">5539</span> restoreEdit(store, cell, memStoreSizing);<a name="line.5539"></a> |
| <span class="sourceLineNo">5540</span> editsCount++;<a name="line.5540"></a> |
| <span class="sourceLineNo">5541</span> }<a name="line.5541"></a> |
| <span class="sourceLineNo">5542</span> MemStoreSize mss = memStoreSizing.getMemStoreSize();<a name="line.5542"></a> |
| <span class="sourceLineNo">5543</span> incMemStoreSize(mss);<a name="line.5543"></a> |
| <span class="sourceLineNo">5544</span> flush = isFlushSize(this.memStoreSizing.getMemStoreSize());<a name="line.5544"></a> |
| <span class="sourceLineNo">5545</span> if (flush) {<a name="line.5545"></a> |
| <span class="sourceLineNo">5546</span> internalFlushcache(null, currentEditSeqId, stores.values(), status, false,<a name="line.5546"></a> |
| <span class="sourceLineNo">5547</span> FlushLifeCycleTracker.DUMMY);<a name="line.5547"></a> |
| <span class="sourceLineNo">5548</span> }<a name="line.5548"></a> |
| <span class="sourceLineNo">5549</span><a name="line.5549"></a> |
| <span class="sourceLineNo">5550</span> if (coprocessorHost != null) {<a name="line.5550"></a> |
| <span class="sourceLineNo">5551</span> coprocessorHost.postWALRestore(this.getRegionInfo(), key, val);<a name="line.5551"></a> |
| <span class="sourceLineNo">5552</span> }<a name="line.5552"></a> |
| <span class="sourceLineNo">5553</span> }<a name="line.5553"></a> |
| <span class="sourceLineNo">5554</span><a name="line.5554"></a> |
| <span class="sourceLineNo">5555</span> if (coprocessorHost != null) {<a name="line.5555"></a> |
| <span class="sourceLineNo">5556</span> coprocessorHost.postReplayWALs(this.getRegionInfo(), edits);<a name="line.5556"></a> |
| <span class="sourceLineNo">5557</span> }<a name="line.5557"></a> |
| <span class="sourceLineNo">5558</span> } catch (EOFException eof) {<a name="line.5558"></a> |
| <span class="sourceLineNo">5559</span> Path p = WALSplitUtil.moveAsideBadEditsFile(walFS, edits);<a name="line.5559"></a> |
| <span class="sourceLineNo">5560</span> msg = "EnLongAddered EOF. Most likely due to Master failure during "<a name="line.5560"></a> |
| <span class="sourceLineNo">5561</span> + "wal splitting, so we have this data in another edit. Continuing, but renaming "<a name="line.5561"></a> |
| <span class="sourceLineNo">5562</span> + edits + " as " + p + " for region " + this;<a name="line.5562"></a> |
| <span class="sourceLineNo">5563</span> LOG.warn(msg, eof);<a name="line.5563"></a> |
| <span class="sourceLineNo">5564</span> status.abort(msg);<a name="line.5564"></a> |
| <span class="sourceLineNo">5565</span> } catch (IOException ioe) {<a name="line.5565"></a> |
| <span class="sourceLineNo">5566</span> // If the IOE resulted from bad file format,<a name="line.5566"></a> |
| <span class="sourceLineNo">5567</span> // then this problem is idempotent and retrying won't help<a name="line.5567"></a> |
| <span class="sourceLineNo">5568</span> if (ioe.getCause() instanceof ParseException) {<a name="line.5568"></a> |
| <span class="sourceLineNo">5569</span> Path p = WALSplitUtil.moveAsideBadEditsFile(walFS, edits);<a name="line.5569"></a> |
| <span class="sourceLineNo">5570</span> msg = "File corruption enLongAddered! " +<a name="line.5570"></a> |
| <span class="sourceLineNo">5571</span> "Continuing, but renaming " + edits + " as " + p;<a name="line.5571"></a> |
| <span class="sourceLineNo">5572</span> LOG.warn(msg, ioe);<a name="line.5572"></a> |
| <span class="sourceLineNo">5573</span> status.setStatus(msg);<a name="line.5573"></a> |
| <span class="sourceLineNo">5574</span> } else {<a name="line.5574"></a> |
| <span class="sourceLineNo">5575</span> status.abort(StringUtils.stringifyException(ioe));<a name="line.5575"></a> |
| <span class="sourceLineNo">5576</span> // other IO errors may be transient (bad network connection,<a name="line.5576"></a> |
| <span class="sourceLineNo">5577</span> // checksum exception on one datanode, etc). throw & retry<a name="line.5577"></a> |
| <span class="sourceLineNo">5578</span> throw ioe;<a name="line.5578"></a> |
| <span class="sourceLineNo">5579</span> }<a name="line.5579"></a> |
| <span class="sourceLineNo">5580</span> }<a name="line.5580"></a> |
| <span class="sourceLineNo">5581</span> if (reporter != null && !reported_once) {<a name="line.5581"></a> |
| <span class="sourceLineNo">5582</span> reporter.progress();<a name="line.5582"></a> |
| <span class="sourceLineNo">5583</span> }<a name="line.5583"></a> |
| <span class="sourceLineNo">5584</span> msg = "Applied " + editsCount + ", skipped " + skippedEdits +<a name="line.5584"></a> |
| <span class="sourceLineNo">5585</span> ", firstSequenceIdInLog=" + firstSeqIdInLog +<a name="line.5585"></a> |
| <span class="sourceLineNo">5586</span> ", maxSequenceIdInLog=" + currentEditSeqId + ", path=" + edits;<a name="line.5586"></a> |
| <span class="sourceLineNo">5587</span> status.markComplete(msg);<a name="line.5587"></a> |
| <span class="sourceLineNo">5588</span> LOG.debug(msg);<a name="line.5588"></a> |
| <span class="sourceLineNo">5589</span> return currentEditSeqId;<a name="line.5589"></a> |
| <span class="sourceLineNo">5590</span> } finally {<a name="line.5590"></a> |
| <span class="sourceLineNo">5591</span> status.cleanup();<a name="line.5591"></a> |
| <span class="sourceLineNo">5592</span> if (reader != null) {<a name="line.5592"></a> |
| <span class="sourceLineNo">5593</span> reader.close();<a name="line.5593"></a> |
| <span class="sourceLineNo">5594</span> }<a name="line.5594"></a> |
| <span class="sourceLineNo">5595</span> }<a name="line.5595"></a> |
| <span class="sourceLineNo">5596</span> }<a name="line.5596"></a> |
| <span class="sourceLineNo">5597</span><a name="line.5597"></a> |
| <span class="sourceLineNo">5598</span> /**<a name="line.5598"></a> |
| <span class="sourceLineNo">5599</span> * Call to complete a compaction. Its for the case where we find in the WAL a compaction<a name="line.5599"></a> |
| <span class="sourceLineNo">5600</span> * that was not finished. We could find one recovering a WAL after a regionserver crash.<a name="line.5600"></a> |
| <span class="sourceLineNo">5601</span> * See HBASE-2331.<a name="line.5601"></a> |
| <span class="sourceLineNo">5602</span> */<a name="line.5602"></a> |
| <span class="sourceLineNo">5603</span> void replayWALCompactionMarker(CompactionDescriptor compaction, boolean pickCompactionFiles,<a name="line.5603"></a> |
| <span class="sourceLineNo">5604</span> boolean removeFiles, long replaySeqId)<a name="line.5604"></a> |
| <span class="sourceLineNo">5605</span> throws IOException {<a name="line.5605"></a> |
| <span class="sourceLineNo">5606</span> try {<a name="line.5606"></a> |
| <span class="sourceLineNo">5607</span> checkTargetRegion(compaction.getEncodedRegionName().toByteArray(),<a name="line.5607"></a> |
| <span class="sourceLineNo">5608</span> "Compaction marker from WAL ", compaction);<a name="line.5608"></a> |
| <span class="sourceLineNo">5609</span> } catch (WrongRegionException wre) {<a name="line.5609"></a> |
| <span class="sourceLineNo">5610</span> if (RegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {<a name="line.5610"></a> |
| <span class="sourceLineNo">5611</span> // skip the compaction marker since it is not for this region<a name="line.5611"></a> |
| <span class="sourceLineNo">5612</span> return;<a name="line.5612"></a> |
| <span class="sourceLineNo">5613</span> }<a name="line.5613"></a> |
| <span class="sourceLineNo">5614</span> throw wre;<a name="line.5614"></a> |
| <span class="sourceLineNo">5615</span> }<a name="line.5615"></a> |
| <span class="sourceLineNo">5616</span><a name="line.5616"></a> |
| <span class="sourceLineNo">5617</span> synchronized (writestate) {<a name="line.5617"></a> |
| <span class="sourceLineNo">5618</span> if (replaySeqId < lastReplayedOpenRegionSeqId) {<a name="line.5618"></a> |
| <span class="sourceLineNo">5619</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5619"></a> |
| <span class="sourceLineNo">5620</span> + "Skipping replaying compaction event :" + TextFormat.shortDebugString(compaction)<a name="line.5620"></a> |
| <span class="sourceLineNo">5621</span> + " because its sequence id " + replaySeqId + " is smaller than this regions "<a name="line.5621"></a> |
| <span class="sourceLineNo">5622</span> + "lastReplayedOpenRegionSeqId of " + lastReplayedOpenRegionSeqId);<a name="line.5622"></a> |
| <span class="sourceLineNo">5623</span> return;<a name="line.5623"></a> |
| <span class="sourceLineNo">5624</span> }<a name="line.5624"></a> |
| <span class="sourceLineNo">5625</span> if (replaySeqId < lastReplayedCompactionSeqId) {<a name="line.5625"></a> |
| <span class="sourceLineNo">5626</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5626"></a> |
| <span class="sourceLineNo">5627</span> + "Skipping replaying compaction event :" + TextFormat.shortDebugString(compaction)<a name="line.5627"></a> |
| <span class="sourceLineNo">5628</span> + " because its sequence id " + replaySeqId + " is smaller than this regions "<a name="line.5628"></a> |
| <span class="sourceLineNo">5629</span> + "lastReplayedCompactionSeqId of " + lastReplayedCompactionSeqId);<a name="line.5629"></a> |
| <span class="sourceLineNo">5630</span> return;<a name="line.5630"></a> |
| <span class="sourceLineNo">5631</span> } else {<a name="line.5631"></a> |
| <span class="sourceLineNo">5632</span> lastReplayedCompactionSeqId = replaySeqId;<a name="line.5632"></a> |
| <span class="sourceLineNo">5633</span> }<a name="line.5633"></a> |
| <span class="sourceLineNo">5634</span><a name="line.5634"></a> |
| <span class="sourceLineNo">5635</span> if (LOG.isDebugEnabled()) {<a name="line.5635"></a> |
| <span class="sourceLineNo">5636</span> LOG.debug(getRegionInfo().getEncodedName() + " : "<a name="line.5636"></a> |
| <span class="sourceLineNo">5637</span> + "Replaying compaction marker " + TextFormat.shortDebugString(compaction)<a name="line.5637"></a> |
| <span class="sourceLineNo">5638</span> + " with seqId=" + replaySeqId + " and lastReplayedOpenRegionSeqId="<a name="line.5638"></a> |
| <span class="sourceLineNo">5639</span> + lastReplayedOpenRegionSeqId);<a name="line.5639"></a> |
| <span class="sourceLineNo">5640</span> }<a name="line.5640"></a> |
| <span class="sourceLineNo">5641</span><a name="line.5641"></a> |
| <span class="sourceLineNo">5642</span> startRegionOperation(Operation.REPLAY_EVENT);<a name="line.5642"></a> |
| <span class="sourceLineNo">5643</span> try {<a name="line.5643"></a> |
| <span class="sourceLineNo">5644</span> HStore store = this.getStore(compaction.getFamilyName().toByteArray());<a name="line.5644"></a> |
| <span class="sourceLineNo">5645</span> if (store == null) {<a name="line.5645"></a> |
| <span class="sourceLineNo">5646</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5646"></a> |
| <span class="sourceLineNo">5647</span> + "Found Compaction WAL edit for deleted family:"<a name="line.5647"></a> |
| <span class="sourceLineNo">5648</span> + Bytes.toString(compaction.getFamilyName().toByteArray()));<a name="line.5648"></a> |
| <span class="sourceLineNo">5649</span> return;<a name="line.5649"></a> |
| <span class="sourceLineNo">5650</span> }<a name="line.5650"></a> |
| <span class="sourceLineNo">5651</span> store.replayCompactionMarker(compaction, pickCompactionFiles, removeFiles);<a name="line.5651"></a> |
| <span class="sourceLineNo">5652</span> logRegionFiles();<a name="line.5652"></a> |
| <span class="sourceLineNo">5653</span> } catch (FileNotFoundException ex) {<a name="line.5653"></a> |
| <span class="sourceLineNo">5654</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5654"></a> |
| <span class="sourceLineNo">5655</span> + "At least one of the store files in compaction: "<a name="line.5655"></a> |
| <span class="sourceLineNo">5656</span> + TextFormat.shortDebugString(compaction)<a name="line.5656"></a> |
| <span class="sourceLineNo">5657</span> + " doesn't exist any more. Skip loading the file(s)", ex);<a name="line.5657"></a> |
| <span class="sourceLineNo">5658</span> } finally {<a name="line.5658"></a> |
| <span class="sourceLineNo">5659</span> closeRegionOperation(Operation.REPLAY_EVENT);<a name="line.5659"></a> |
| <span class="sourceLineNo">5660</span> }<a name="line.5660"></a> |
| <span class="sourceLineNo">5661</span> }<a name="line.5661"></a> |
| <span class="sourceLineNo">5662</span> }<a name="line.5662"></a> |
| <span class="sourceLineNo">5663</span><a name="line.5663"></a> |
| <span class="sourceLineNo">5664</span> void replayWALFlushMarker(FlushDescriptor flush, long replaySeqId) throws IOException {<a name="line.5664"></a> |
| <span class="sourceLineNo">5665</span> checkTargetRegion(flush.getEncodedRegionName().toByteArray(),<a name="line.5665"></a> |
| <span class="sourceLineNo">5666</span> "Flush marker from WAL ", flush);<a name="line.5666"></a> |
| <span class="sourceLineNo">5667</span><a name="line.5667"></a> |
| <span class="sourceLineNo">5668</span> if (ServerRegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {<a name="line.5668"></a> |
| <span class="sourceLineNo">5669</span> return; // if primary nothing to do<a name="line.5669"></a> |
| <span class="sourceLineNo">5670</span> }<a name="line.5670"></a> |
| <span class="sourceLineNo">5671</span><a name="line.5671"></a> |
| <span class="sourceLineNo">5672</span> if (LOG.isDebugEnabled()) {<a name="line.5672"></a> |
| <span class="sourceLineNo">5673</span> LOG.debug(getRegionInfo().getEncodedName() + " : "<a name="line.5673"></a> |
| <span class="sourceLineNo">5674</span> + "Replaying flush marker " + TextFormat.shortDebugString(flush));<a name="line.5674"></a> |
| <span class="sourceLineNo">5675</span> }<a name="line.5675"></a> |
| <span class="sourceLineNo">5676</span><a name="line.5676"></a> |
| <span class="sourceLineNo">5677</span> startRegionOperation(Operation.REPLAY_EVENT); // use region close lock to guard against close<a name="line.5677"></a> |
| <span class="sourceLineNo">5678</span> try {<a name="line.5678"></a> |
| <span class="sourceLineNo">5679</span> FlushAction action = flush.getAction();<a name="line.5679"></a> |
| <span class="sourceLineNo">5680</span> switch (action) {<a name="line.5680"></a> |
| <span class="sourceLineNo">5681</span> case START_FLUSH:<a name="line.5681"></a> |
| <span class="sourceLineNo">5682</span> replayWALFlushStartMarker(flush);<a name="line.5682"></a> |
| <span class="sourceLineNo">5683</span> break;<a name="line.5683"></a> |
| <span class="sourceLineNo">5684</span> case COMMIT_FLUSH:<a name="line.5684"></a> |
| <span class="sourceLineNo">5685</span> replayWALFlushCommitMarker(flush);<a name="line.5685"></a> |
| <span class="sourceLineNo">5686</span> break;<a name="line.5686"></a> |
| <span class="sourceLineNo">5687</span> case ABORT_FLUSH:<a name="line.5687"></a> |
| <span class="sourceLineNo">5688</span> replayWALFlushAbortMarker(flush);<a name="line.5688"></a> |
| <span class="sourceLineNo">5689</span> break;<a name="line.5689"></a> |
| <span class="sourceLineNo">5690</span> case CANNOT_FLUSH:<a name="line.5690"></a> |
| <span class="sourceLineNo">5691</span> replayWALFlushCannotFlushMarker(flush, replaySeqId);<a name="line.5691"></a> |
| <span class="sourceLineNo">5692</span> break;<a name="line.5692"></a> |
| <span class="sourceLineNo">5693</span> default:<a name="line.5693"></a> |
| <span class="sourceLineNo">5694</span> LOG.warn(getRegionInfo().getEncodedName() + " : " +<a name="line.5694"></a> |
| <span class="sourceLineNo">5695</span> "Received a flush event with unknown action, ignoring. " +<a name="line.5695"></a> |
| <span class="sourceLineNo">5696</span> TextFormat.shortDebugString(flush));<a name="line.5696"></a> |
| <span class="sourceLineNo">5697</span> break;<a name="line.5697"></a> |
| <span class="sourceLineNo">5698</span> }<a name="line.5698"></a> |
| <span class="sourceLineNo">5699</span><a name="line.5699"></a> |
| <span class="sourceLineNo">5700</span> logRegionFiles();<a name="line.5700"></a> |
| <span class="sourceLineNo">5701</span> } finally {<a name="line.5701"></a> |
| <span class="sourceLineNo">5702</span> closeRegionOperation(Operation.REPLAY_EVENT);<a name="line.5702"></a> |
| <span class="sourceLineNo">5703</span> }<a name="line.5703"></a> |
| <span class="sourceLineNo">5704</span> }<a name="line.5704"></a> |
| <span class="sourceLineNo">5705</span><a name="line.5705"></a> |
| <span class="sourceLineNo">5706</span> /** Replay the flush marker from primary region by creating a corresponding snapshot of<a name="line.5706"></a> |
| <span class="sourceLineNo">5707</span> * the store memstores, only if the memstores do not have a higher seqId from an earlier wal<a name="line.5707"></a> |
| <span class="sourceLineNo">5708</span> * edit (because the events may be coming out of order).<a name="line.5708"></a> |
| <span class="sourceLineNo">5709</span> */<a name="line.5709"></a> |
| <span class="sourceLineNo">5710</span> PrepareFlushResult replayWALFlushStartMarker(FlushDescriptor flush) throws IOException {<a name="line.5710"></a> |
| <span class="sourceLineNo">5711</span> long flushSeqId = flush.getFlushSequenceNumber();<a name="line.5711"></a> |
| <span class="sourceLineNo">5712</span><a name="line.5712"></a> |
| <span class="sourceLineNo">5713</span> HashSet<HStore> storesToFlush = new HashSet<>();<a name="line.5713"></a> |
| <span class="sourceLineNo">5714</span> for (StoreFlushDescriptor storeFlush : flush.getStoreFlushesList()) {<a name="line.5714"></a> |
| <span class="sourceLineNo">5715</span> byte[] family = storeFlush.getFamilyName().toByteArray();<a name="line.5715"></a> |
| <span class="sourceLineNo">5716</span> HStore store = getStore(family);<a name="line.5716"></a> |
| <span class="sourceLineNo">5717</span> if (store == null) {<a name="line.5717"></a> |
| <span class="sourceLineNo">5718</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5718"></a> |
| <span class="sourceLineNo">5719</span> + "Received a flush start marker from primary, but the family is not found. Ignoring"<a name="line.5719"></a> |
| <span class="sourceLineNo">5720</span> + " StoreFlushDescriptor:" + TextFormat.shortDebugString(storeFlush));<a name="line.5720"></a> |
| <span class="sourceLineNo">5721</span> continue;<a name="line.5721"></a> |
| <span class="sourceLineNo">5722</span> }<a name="line.5722"></a> |
| <span class="sourceLineNo">5723</span> storesToFlush.add(store);<a name="line.5723"></a> |
| <span class="sourceLineNo">5724</span> }<a name="line.5724"></a> |
| <span class="sourceLineNo">5725</span><a name="line.5725"></a> |
| <span class="sourceLineNo">5726</span> MonitoredTask status = TaskMonitor.get().createStatus("Preparing flush " + this);<a name="line.5726"></a> |
| <span class="sourceLineNo">5727</span><a name="line.5727"></a> |
| <span class="sourceLineNo">5728</span> // we will use writestate as a coarse-grain lock for all the replay events<a name="line.5728"></a> |
| <span class="sourceLineNo">5729</span> // (flush, compaction, region open etc)<a name="line.5729"></a> |
| <span class="sourceLineNo">5730</span> synchronized (writestate) {<a name="line.5730"></a> |
| <span class="sourceLineNo">5731</span> try {<a name="line.5731"></a> |
| <span class="sourceLineNo">5732</span> if (flush.getFlushSequenceNumber() < lastReplayedOpenRegionSeqId) {<a name="line.5732"></a> |
| <span class="sourceLineNo">5733</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5733"></a> |
| <span class="sourceLineNo">5734</span> + "Skipping replaying flush event :" + TextFormat.shortDebugString(flush)<a name="line.5734"></a> |
| <span class="sourceLineNo">5735</span> + " because its sequence id is smaller than this regions lastReplayedOpenRegionSeqId "<a name="line.5735"></a> |
| <span class="sourceLineNo">5736</span> + " of " + lastReplayedOpenRegionSeqId);<a name="line.5736"></a> |
| <span class="sourceLineNo">5737</span> return null;<a name="line.5737"></a> |
| <span class="sourceLineNo">5738</span> }<a name="line.5738"></a> |
| <span class="sourceLineNo">5739</span> if (numMutationsWithoutWAL.sum() > 0) {<a name="line.5739"></a> |
| <span class="sourceLineNo">5740</span> numMutationsWithoutWAL.reset();<a name="line.5740"></a> |
| <span class="sourceLineNo">5741</span> dataInMemoryWithoutWAL.reset();<a name="line.5741"></a> |
| <span class="sourceLineNo">5742</span> }<a name="line.5742"></a> |
| <span class="sourceLineNo">5743</span><a name="line.5743"></a> |
| <span class="sourceLineNo">5744</span> if (!writestate.flushing) {<a name="line.5744"></a> |
| <span class="sourceLineNo">5745</span> // we do not have an active snapshot and corresponding this.prepareResult. This means<a name="line.5745"></a> |
| <span class="sourceLineNo">5746</span> // we can just snapshot our memstores and continue as normal.<a name="line.5746"></a> |
| <span class="sourceLineNo">5747</span><a name="line.5747"></a> |
| <span class="sourceLineNo">5748</span> // invoke prepareFlushCache. Send null as wal since we do not want the flush events in wal<a name="line.5748"></a> |
| <span class="sourceLineNo">5749</span> PrepareFlushResult prepareResult = internalPrepareFlushCache(null, flushSeqId,<a name="line.5749"></a> |
| <span class="sourceLineNo">5750</span> storesToFlush, status, false, FlushLifeCycleTracker.DUMMY);<a name="line.5750"></a> |
| <span class="sourceLineNo">5751</span> if (prepareResult.result == null) {<a name="line.5751"></a> |
| <span class="sourceLineNo">5752</span> // save the PrepareFlushResult so that we can use it later from commit flush<a name="line.5752"></a> |
| <span class="sourceLineNo">5753</span> this.writestate.flushing = true;<a name="line.5753"></a> |
| <span class="sourceLineNo">5754</span> this.prepareFlushResult = prepareResult;<a name="line.5754"></a> |
| <span class="sourceLineNo">5755</span> status.markComplete("Flush prepare successful");<a name="line.5755"></a> |
| <span class="sourceLineNo">5756</span> if (LOG.isDebugEnabled()) {<a name="line.5756"></a> |
| <span class="sourceLineNo">5757</span> LOG.debug(getRegionInfo().getEncodedName() + " : "<a name="line.5757"></a> |
| <span class="sourceLineNo">5758</span> + " Prepared flush with seqId:" + flush.getFlushSequenceNumber());<a name="line.5758"></a> |
| <span class="sourceLineNo">5759</span> }<a name="line.5759"></a> |
| <span class="sourceLineNo">5760</span> } else {<a name="line.5760"></a> |
| <span class="sourceLineNo">5761</span> // special case empty memstore. We will still save the flush result in this case, since<a name="line.5761"></a> |
| <span class="sourceLineNo">5762</span> // our memstore ie empty, but the primary is still flushing<a name="line.5762"></a> |
| <span class="sourceLineNo">5763</span> if (prepareResult.getResult().getResult() ==<a name="line.5763"></a> |
| <span class="sourceLineNo">5764</span> FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY) {<a name="line.5764"></a> |
| <span class="sourceLineNo">5765</span> this.writestate.flushing = true;<a name="line.5765"></a> |
| <span class="sourceLineNo">5766</span> this.prepareFlushResult = prepareResult;<a name="line.5766"></a> |
| <span class="sourceLineNo">5767</span> if (LOG.isDebugEnabled()) {<a name="line.5767"></a> |
| <span class="sourceLineNo">5768</span> LOG.debug(getRegionInfo().getEncodedName() + " : "<a name="line.5768"></a> |
| <span class="sourceLineNo">5769</span> + " Prepared empty flush with seqId:" + flush.getFlushSequenceNumber());<a name="line.5769"></a> |
| <span class="sourceLineNo">5770</span> }<a name="line.5770"></a> |
| <span class="sourceLineNo">5771</span> }<a name="line.5771"></a> |
| <span class="sourceLineNo">5772</span> status.abort("Flush prepare failed with " + prepareResult.result);<a name="line.5772"></a> |
| <span class="sourceLineNo">5773</span> // nothing much to do. prepare flush failed because of some reason.<a name="line.5773"></a> |
| <span class="sourceLineNo">5774</span> }<a name="line.5774"></a> |
| <span class="sourceLineNo">5775</span> return prepareResult;<a name="line.5775"></a> |
| <span class="sourceLineNo">5776</span> } else {<a name="line.5776"></a> |
| <span class="sourceLineNo">5777</span> // we already have an active snapshot.<a name="line.5777"></a> |
| <span class="sourceLineNo">5778</span> if (flush.getFlushSequenceNumber() == this.prepareFlushResult.flushOpSeqId) {<a name="line.5778"></a> |
| <span class="sourceLineNo">5779</span> // They define the same flush. Log and continue.<a name="line.5779"></a> |
| <span class="sourceLineNo">5780</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5780"></a> |
| <span class="sourceLineNo">5781</span> + "Received a flush prepare marker with the same seqId: " +<a name="line.5781"></a> |
| <span class="sourceLineNo">5782</span> + flush.getFlushSequenceNumber() + " before clearing the previous one with seqId: "<a name="line.5782"></a> |
| <span class="sourceLineNo">5783</span> + prepareFlushResult.flushOpSeqId + ". Ignoring");<a name="line.5783"></a> |
| <span class="sourceLineNo">5784</span> // ignore<a name="line.5784"></a> |
| <span class="sourceLineNo">5785</span> } else if (flush.getFlushSequenceNumber() < this.prepareFlushResult.flushOpSeqId) {<a name="line.5785"></a> |
| <span class="sourceLineNo">5786</span> // We received a flush with a smaller seqNum than what we have prepared. We can only<a name="line.5786"></a> |
| <span class="sourceLineNo">5787</span> // ignore this prepare flush request.<a name="line.5787"></a> |
| <span class="sourceLineNo">5788</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5788"></a> |
| <span class="sourceLineNo">5789</span> + "Received a flush prepare marker with a smaller seqId: " +<a name="line.5789"></a> |
| <span class="sourceLineNo">5790</span> + flush.getFlushSequenceNumber() + " before clearing the previous one with seqId: "<a name="line.5790"></a> |
| <span class="sourceLineNo">5791</span> + prepareFlushResult.flushOpSeqId + ". Ignoring");<a name="line.5791"></a> |
| <span class="sourceLineNo">5792</span> // ignore<a name="line.5792"></a> |
| <span class="sourceLineNo">5793</span> } else {<a name="line.5793"></a> |
| <span class="sourceLineNo">5794</span> // We received a flush with a larger seqNum than what we have prepared<a name="line.5794"></a> |
| <span class="sourceLineNo">5795</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5795"></a> |
| <span class="sourceLineNo">5796</span> + "Received a flush prepare marker with a larger seqId: " +<a name="line.5796"></a> |
| <span class="sourceLineNo">5797</span> + flush.getFlushSequenceNumber() + " before clearing the previous one with seqId: "<a name="line.5797"></a> |
| <span class="sourceLineNo">5798</span> + prepareFlushResult.flushOpSeqId + ". Ignoring");<a name="line.5798"></a> |
| <span class="sourceLineNo">5799</span> // We do not have multiple active snapshots in the memstore or a way to merge current<a name="line.5799"></a> |
| <span class="sourceLineNo">5800</span> // memstore snapshot with the contents and resnapshot for now. We cannot take<a name="line.5800"></a> |
| <span class="sourceLineNo">5801</span> // another snapshot and drop the previous one because that will cause temporary<a name="line.5801"></a> |
| <span class="sourceLineNo">5802</span> // data loss in the secondary. So we ignore this for now, deferring the resolution<a name="line.5802"></a> |
| <span class="sourceLineNo">5803</span> // to happen when we see the corresponding flush commit marker. If we have a memstore<a name="line.5803"></a> |
| <span class="sourceLineNo">5804</span> // snapshot with x, and later received another prepare snapshot with y (where x < y),<a name="line.5804"></a> |
| <span class="sourceLineNo">5805</span> // when we see flush commit for y, we will drop snapshot for x, and can also drop all<a name="line.5805"></a> |
| <span class="sourceLineNo">5806</span> // the memstore edits if everything in memstore is < y. This is the usual case for<a name="line.5806"></a> |
| <span class="sourceLineNo">5807</span> // RS crash + recovery where we might see consequtive prepare flush wal markers.<a name="line.5807"></a> |
| <span class="sourceLineNo">5808</span> // Otherwise, this will cause more memory to be used in secondary replica until a<a name="line.5808"></a> |
| <span class="sourceLineNo">5809</span> // further prapare + commit flush is seen and replayed.<a name="line.5809"></a> |
| <span class="sourceLineNo">5810</span> }<a name="line.5810"></a> |
| <span class="sourceLineNo">5811</span> }<a name="line.5811"></a> |
| <span class="sourceLineNo">5812</span> } finally {<a name="line.5812"></a> |
| <span class="sourceLineNo">5813</span> status.cleanup();<a name="line.5813"></a> |
| <span class="sourceLineNo">5814</span> writestate.notifyAll();<a name="line.5814"></a> |
| <span class="sourceLineNo">5815</span> }<a name="line.5815"></a> |
| <span class="sourceLineNo">5816</span> }<a name="line.5816"></a> |
| <span class="sourceLineNo">5817</span> return null;<a name="line.5817"></a> |
| <span class="sourceLineNo">5818</span> }<a name="line.5818"></a> |
| <span class="sourceLineNo">5819</span><a name="line.5819"></a> |
| <span class="sourceLineNo">5820</span> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NN_NAKED_NOTIFY",<a name="line.5820"></a> |
| <span class="sourceLineNo">5821</span> justification="Intentional; post memstore flush")<a name="line.5821"></a> |
| <span class="sourceLineNo">5822</span> void replayWALFlushCommitMarker(FlushDescriptor flush) throws IOException {<a name="line.5822"></a> |
| <span class="sourceLineNo">5823</span> MonitoredTask status = TaskMonitor.get().createStatus("Committing flush " + this);<a name="line.5823"></a> |
| <span class="sourceLineNo">5824</span><a name="line.5824"></a> |
| <span class="sourceLineNo">5825</span> // check whether we have the memstore snapshot with the corresponding seqId. Replay to<a name="line.5825"></a> |
| <span class="sourceLineNo">5826</span> // secondary region replicas are in order, except for when the region moves or then the<a name="line.5826"></a> |
| <span class="sourceLineNo">5827</span> // region server crashes. In those cases, we may receive replay requests out of order from<a name="line.5827"></a> |
| <span class="sourceLineNo">5828</span> // the original seqIds.<a name="line.5828"></a> |
| <span class="sourceLineNo">5829</span> synchronized (writestate) {<a name="line.5829"></a> |
| <span class="sourceLineNo">5830</span> try {<a name="line.5830"></a> |
| <span class="sourceLineNo">5831</span> if (flush.getFlushSequenceNumber() < lastReplayedOpenRegionSeqId) {<a name="line.5831"></a> |
| <span class="sourceLineNo">5832</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5832"></a> |
| <span class="sourceLineNo">5833</span> + "Skipping replaying flush event :" + TextFormat.shortDebugString(flush)<a name="line.5833"></a> |
| <span class="sourceLineNo">5834</span> + " because its sequence id is smaller than this regions lastReplayedOpenRegionSeqId "<a name="line.5834"></a> |
| <span class="sourceLineNo">5835</span> + " of " + lastReplayedOpenRegionSeqId);<a name="line.5835"></a> |
| <span class="sourceLineNo">5836</span> return;<a name="line.5836"></a> |
| <span class="sourceLineNo">5837</span> }<a name="line.5837"></a> |
| <span class="sourceLineNo">5838</span><a name="line.5838"></a> |
| <span class="sourceLineNo">5839</span> if (writestate.flushing) {<a name="line.5839"></a> |
| <span class="sourceLineNo">5840</span> PrepareFlushResult prepareFlushResult = this.prepareFlushResult;<a name="line.5840"></a> |
| <span class="sourceLineNo">5841</span> if (flush.getFlushSequenceNumber() == prepareFlushResult.flushOpSeqId) {<a name="line.5841"></a> |
| <span class="sourceLineNo">5842</span> if (LOG.isDebugEnabled()) {<a name="line.5842"></a> |
| <span class="sourceLineNo">5843</span> LOG.debug(getRegionInfo().getEncodedName() + " : "<a name="line.5843"></a> |
| <span class="sourceLineNo">5844</span> + "Received a flush commit marker with seqId:" + flush.getFlushSequenceNumber()<a name="line.5844"></a> |
| <span class="sourceLineNo">5845</span> + " and a previous prepared snapshot was found");<a name="line.5845"></a> |
| <span class="sourceLineNo">5846</span> }<a name="line.5846"></a> |
| <span class="sourceLineNo">5847</span> // This is the regular case where we received commit flush after prepare flush<a name="line.5847"></a> |
| <span class="sourceLineNo">5848</span> // corresponding to the same seqId.<a name="line.5848"></a> |
| <span class="sourceLineNo">5849</span> replayFlushInStores(flush, prepareFlushResult, true);<a name="line.5849"></a> |
| <span class="sourceLineNo">5850</span><a name="line.5850"></a> |
| <span class="sourceLineNo">5851</span> // Set down the memstore size by amount of flush.<a name="line.5851"></a> |
| <span class="sourceLineNo">5852</span> this.decrMemStoreSize(prepareFlushResult.totalFlushableSize.getMemStoreSize());<a name="line.5852"></a> |
| <span class="sourceLineNo">5853</span> this.prepareFlushResult = null;<a name="line.5853"></a> |
| <span class="sourceLineNo">5854</span> writestate.flushing = false;<a name="line.5854"></a> |
| <span class="sourceLineNo">5855</span> } else if (flush.getFlushSequenceNumber() < prepareFlushResult.flushOpSeqId) {<a name="line.5855"></a> |
| <span class="sourceLineNo">5856</span> // This should not happen normally. However, lets be safe and guard against these cases<a name="line.5856"></a> |
| <span class="sourceLineNo">5857</span> // we received a flush commit with a smaller seqId than what we have prepared<a name="line.5857"></a> |
| <span class="sourceLineNo">5858</span> // we will pick the flush file up from this commit (if we have not seen it), but we<a name="line.5858"></a> |
| <span class="sourceLineNo">5859</span> // will not drop the memstore<a name="line.5859"></a> |
| <span class="sourceLineNo">5860</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5860"></a> |
| <span class="sourceLineNo">5861</span> + "Received a flush commit marker with smaller seqId: "<a name="line.5861"></a> |
| <span class="sourceLineNo">5862</span> + flush.getFlushSequenceNumber() + " than what we have prepared with seqId: "<a name="line.5862"></a> |
| <span class="sourceLineNo">5863</span> + prepareFlushResult.flushOpSeqId + ". Picking up new file, but not dropping"<a name="line.5863"></a> |
| <span class="sourceLineNo">5864</span> +" prepared memstore snapshot");<a name="line.5864"></a> |
| <span class="sourceLineNo">5865</span> replayFlushInStores(flush, prepareFlushResult, false);<a name="line.5865"></a> |
| <span class="sourceLineNo">5866</span><a name="line.5866"></a> |
| <span class="sourceLineNo">5867</span> // snapshot is not dropped, so memstore sizes should not be decremented<a name="line.5867"></a> |
| <span class="sourceLineNo">5868</span> // we still have the prepared snapshot, flushing should still be true<a name="line.5868"></a> |
| <span class="sourceLineNo">5869</span> } else {<a name="line.5869"></a> |
| <span class="sourceLineNo">5870</span> // This should not happen normally. However, lets be safe and guard against these cases<a name="line.5870"></a> |
| <span class="sourceLineNo">5871</span> // we received a flush commit with a larger seqId than what we have prepared<a name="line.5871"></a> |
| <span class="sourceLineNo">5872</span> // we will pick the flush file for this. We will also obtain the updates lock and<a name="line.5872"></a> |
| <span class="sourceLineNo">5873</span> // look for contents of the memstore to see whether we have edits after this seqId.<a name="line.5873"></a> |
| <span class="sourceLineNo">5874</span> // If not, we will drop all the memstore edits and the snapshot as well.<a name="line.5874"></a> |
| <span class="sourceLineNo">5875</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5875"></a> |
| <span class="sourceLineNo">5876</span> + "Received a flush commit marker with larger seqId: "<a name="line.5876"></a> |
| <span class="sourceLineNo">5877</span> + flush.getFlushSequenceNumber() + " than what we have prepared with seqId: " +<a name="line.5877"></a> |
| <span class="sourceLineNo">5878</span> prepareFlushResult.flushOpSeqId + ". Picking up new file and dropping prepared"<a name="line.5878"></a> |
| <span class="sourceLineNo">5879</span> +" memstore snapshot");<a name="line.5879"></a> |
| <span class="sourceLineNo">5880</span><a name="line.5880"></a> |
| <span class="sourceLineNo">5881</span> replayFlushInStores(flush, prepareFlushResult, true);<a name="line.5881"></a> |
| <span class="sourceLineNo">5882</span><a name="line.5882"></a> |
| <span class="sourceLineNo">5883</span> // Set down the memstore size by amount of flush.<a name="line.5883"></a> |
| <span class="sourceLineNo">5884</span> this.decrMemStoreSize(prepareFlushResult.totalFlushableSize.getMemStoreSize());<a name="line.5884"></a> |
| <span class="sourceLineNo">5885</span><a name="line.5885"></a> |
| <span class="sourceLineNo">5886</span> // Inspect the memstore contents to see whether the memstore contains only edits<a name="line.5886"></a> |
| <span class="sourceLineNo">5887</span> // with seqId smaller than the flush seqId. If so, we can discard those edits.<a name="line.5887"></a> |
| <span class="sourceLineNo">5888</span> dropMemStoreContentsForSeqId(flush.getFlushSequenceNumber(), null);<a name="line.5888"></a> |
| <span class="sourceLineNo">5889</span><a name="line.5889"></a> |
| <span class="sourceLineNo">5890</span> this.prepareFlushResult = null;<a name="line.5890"></a> |
| <span class="sourceLineNo">5891</span> writestate.flushing = false;<a name="line.5891"></a> |
| <span class="sourceLineNo">5892</span> }<a name="line.5892"></a> |
| <span class="sourceLineNo">5893</span> // If we were waiting for observing a flush or region opening event for not showing<a name="line.5893"></a> |
| <span class="sourceLineNo">5894</span> // partial data after a secondary region crash, we can allow reads now. We can only make<a name="line.5894"></a> |
| <span class="sourceLineNo">5895</span> // sure that we are not showing partial data (for example skipping some previous edits)<a name="line.5895"></a> |
| <span class="sourceLineNo">5896</span> // until we observe a full flush start and flush commit. So if we were not able to find<a name="line.5896"></a> |
| <span class="sourceLineNo">5897</span> // a previous flush we will not enable reads now.<a name="line.5897"></a> |
| <span class="sourceLineNo">5898</span> this.setReadsEnabled(true);<a name="line.5898"></a> |
| <span class="sourceLineNo">5899</span> } else {<a name="line.5899"></a> |
| <span class="sourceLineNo">5900</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5900"></a> |
| <span class="sourceLineNo">5901</span> + "Received a flush commit marker with seqId:" + flush.getFlushSequenceNumber()<a name="line.5901"></a> |
| <span class="sourceLineNo">5902</span> + ", but no previous prepared snapshot was found");<a name="line.5902"></a> |
| <span class="sourceLineNo">5903</span> // There is no corresponding prepare snapshot from before.<a name="line.5903"></a> |
| <span class="sourceLineNo">5904</span> // We will pick up the new flushed file<a name="line.5904"></a> |
| <span class="sourceLineNo">5905</span> replayFlushInStores(flush, null, false);<a name="line.5905"></a> |
| <span class="sourceLineNo">5906</span><a name="line.5906"></a> |
| <span class="sourceLineNo">5907</span> // Inspect the memstore contents to see whether the memstore contains only edits<a name="line.5907"></a> |
| <span class="sourceLineNo">5908</span> // with seqId smaller than the flush seqId. If so, we can discard those edits.<a name="line.5908"></a> |
| <span class="sourceLineNo">5909</span> dropMemStoreContentsForSeqId(flush.getFlushSequenceNumber(), null);<a name="line.5909"></a> |
| <span class="sourceLineNo">5910</span> }<a name="line.5910"></a> |
| <span class="sourceLineNo">5911</span><a name="line.5911"></a> |
| <span class="sourceLineNo">5912</span> status.markComplete("Flush commit successful");<a name="line.5912"></a> |
| <span class="sourceLineNo">5913</span><a name="line.5913"></a> |
| <span class="sourceLineNo">5914</span> // Update the last flushed sequence id for region.<a name="line.5914"></a> |
| <span class="sourceLineNo">5915</span> this.maxFlushedSeqId = flush.getFlushSequenceNumber();<a name="line.5915"></a> |
| <span class="sourceLineNo">5916</span><a name="line.5916"></a> |
| <span class="sourceLineNo">5917</span> // advance the mvcc read point so that the new flushed file is visible.<a name="line.5917"></a> |
| <span class="sourceLineNo">5918</span> mvcc.advanceTo(flush.getFlushSequenceNumber());<a name="line.5918"></a> |
| <span class="sourceLineNo">5919</span><a name="line.5919"></a> |
| <span class="sourceLineNo">5920</span> } catch (FileNotFoundException ex) {<a name="line.5920"></a> |
| <span class="sourceLineNo">5921</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5921"></a> |
| <span class="sourceLineNo">5922</span> + "At least one of the store files in flush: " + TextFormat.shortDebugString(flush)<a name="line.5922"></a> |
| <span class="sourceLineNo">5923</span> + " doesn't exist any more. Skip loading the file(s)", ex);<a name="line.5923"></a> |
| <span class="sourceLineNo">5924</span> }<a name="line.5924"></a> |
| <span class="sourceLineNo">5925</span> finally {<a name="line.5925"></a> |
| <span class="sourceLineNo">5926</span> status.cleanup();<a name="line.5926"></a> |
| <span class="sourceLineNo">5927</span> writestate.notifyAll();<a name="line.5927"></a> |
| <span class="sourceLineNo">5928</span> }<a name="line.5928"></a> |
| <span class="sourceLineNo">5929</span> }<a name="line.5929"></a> |
| <span class="sourceLineNo">5930</span><a name="line.5930"></a> |
| <span class="sourceLineNo">5931</span> // C. Finally notify anyone waiting on memstore to clear:<a name="line.5931"></a> |
| <span class="sourceLineNo">5932</span> // e.g. checkResources().<a name="line.5932"></a> |
| <span class="sourceLineNo">5933</span> synchronized (this) {<a name="line.5933"></a> |
| <span class="sourceLineNo">5934</span> notifyAll(); // FindBugs NN_NAKED_NOTIFY<a name="line.5934"></a> |
| <span class="sourceLineNo">5935</span> }<a name="line.5935"></a> |
| <span class="sourceLineNo">5936</span> }<a name="line.5936"></a> |
| <span class="sourceLineNo">5937</span><a name="line.5937"></a> |
| <span class="sourceLineNo">5938</span> /**<a name="line.5938"></a> |
| <span class="sourceLineNo">5939</span> * Replays the given flush descriptor by opening the flush files in stores and dropping the<a name="line.5939"></a> |
| <span class="sourceLineNo">5940</span> * memstore snapshots if requested.<a name="line.5940"></a> |
| <span class="sourceLineNo">5941</span> * @param flush<a name="line.5941"></a> |
| <span class="sourceLineNo">5942</span> * @param prepareFlushResult<a name="line.5942"></a> |
| <span class="sourceLineNo">5943</span> * @param dropMemstoreSnapshot<a name="line.5943"></a> |
| <span class="sourceLineNo">5944</span> * @throws IOException<a name="line.5944"></a> |
| <span class="sourceLineNo">5945</span> */<a name="line.5945"></a> |
| <span class="sourceLineNo">5946</span> private void replayFlushInStores(FlushDescriptor flush, PrepareFlushResult prepareFlushResult,<a name="line.5946"></a> |
| <span class="sourceLineNo">5947</span> boolean dropMemstoreSnapshot)<a name="line.5947"></a> |
| <span class="sourceLineNo">5948</span> throws IOException {<a name="line.5948"></a> |
| <span class="sourceLineNo">5949</span> for (StoreFlushDescriptor storeFlush : flush.getStoreFlushesList()) {<a name="line.5949"></a> |
| <span class="sourceLineNo">5950</span> byte[] family = storeFlush.getFamilyName().toByteArray();<a name="line.5950"></a> |
| <span class="sourceLineNo">5951</span> HStore store = getStore(family);<a name="line.5951"></a> |
| <span class="sourceLineNo">5952</span> if (store == null) {<a name="line.5952"></a> |
| <span class="sourceLineNo">5953</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5953"></a> |
| <span class="sourceLineNo">5954</span> + "Received a flush commit marker from primary, but the family is not found."<a name="line.5954"></a> |
| <span class="sourceLineNo">5955</span> + "Ignoring StoreFlushDescriptor:" + storeFlush);<a name="line.5955"></a> |
| <span class="sourceLineNo">5956</span> continue;<a name="line.5956"></a> |
| <span class="sourceLineNo">5957</span> }<a name="line.5957"></a> |
| <span class="sourceLineNo">5958</span> List<String> flushFiles = storeFlush.getFlushOutputList();<a name="line.5958"></a> |
| <span class="sourceLineNo">5959</span> StoreFlushContext ctx = null;<a name="line.5959"></a> |
| <span class="sourceLineNo">5960</span> long startTime = EnvironmentEdgeManager.currentTime();<a name="line.5960"></a> |
| <span class="sourceLineNo">5961</span> if (prepareFlushResult == null || prepareFlushResult.storeFlushCtxs == null) {<a name="line.5961"></a> |
| <span class="sourceLineNo">5962</span> ctx = store.createFlushContext(flush.getFlushSequenceNumber(), FlushLifeCycleTracker.DUMMY);<a name="line.5962"></a> |
| <span class="sourceLineNo">5963</span> } else {<a name="line.5963"></a> |
| <span class="sourceLineNo">5964</span> ctx = prepareFlushResult.storeFlushCtxs.get(family);<a name="line.5964"></a> |
| <span class="sourceLineNo">5965</span> startTime = prepareFlushResult.startTime;<a name="line.5965"></a> |
| <span class="sourceLineNo">5966</span> }<a name="line.5966"></a> |
| <span class="sourceLineNo">5967</span><a name="line.5967"></a> |
| <span class="sourceLineNo">5968</span> if (ctx == null) {<a name="line.5968"></a> |
| <span class="sourceLineNo">5969</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.5969"></a> |
| <span class="sourceLineNo">5970</span> + "Unexpected: flush commit marker received from store "<a name="line.5970"></a> |
| <span class="sourceLineNo">5971</span> + Bytes.toString(family) + " but no associated flush context. Ignoring");<a name="line.5971"></a> |
| <span class="sourceLineNo">5972</span> continue;<a name="line.5972"></a> |
| <span class="sourceLineNo">5973</span> }<a name="line.5973"></a> |
| <span class="sourceLineNo">5974</span><a name="line.5974"></a> |
| <span class="sourceLineNo">5975</span> ctx.replayFlush(flushFiles, dropMemstoreSnapshot); // replay the flush<a name="line.5975"></a> |
| <span class="sourceLineNo">5976</span><a name="line.5976"></a> |
| <span class="sourceLineNo">5977</span> // Record latest flush time<a name="line.5977"></a> |
| <span class="sourceLineNo">5978</span> this.lastStoreFlushTimeMap.put(store, startTime);<a name="line.5978"></a> |
| <span class="sourceLineNo">5979</span> }<a name="line.5979"></a> |
| <span class="sourceLineNo">5980</span> }<a name="line.5980"></a> |
| <span class="sourceLineNo">5981</span><a name="line.5981"></a> |
| <span class="sourceLineNo">5982</span> private long loadRecoveredHFilesIfAny(Collection<HStore> stores) throws IOException {<a name="line.5982"></a> |
| <span class="sourceLineNo">5983</span> Path regionDir = fs.getRegionDir();<a name="line.5983"></a> |
| <span class="sourceLineNo">5984</span> long maxSeqId = -1;<a name="line.5984"></a> |
| <span class="sourceLineNo">5985</span> for (HStore store : stores) {<a name="line.5985"></a> |
| <span class="sourceLineNo">5986</span> String familyName = store.getColumnFamilyName();<a name="line.5986"></a> |
| <span class="sourceLineNo">5987</span> FileStatus[] files =<a name="line.5987"></a> |
| <span class="sourceLineNo">5988</span> WALSplitUtil.getRecoveredHFiles(fs.getFileSystem(), regionDir, familyName);<a name="line.5988"></a> |
| <span class="sourceLineNo">5989</span> if (files != null && files.length != 0) {<a name="line.5989"></a> |
| <span class="sourceLineNo">5990</span> for (FileStatus file : files) {<a name="line.5990"></a> |
| <span class="sourceLineNo">5991</span> Path filePath = file.getPath();<a name="line.5991"></a> |
| <span class="sourceLineNo">5992</span> // If file length is zero then delete it<a name="line.5992"></a> |
| <span class="sourceLineNo">5993</span> if (isZeroLengthThenDelete(fs.getFileSystem(), file, filePath)) {<a name="line.5993"></a> |
| <span class="sourceLineNo">5994</span> continue;<a name="line.5994"></a> |
| <span class="sourceLineNo">5995</span> }<a name="line.5995"></a> |
| <span class="sourceLineNo">5996</span> try {<a name="line.5996"></a> |
| <span class="sourceLineNo">5997</span> HStoreFile storefile = store.tryCommitRecoveredHFile(file.getPath());<a name="line.5997"></a> |
| <span class="sourceLineNo">5998</span> maxSeqId = Math.max(maxSeqId, storefile.getReader().getSequenceID());<a name="line.5998"></a> |
| <span class="sourceLineNo">5999</span> } catch (IOException e) {<a name="line.5999"></a> |
| <span class="sourceLineNo">6000</span> handleException(fs.getFileSystem(), filePath, e);<a name="line.6000"></a> |
| <span class="sourceLineNo">6001</span> continue;<a name="line.6001"></a> |
| <span class="sourceLineNo">6002</span> }<a name="line.6002"></a> |
| <span class="sourceLineNo">6003</span> }<a name="line.6003"></a> |
| <span class="sourceLineNo">6004</span> if (this.rsServices != null && store.needsCompaction()) {<a name="line.6004"></a> |
| <span class="sourceLineNo">6005</span> this.rsServices.getCompactionRequestor()<a name="line.6005"></a> |
| <span class="sourceLineNo">6006</span> .requestCompaction(this, store, "load recovered hfiles request compaction",<a name="line.6006"></a> |
| <span class="sourceLineNo">6007</span> Store.PRIORITY_USER + 1, CompactionLifeCycleTracker.DUMMY, null);<a name="line.6007"></a> |
| <span class="sourceLineNo">6008</span> }<a name="line.6008"></a> |
| <span class="sourceLineNo">6009</span> }<a name="line.6009"></a> |
| <span class="sourceLineNo">6010</span> }<a name="line.6010"></a> |
| <span class="sourceLineNo">6011</span> return maxSeqId;<a name="line.6011"></a> |
| <span class="sourceLineNo">6012</span> }<a name="line.6012"></a> |
| <span class="sourceLineNo">6013</span><a name="line.6013"></a> |
| <span class="sourceLineNo">6014</span> /**<a name="line.6014"></a> |
| <span class="sourceLineNo">6015</span> * Be careful, this method will drop all data in the memstore of this region.<a name="line.6015"></a> |
| <span class="sourceLineNo">6016</span> * Currently, this method is used to drop memstore to prevent memory leak<a name="line.6016"></a> |
| <span class="sourceLineNo">6017</span> * when replaying recovered.edits while opening region.<a name="line.6017"></a> |
| <span class="sourceLineNo">6018</span> */<a name="line.6018"></a> |
| <span class="sourceLineNo">6019</span> private MemStoreSize dropMemStoreContents() throws IOException {<a name="line.6019"></a> |
| <span class="sourceLineNo">6020</span> MemStoreSizing totalFreedSize = new NonThreadSafeMemStoreSizing();<a name="line.6020"></a> |
| <span class="sourceLineNo">6021</span> this.updatesLock.writeLock().lock();<a name="line.6021"></a> |
| <span class="sourceLineNo">6022</span> try {<a name="line.6022"></a> |
| <span class="sourceLineNo">6023</span> for (HStore s : stores.values()) {<a name="line.6023"></a> |
| <span class="sourceLineNo">6024</span> MemStoreSize memStoreSize = doDropStoreMemStoreContentsForSeqId(s, HConstants.NO_SEQNUM);<a name="line.6024"></a> |
| <span class="sourceLineNo">6025</span> LOG.info("Drop memstore for Store " + s.getColumnFamilyName() + " in region "<a name="line.6025"></a> |
| <span class="sourceLineNo">6026</span> + this.getRegionInfo().getRegionNameAsString()<a name="line.6026"></a> |
| <span class="sourceLineNo">6027</span> + " , dropped memstoresize: [" + memStoreSize + " }");<a name="line.6027"></a> |
| <span class="sourceLineNo">6028</span> totalFreedSize.incMemStoreSize(memStoreSize);<a name="line.6028"></a> |
| <span class="sourceLineNo">6029</span> }<a name="line.6029"></a> |
| <span class="sourceLineNo">6030</span> return totalFreedSize.getMemStoreSize();<a name="line.6030"></a> |
| <span class="sourceLineNo">6031</span> } finally {<a name="line.6031"></a> |
| <span class="sourceLineNo">6032</span> this.updatesLock.writeLock().unlock();<a name="line.6032"></a> |
| <span class="sourceLineNo">6033</span> }<a name="line.6033"></a> |
| <span class="sourceLineNo">6034</span> }<a name="line.6034"></a> |
| <span class="sourceLineNo">6035</span><a name="line.6035"></a> |
| <span class="sourceLineNo">6036</span> /**<a name="line.6036"></a> |
| <span class="sourceLineNo">6037</span> * Drops the memstore contents after replaying a flush descriptor or region open event replay<a name="line.6037"></a> |
| <span class="sourceLineNo">6038</span> * if the memstore edits have seqNums smaller than the given seq id<a name="line.6038"></a> |
| <span class="sourceLineNo">6039</span> * @throws IOException<a name="line.6039"></a> |
| <span class="sourceLineNo">6040</span> */<a name="line.6040"></a> |
| <span class="sourceLineNo">6041</span> private MemStoreSize dropMemStoreContentsForSeqId(long seqId, HStore store) throws IOException {<a name="line.6041"></a> |
| <span class="sourceLineNo">6042</span> MemStoreSizing totalFreedSize = new NonThreadSafeMemStoreSizing();<a name="line.6042"></a> |
| <span class="sourceLineNo">6043</span> this.updatesLock.writeLock().lock();<a name="line.6043"></a> |
| <span class="sourceLineNo">6044</span> try {<a name="line.6044"></a> |
| <span class="sourceLineNo">6045</span><a name="line.6045"></a> |
| <span class="sourceLineNo">6046</span> long currentSeqId = mvcc.getReadPoint();<a name="line.6046"></a> |
| <span class="sourceLineNo">6047</span> if (seqId >= currentSeqId) {<a name="line.6047"></a> |
| <span class="sourceLineNo">6048</span> // then we can drop the memstore contents since everything is below this seqId<a name="line.6048"></a> |
| <span class="sourceLineNo">6049</span> LOG.info(getRegionInfo().getEncodedName() + " : "<a name="line.6049"></a> |
| <span class="sourceLineNo">6050</span> + "Dropping memstore contents as well since replayed flush seqId: "<a name="line.6050"></a> |
| <span class="sourceLineNo">6051</span> + seqId + " is greater than current seqId:" + currentSeqId);<a name="line.6051"></a> |
| <span class="sourceLineNo">6052</span><a name="line.6052"></a> |
| <span class="sourceLineNo">6053</span> // Prepare flush (take a snapshot) and then abort (drop the snapshot)<a name="line.6053"></a> |
| <span class="sourceLineNo">6054</span> if (store == null) {<a name="line.6054"></a> |
| <span class="sourceLineNo">6055</span> for (HStore s : stores.values()) {<a name="line.6055"></a> |
| <span class="sourceLineNo">6056</span> totalFreedSize.incMemStoreSize(doDropStoreMemStoreContentsForSeqId(s, currentSeqId));<a name="line.6056"></a> |
| <span class="sourceLineNo">6057</span> }<a name="line.6057"></a> |
| <span class="sourceLineNo">6058</span> } else {<a name="line.6058"></a> |
| <span class="sourceLineNo">6059</span> totalFreedSize.incMemStoreSize(doDropStoreMemStoreContentsForSeqId(store, currentSeqId));<a name="line.6059"></a> |
| <span class="sourceLineNo">6060</span> }<a name="line.6060"></a> |
| <span class="sourceLineNo">6061</span> } else {<a name="line.6061"></a> |
| <span class="sourceLineNo">6062</span> LOG.info(getRegionInfo().getEncodedName() + " : "<a name="line.6062"></a> |
| <span class="sourceLineNo">6063</span> + "Not dropping memstore contents since replayed flush seqId: "<a name="line.6063"></a> |
| <span class="sourceLineNo">6064</span> + seqId + " is smaller than current seqId:" + currentSeqId);<a name="line.6064"></a> |
| <span class="sourceLineNo">6065</span> }<a name="line.6065"></a> |
| <span class="sourceLineNo">6066</span> } finally {<a name="line.6066"></a> |
| <span class="sourceLineNo">6067</span> this.updatesLock.writeLock().unlock();<a name="line.6067"></a> |
| <span class="sourceLineNo">6068</span> }<a name="line.6068"></a> |
| <span class="sourceLineNo">6069</span> return totalFreedSize.getMemStoreSize();<a name="line.6069"></a> |
| <span class="sourceLineNo">6070</span> }<a name="line.6070"></a> |
| <span class="sourceLineNo">6071</span><a name="line.6071"></a> |
| <span class="sourceLineNo">6072</span> private MemStoreSize doDropStoreMemStoreContentsForSeqId(HStore s, long currentSeqId)<a name="line.6072"></a> |
| <span class="sourceLineNo">6073</span> throws IOException {<a name="line.6073"></a> |
| <span class="sourceLineNo">6074</span> MemStoreSize flushableSize = s.getFlushableSize();<a name="line.6074"></a> |
| <span class="sourceLineNo">6075</span> this.decrMemStoreSize(flushableSize);<a name="line.6075"></a> |
| <span class="sourceLineNo">6076</span> StoreFlushContext ctx = s.createFlushContext(currentSeqId, FlushLifeCycleTracker.DUMMY);<a name="line.6076"></a> |
| <span class="sourceLineNo">6077</span> ctx.prepare();<a name="line.6077"></a> |
| <span class="sourceLineNo">6078</span> ctx.abort();<a name="line.6078"></a> |
| <span class="sourceLineNo">6079</span> return flushableSize;<a name="line.6079"></a> |
| <span class="sourceLineNo">6080</span> }<a name="line.6080"></a> |
| <span class="sourceLineNo">6081</span><a name="line.6081"></a> |
| <span class="sourceLineNo">6082</span> private void replayWALFlushAbortMarker(FlushDescriptor flush) {<a name="line.6082"></a> |
| <span class="sourceLineNo">6083</span> // nothing to do for now. A flush abort will cause a RS abort which means that the region<a name="line.6083"></a> |
| <span class="sourceLineNo">6084</span> // will be opened somewhere else later. We will see the region open event soon, and replaying<a name="line.6084"></a> |
| <span class="sourceLineNo">6085</span> // that will drop the snapshot<a name="line.6085"></a> |
| <span class="sourceLineNo">6086</span> }<a name="line.6086"></a> |
| <span class="sourceLineNo">6087</span><a name="line.6087"></a> |
| <span class="sourceLineNo">6088</span> private void replayWALFlushCannotFlushMarker(FlushDescriptor flush, long replaySeqId) {<a name="line.6088"></a> |
| <span class="sourceLineNo">6089</span> synchronized (writestate) {<a name="line.6089"></a> |
| <span class="sourceLineNo">6090</span> if (this.lastReplayedOpenRegionSeqId > replaySeqId) {<a name="line.6090"></a> |
| <span class="sourceLineNo">6091</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.6091"></a> |
| <span class="sourceLineNo">6092</span> + "Skipping replaying flush event :" + TextFormat.shortDebugString(flush)<a name="line.6092"></a> |
| <span class="sourceLineNo">6093</span> + " because its sequence id " + replaySeqId + " is smaller than this regions "<a name="line.6093"></a> |
| <span class="sourceLineNo">6094</span> + "lastReplayedOpenRegionSeqId of " + lastReplayedOpenRegionSeqId);<a name="line.6094"></a> |
| <span class="sourceLineNo">6095</span> return;<a name="line.6095"></a> |
| <span class="sourceLineNo">6096</span> }<a name="line.6096"></a> |
| <span class="sourceLineNo">6097</span><a name="line.6097"></a> |
| <span class="sourceLineNo">6098</span> // If we were waiting for observing a flush or region opening event for not showing partial<a name="line.6098"></a> |
| <span class="sourceLineNo">6099</span> // data after a secondary region crash, we can allow reads now. This event means that the<a name="line.6099"></a> |
| <span class="sourceLineNo">6100</span> // primary was not able to flush because memstore is empty when we requested flush. By the<a name="line.6100"></a> |
| <span class="sourceLineNo">6101</span> // time we observe this, we are guaranteed to have up to date seqId with our previous<a name="line.6101"></a> |
| <span class="sourceLineNo">6102</span> // assignment.<a name="line.6102"></a> |
| <span class="sourceLineNo">6103</span> this.setReadsEnabled(true);<a name="line.6103"></a> |
| <span class="sourceLineNo">6104</span> }<a name="line.6104"></a> |
| <span class="sourceLineNo">6105</span> }<a name="line.6105"></a> |
| <span class="sourceLineNo">6106</span><a name="line.6106"></a> |
| <span class="sourceLineNo">6107</span> PrepareFlushResult getPrepareFlushResult() {<a name="line.6107"></a> |
| <span class="sourceLineNo">6108</span> return prepareFlushResult;<a name="line.6108"></a> |
| <span class="sourceLineNo">6109</span> }<a name="line.6109"></a> |
| <span class="sourceLineNo">6110</span><a name="line.6110"></a> |
| <span class="sourceLineNo">6111</span> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NN_NAKED_NOTIFY",<a name="line.6111"></a> |
| <span class="sourceLineNo">6112</span> justification="Intentional; cleared the memstore")<a name="line.6112"></a> |
| <span class="sourceLineNo">6113</span> void replayWALRegionEventMarker(RegionEventDescriptor regionEvent) throws IOException {<a name="line.6113"></a> |
| <span class="sourceLineNo">6114</span> checkTargetRegion(regionEvent.getEncodedRegionName().toByteArray(),<a name="line.6114"></a> |
| <span class="sourceLineNo">6115</span> "RegionEvent marker from WAL ", regionEvent);<a name="line.6115"></a> |
| <span class="sourceLineNo">6116</span><a name="line.6116"></a> |
| <span class="sourceLineNo">6117</span> startRegionOperation(Operation.REPLAY_EVENT);<a name="line.6117"></a> |
| <span class="sourceLineNo">6118</span> try {<a name="line.6118"></a> |
| <span class="sourceLineNo">6119</span> if (ServerRegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {<a name="line.6119"></a> |
| <span class="sourceLineNo">6120</span> return; // if primary nothing to do<a name="line.6120"></a> |
| <span class="sourceLineNo">6121</span> }<a name="line.6121"></a> |
| <span class="sourceLineNo">6122</span><a name="line.6122"></a> |
| <span class="sourceLineNo">6123</span> if (regionEvent.getEventType() == EventType.REGION_CLOSE) {<a name="line.6123"></a> |
| <span class="sourceLineNo">6124</span> // nothing to do on REGION_CLOSE for now.<a name="line.6124"></a> |
| <span class="sourceLineNo">6125</span> return;<a name="line.6125"></a> |
| <span class="sourceLineNo">6126</span> }<a name="line.6126"></a> |
| <span class="sourceLineNo">6127</span> if (regionEvent.getEventType() != EventType.REGION_OPEN) {<a name="line.6127"></a> |
| <span class="sourceLineNo">6128</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.6128"></a> |
| <span class="sourceLineNo">6129</span> + "Unknown region event received, ignoring :"<a name="line.6129"></a> |
| <span class="sourceLineNo">6130</span> + TextFormat.shortDebugString(regionEvent));<a name="line.6130"></a> |
| <span class="sourceLineNo">6131</span> return;<a name="line.6131"></a> |
| <span class="sourceLineNo">6132</span> }<a name="line.6132"></a> |
| <span class="sourceLineNo">6133</span><a name="line.6133"></a> |
| <span class="sourceLineNo">6134</span> if (LOG.isDebugEnabled()) {<a name="line.6134"></a> |
| <span class="sourceLineNo">6135</span> LOG.debug(getRegionInfo().getEncodedName() + " : "<a name="line.6135"></a> |
| <span class="sourceLineNo">6136</span> + "Replaying region open event marker " + TextFormat.shortDebugString(regionEvent));<a name="line.6136"></a> |
| <span class="sourceLineNo">6137</span> }<a name="line.6137"></a> |
| <span class="sourceLineNo">6138</span><a name="line.6138"></a> |
| <span class="sourceLineNo">6139</span> // we will use writestate as a coarse-grain lock for all the replay events<a name="line.6139"></a> |
| <span class="sourceLineNo">6140</span> synchronized (writestate) {<a name="line.6140"></a> |
| <span class="sourceLineNo">6141</span> // Replication can deliver events out of order when primary region moves or the region<a name="line.6141"></a> |
| <span class="sourceLineNo">6142</span> // server crashes, since there is no coordination between replication of different wal files<a name="line.6142"></a> |
| <span class="sourceLineNo">6143</span> // belonging to different region servers. We have to safe guard against this case by using<a name="line.6143"></a> |
| <span class="sourceLineNo">6144</span> // region open event's seqid. Since this is the first event that the region puts (after<a name="line.6144"></a> |
| <span class="sourceLineNo">6145</span> // possibly flushing recovered.edits), after seeing this event, we can ignore every edit<a name="line.6145"></a> |
| <span class="sourceLineNo">6146</span> // smaller than this seqId<a name="line.6146"></a> |
| <span class="sourceLineNo">6147</span> if (this.lastReplayedOpenRegionSeqId <= regionEvent.getLogSequenceNumber()) {<a name="line.6147"></a> |
| <span class="sourceLineNo">6148</span> this.lastReplayedOpenRegionSeqId = regionEvent.getLogSequenceNumber();<a name="line.6148"></a> |
| <span class="sourceLineNo">6149</span> } else {<a name="line.6149"></a> |
| <span class="sourceLineNo">6150</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.6150"></a> |
| <span class="sourceLineNo">6151</span> + "Skipping replaying region event :" + TextFormat.shortDebugString(regionEvent)<a name="line.6151"></a> |
| <span class="sourceLineNo">6152</span> + " because its sequence id is smaller than this regions lastReplayedOpenRegionSeqId "<a name="line.6152"></a> |
| <span class="sourceLineNo">6153</span> + " of " + lastReplayedOpenRegionSeqId);<a name="line.6153"></a> |
| <span class="sourceLineNo">6154</span> return;<a name="line.6154"></a> |
| <span class="sourceLineNo">6155</span> }<a name="line.6155"></a> |
| <span class="sourceLineNo">6156</span><a name="line.6156"></a> |
| <span class="sourceLineNo">6157</span> // region open lists all the files that the region has at the time of the opening. Just pick<a name="line.6157"></a> |
| <span class="sourceLineNo">6158</span> // all the files and drop prepared flushes and empty memstores<a name="line.6158"></a> |
| <span class="sourceLineNo">6159</span> for (StoreDescriptor storeDescriptor : regionEvent.getStoresList()) {<a name="line.6159"></a> |
| <span class="sourceLineNo">6160</span> // stores of primary may be different now<a name="line.6160"></a> |
| <span class="sourceLineNo">6161</span> byte[] family = storeDescriptor.getFamilyName().toByteArray();<a name="line.6161"></a> |
| <span class="sourceLineNo">6162</span> HStore store = getStore(family);<a name="line.6162"></a> |
| <span class="sourceLineNo">6163</span> if (store == null) {<a name="line.6163"></a> |
| <span class="sourceLineNo">6164</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.6164"></a> |
| <span class="sourceLineNo">6165</span> + "Received a region open marker from primary, but the family is not found. "<a name="line.6165"></a> |
| <span class="sourceLineNo">6166</span> + "Ignoring. StoreDescriptor:" + storeDescriptor);<a name="line.6166"></a> |
| <span class="sourceLineNo">6167</span> continue;<a name="line.6167"></a> |
| <span class="sourceLineNo">6168</span> }<a name="line.6168"></a> |
| <span class="sourceLineNo">6169</span><a name="line.6169"></a> |
| <span class="sourceLineNo">6170</span> long storeSeqId = store.getMaxSequenceId().orElse(0L);<a name="line.6170"></a> |
| <span class="sourceLineNo">6171</span> List<String> storeFiles = storeDescriptor.getStoreFileList();<a name="line.6171"></a> |
| <span class="sourceLineNo">6172</span> try {<a name="line.6172"></a> |
| <span class="sourceLineNo">6173</span> store.refreshStoreFiles(storeFiles); // replace the files with the new ones<a name="line.6173"></a> |
| <span class="sourceLineNo">6174</span> } catch (FileNotFoundException ex) {<a name="line.6174"></a> |
| <span class="sourceLineNo">6175</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.6175"></a> |
| <span class="sourceLineNo">6176</span> + "At least one of the store files: " + storeFiles<a name="line.6176"></a> |
| <span class="sourceLineNo">6177</span> + " doesn't exist any more. Skip loading the file(s)", ex);<a name="line.6177"></a> |
| <span class="sourceLineNo">6178</span> continue;<a name="line.6178"></a> |
| <span class="sourceLineNo">6179</span> }<a name="line.6179"></a> |
| <span class="sourceLineNo">6180</span> if (store.getMaxSequenceId().orElse(0L) != storeSeqId) {<a name="line.6180"></a> |
| <span class="sourceLineNo">6181</span> // Record latest flush time if we picked up new files<a name="line.6181"></a> |
| <span class="sourceLineNo">6182</span> lastStoreFlushTimeMap.put(store, EnvironmentEdgeManager.currentTime());<a name="line.6182"></a> |
| <span class="sourceLineNo">6183</span> }<a name="line.6183"></a> |
| <span class="sourceLineNo">6184</span><a name="line.6184"></a> |
| <span class="sourceLineNo">6185</span> if (writestate.flushing) {<a name="line.6185"></a> |
| <span class="sourceLineNo">6186</span> // only drop memstore snapshots if they are smaller than last flush for the store<a name="line.6186"></a> |
| <span class="sourceLineNo">6187</span> if (this.prepareFlushResult.flushOpSeqId <= regionEvent.getLogSequenceNumber()) {<a name="line.6187"></a> |
| <span class="sourceLineNo">6188</span> StoreFlushContext ctx = this.prepareFlushResult.storeFlushCtxs == null ?<a name="line.6188"></a> |
| <span class="sourceLineNo">6189</span> null : this.prepareFlushResult.storeFlushCtxs.get(family);<a name="line.6189"></a> |
| <span class="sourceLineNo">6190</span> if (ctx != null) {<a name="line.6190"></a> |
| <span class="sourceLineNo">6191</span> MemStoreSize mss = store.getFlushableSize();<a name="line.6191"></a> |
| <span class="sourceLineNo">6192</span> ctx.abort();<a name="line.6192"></a> |
| <span class="sourceLineNo">6193</span> this.decrMemStoreSize(mss);<a name="line.6193"></a> |
| <span class="sourceLineNo">6194</span> this.prepareFlushResult.storeFlushCtxs.remove(family);<a name="line.6194"></a> |
| <span class="sourceLineNo">6195</span> }<a name="line.6195"></a> |
| <span class="sourceLineNo">6196</span> }<a name="line.6196"></a> |
| <span class="sourceLineNo">6197</span> }<a name="line.6197"></a> |
| <span class="sourceLineNo">6198</span><a name="line.6198"></a> |
| <span class="sourceLineNo">6199</span> // Drop the memstore contents if they are now smaller than the latest seen flushed file<a name="line.6199"></a> |
| <span class="sourceLineNo">6200</span> dropMemStoreContentsForSeqId(regionEvent.getLogSequenceNumber(), store);<a name="line.6200"></a> |
| <span class="sourceLineNo">6201</span> if (storeSeqId > this.maxFlushedSeqId) {<a name="line.6201"></a> |
| <span class="sourceLineNo">6202</span> this.maxFlushedSeqId = storeSeqId;<a name="line.6202"></a> |
| <span class="sourceLineNo">6203</span> }<a name="line.6203"></a> |
| <span class="sourceLineNo">6204</span> }<a name="line.6204"></a> |
| <span class="sourceLineNo">6205</span><a name="line.6205"></a> |
| <span class="sourceLineNo">6206</span> // if all stores ended up dropping their snapshots, we can safely drop the<a name="line.6206"></a> |
| <span class="sourceLineNo">6207</span> // prepareFlushResult<a name="line.6207"></a> |
| <span class="sourceLineNo">6208</span> dropPrepareFlushIfPossible();<a name="line.6208"></a> |
| <span class="sourceLineNo">6209</span><a name="line.6209"></a> |
| <span class="sourceLineNo">6210</span> // advance the mvcc read point so that the new flushed file is visible.<a name="line.6210"></a> |
| <span class="sourceLineNo">6211</span> mvcc.await();<a name="line.6211"></a> |
| <span class="sourceLineNo">6212</span><a name="line.6212"></a> |
| <span class="sourceLineNo">6213</span> // If we were waiting for observing a flush or region opening event for not showing partial<a name="line.6213"></a> |
| <span class="sourceLineNo">6214</span> // data after a secondary region crash, we can allow reads now.<a name="line.6214"></a> |
| <span class="sourceLineNo">6215</span> this.setReadsEnabled(true);<a name="line.6215"></a> |
| <span class="sourceLineNo">6216</span><a name="line.6216"></a> |
| <span class="sourceLineNo">6217</span> // C. Finally notify anyone waiting on memstore to clear:<a name="line.6217"></a> |
| <span class="sourceLineNo">6218</span> // e.g. checkResources().<a name="line.6218"></a> |
| <span class="sourceLineNo">6219</span> synchronized (this) {<a name="line.6219"></a> |
| <span class="sourceLineNo">6220</span> notifyAll(); // FindBugs NN_NAKED_NOTIFY<a name="line.6220"></a> |
| <span class="sourceLineNo">6221</span> }<a name="line.6221"></a> |
| <span class="sourceLineNo">6222</span> }<a name="line.6222"></a> |
| <span class="sourceLineNo">6223</span> logRegionFiles();<a name="line.6223"></a> |
| <span class="sourceLineNo">6224</span> } finally {<a name="line.6224"></a> |
| <span class="sourceLineNo">6225</span> closeRegionOperation(Operation.REPLAY_EVENT);<a name="line.6225"></a> |
| <span class="sourceLineNo">6226</span> }<a name="line.6226"></a> |
| <span class="sourceLineNo">6227</span> }<a name="line.6227"></a> |
| <span class="sourceLineNo">6228</span><a name="line.6228"></a> |
| <span class="sourceLineNo">6229</span> void replayWALBulkLoadEventMarker(WALProtos.BulkLoadDescriptor bulkLoadEvent) throws IOException {<a name="line.6229"></a> |
| <span class="sourceLineNo">6230</span> checkTargetRegion(bulkLoadEvent.getEncodedRegionName().toByteArray(),<a name="line.6230"></a> |
| <span class="sourceLineNo">6231</span> "BulkLoad marker from WAL ", bulkLoadEvent);<a name="line.6231"></a> |
| <span class="sourceLineNo">6232</span><a name="line.6232"></a> |
| <span class="sourceLineNo">6233</span> if (ServerRegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {<a name="line.6233"></a> |
| <span class="sourceLineNo">6234</span> return; // if primary nothing to do<a name="line.6234"></a> |
| <span class="sourceLineNo">6235</span> }<a name="line.6235"></a> |
| <span class="sourceLineNo">6236</span><a name="line.6236"></a> |
| <span class="sourceLineNo">6237</span> if (LOG.isDebugEnabled()) {<a name="line.6237"></a> |
| <span class="sourceLineNo">6238</span> LOG.debug(getRegionInfo().getEncodedName() + " : "<a name="line.6238"></a> |
| <span class="sourceLineNo">6239</span> + "Replaying bulkload event marker " + TextFormat.shortDebugString(bulkLoadEvent));<a name="line.6239"></a> |
| <span class="sourceLineNo">6240</span> }<a name="line.6240"></a> |
| <span class="sourceLineNo">6241</span> // check if multiple families involved<a name="line.6241"></a> |
| <span class="sourceLineNo">6242</span> boolean multipleFamilies = false;<a name="line.6242"></a> |
| <span class="sourceLineNo">6243</span> byte[] family = null;<a name="line.6243"></a> |
| <span class="sourceLineNo">6244</span> for (StoreDescriptor storeDescriptor : bulkLoadEvent.getStoresList()) {<a name="line.6244"></a> |
| <span class="sourceLineNo">6245</span> byte[] fam = storeDescriptor.getFamilyName().toByteArray();<a name="line.6245"></a> |
| <span class="sourceLineNo">6246</span> if (family == null) {<a name="line.6246"></a> |
| <span class="sourceLineNo">6247</span> family = fam;<a name="line.6247"></a> |
| <span class="sourceLineNo">6248</span> } else if (!Bytes.equals(family, fam)) {<a name="line.6248"></a> |
| <span class="sourceLineNo">6249</span> multipleFamilies = true;<a name="line.6249"></a> |
| <span class="sourceLineNo">6250</span> break;<a name="line.6250"></a> |
| <span class="sourceLineNo">6251</span> }<a name="line.6251"></a> |
| <span class="sourceLineNo">6252</span> }<a name="line.6252"></a> |
| <span class="sourceLineNo">6253</span><a name="line.6253"></a> |
| <span class="sourceLineNo">6254</span> startBulkRegionOperation(multipleFamilies);<a name="line.6254"></a> |
| <span class="sourceLineNo">6255</span> try {<a name="line.6255"></a> |
| <span class="sourceLineNo">6256</span> // we will use writestate as a coarse-grain lock for all the replay events<a name="line.6256"></a> |
| <span class="sourceLineNo">6257</span> synchronized (writestate) {<a name="line.6257"></a> |
| <span class="sourceLineNo">6258</span> // Replication can deliver events out of order when primary region moves or the region<a name="line.6258"></a> |
| <span class="sourceLineNo">6259</span> // server crashes, since there is no coordination between replication of different wal files<a name="line.6259"></a> |
| <span class="sourceLineNo">6260</span> // belonging to different region servers. We have to safe guard against this case by using<a name="line.6260"></a> |
| <span class="sourceLineNo">6261</span> // region open event's seqid. Since this is the first event that the region puts (after<a name="line.6261"></a> |
| <span class="sourceLineNo">6262</span> // possibly flushing recovered.edits), after seeing this event, we can ignore every edit<a name="line.6262"></a> |
| <span class="sourceLineNo">6263</span> // smaller than this seqId<a name="line.6263"></a> |
| <span class="sourceLineNo">6264</span> if (bulkLoadEvent.getBulkloadSeqNum() >= 0<a name="line.6264"></a> |
| <span class="sourceLineNo">6265</span> && this.lastReplayedOpenRegionSeqId >= bulkLoadEvent.getBulkloadSeqNum()) {<a name="line.6265"></a> |
| <span class="sourceLineNo">6266</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.6266"></a> |
| <span class="sourceLineNo">6267</span> + "Skipping replaying bulkload event :"<a name="line.6267"></a> |
| <span class="sourceLineNo">6268</span> + TextFormat.shortDebugString(bulkLoadEvent)<a name="line.6268"></a> |
| <span class="sourceLineNo">6269</span> + " because its sequence id is smaller than this region's lastReplayedOpenRegionSeqId"<a name="line.6269"></a> |
| <span class="sourceLineNo">6270</span> + " =" + lastReplayedOpenRegionSeqId);<a name="line.6270"></a> |
| <span class="sourceLineNo">6271</span><a name="line.6271"></a> |
| <span class="sourceLineNo">6272</span> return;<a name="line.6272"></a> |
| <span class="sourceLineNo">6273</span> }<a name="line.6273"></a> |
| <span class="sourceLineNo">6274</span><a name="line.6274"></a> |
| <span class="sourceLineNo">6275</span> for (StoreDescriptor storeDescriptor : bulkLoadEvent.getStoresList()) {<a name="line.6275"></a> |
| <span class="sourceLineNo">6276</span> // stores of primary may be different now<a name="line.6276"></a> |
| <span class="sourceLineNo">6277</span> family = storeDescriptor.getFamilyName().toByteArray();<a name="line.6277"></a> |
| <span class="sourceLineNo">6278</span> HStore store = getStore(family);<a name="line.6278"></a> |
| <span class="sourceLineNo">6279</span> if (store == null) {<a name="line.6279"></a> |
| <span class="sourceLineNo">6280</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.6280"></a> |
| <span class="sourceLineNo">6281</span> + "Received a bulk load marker from primary, but the family is not found. "<a name="line.6281"></a> |
| <span class="sourceLineNo">6282</span> + "Ignoring. StoreDescriptor:" + storeDescriptor);<a name="line.6282"></a> |
| <span class="sourceLineNo">6283</span> continue;<a name="line.6283"></a> |
| <span class="sourceLineNo">6284</span> }<a name="line.6284"></a> |
| <span class="sourceLineNo">6285</span><a name="line.6285"></a> |
| <span class="sourceLineNo">6286</span> List<String> storeFiles = storeDescriptor.getStoreFileList();<a name="line.6286"></a> |
| <span class="sourceLineNo">6287</span> for (String storeFile : storeFiles) {<a name="line.6287"></a> |
| <span class="sourceLineNo">6288</span> StoreFileInfo storeFileInfo = null;<a name="line.6288"></a> |
| <span class="sourceLineNo">6289</span> try {<a name="line.6289"></a> |
| <span class="sourceLineNo">6290</span> storeFileInfo = fs.getStoreFileInfo(Bytes.toString(family), storeFile);<a name="line.6290"></a> |
| <span class="sourceLineNo">6291</span> store.bulkLoadHFile(storeFileInfo);<a name="line.6291"></a> |
| <span class="sourceLineNo">6292</span> } catch(FileNotFoundException ex) {<a name="line.6292"></a> |
| <span class="sourceLineNo">6293</span> LOG.warn(getRegionInfo().getEncodedName() + " : "<a name="line.6293"></a> |
| <span class="sourceLineNo">6294</span> + ((storeFileInfo != null) ? storeFileInfo.toString() :<a name="line.6294"></a> |
| <span class="sourceLineNo">6295</span> (new Path(Bytes.toString(family), storeFile)).toString())<a name="line.6295"></a> |
| <span class="sourceLineNo">6296</span> + " doesn't exist any more. Skip loading the file");<a name="line.6296"></a> |
| <span class="sourceLineNo">6297</span> }<a name="line.6297"></a> |
| <span class="sourceLineNo">6298</span> }<a name="line.6298"></a> |
| <span class="sourceLineNo">6299</span> }<a name="line.6299"></a> |
| <span class="sourceLineNo">6300</span> }<a name="line.6300"></a> |
| <span class="sourceLineNo">6301</span> if (bulkLoadEvent.getBulkloadSeqNum() > 0) {<a name="line.6301"></a> |
| <span class="sourceLineNo">6302</span> mvcc.advanceTo(bulkLoadEvent.getBulkloadSeqNum());<a name="line.6302"></a> |
| <span class="sourceLineNo">6303</span> }<a name="line.6303"></a> |
| <span class="sourceLineNo">6304</span> } finally {<a name="line.6304"></a> |
| <span class="sourceLineNo">6305</span> closeBulkRegionOperation();<a name="line.6305"></a> |
| <span class="sourceLineNo">6306</span> }<a name="line.6306"></a> |
| <span class="sourceLineNo">6307</span> }<a name="line.6307"></a> |
| <span class="sourceLineNo">6308</span><a name="line.6308"></a> |
| <span class="sourceLineNo">6309</span> /**<a name="line.6309"></a> |
| <span class="sourceLineNo">6310</span> * If all stores ended up dropping their snapshots, we can safely drop the prepareFlushResult<a name="line.6310"></a> |
| <span class="sourceLineNo">6311</span> */<a name="line.6311"></a> |
| <span class="sourceLineNo">6312</span> private void dropPrepareFlushIfPossible() {<a name="line.6312"></a> |
| <span class="sourceLineNo">6313</span> if (writestate.flushing) {<a name="line.6313"></a> |
| <span class="sourceLineNo">6314</span> boolean canDrop = true;<a name="line.6314"></a> |
| <span class="sourceLineNo">6315</span> if (prepareFlushResult.storeFlushCtxs != null) {<a name="line.6315"></a> |
| <span class="sourceLineNo">6316</span> for (Entry<byte[], StoreFlushContext> entry : prepareFlushResult.storeFlushCtxs<a name="line.6316"></a> |
| <span class="sourceLineNo">6317</span> .entrySet()) {<a name="line.6317"></a> |
| <span class="sourceLineNo">6318</span> HStore store = getStore(entry.getKey());<a name="line.6318"></a> |
| <span class="sourceLineNo">6319</span> if (store == null) {<a name="line.6319"></a> |
| <span class="sourceLineNo">6320</span> continue;<a name="line.6320"></a> |
| <span class="sourceLineNo">6321</span> }<a name="line.6321"></a> |
| <span class="sourceLineNo">6322</span> if (store.getSnapshotSize().getDataSize() > 0) {<a name="line.6322"></a> |
| <span class="sourceLineNo">6323</span> canDrop = false;<a name="line.6323"></a> |
| <span class="sourceLineNo">6324</span> break;<a name="line.6324"></a> |
| <span class="sourceLineNo">6325</span> }<a name="line.6325"></a> |
| <span class="sourceLineNo">6326</span> }<a name="line.6326"></a> |
| <span class="sourceLineNo">6327</span> }<a name="line.6327"></a> |
| <span class="sourceLineNo">6328</span><a name="line.6328"></a> |
| <span class="sourceLineNo">6329</span> // this means that all the stores in the region has finished flushing, but the WAL marker<a name="line.6329"></a> |
| <span class="sourceLineNo">6330</span> // may not have been written or we did not receive it yet.<a name="line.6330"></a> |
| <span class="sourceLineNo">6331</span> if (canDrop) {<a name="line.6331"></a> |
| <span class="sourceLineNo">6332</span> writestate.flushing = false;<a name="line.6332"></a> |
| <span class="sourceLineNo">6333</span> this.prepareFlushResult = null;<a name="line.6333"></a> |
| <span class="sourceLineNo">6334</span> }<a name="line.6334"></a> |
| <span class="sourceLineNo">6335</span> }<a name="line.6335"></a> |
| <span class="sourceLineNo">6336</span> }<a name="line.6336"></a> |
| <span class="sourceLineNo">6337</span><a name="line.6337"></a> |
| <span class="sourceLineNo">6338</span> @Override<a name="line.6338"></a> |
| <span class="sourceLineNo">6339</span> public boolean refreshStoreFiles() throws IOException {<a name="line.6339"></a> |
| <span class="sourceLineNo">6340</span> return refreshStoreFiles(false);<a name="line.6340"></a> |
| <span class="sourceLineNo">6341</span> }<a name="line.6341"></a> |
| <span class="sourceLineNo">6342</span><a name="line.6342"></a> |
| <span class="sourceLineNo">6343</span> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NN_NAKED_NOTIFY",<a name="line.6343"></a> |
| <span class="sourceLineNo">6344</span> justification = "Notify is about post replay. Intentional")<a name="line.6344"></a> |
| <span class="sourceLineNo">6345</span> protected boolean refreshStoreFiles(boolean force) throws IOException {<a name="line.6345"></a> |
| <span class="sourceLineNo">6346</span> if (!force && ServerRegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {<a name="line.6346"></a> |
| <span class="sourceLineNo">6347</span> return false; // if primary nothing to do<a name="line.6347"></a> |
| <span class="sourceLineNo">6348</span> }<a name="line.6348"></a> |
| <span class="sourceLineNo">6349</span><a name="line.6349"></a> |
| <span class="sourceLineNo">6350</span> if (LOG.isDebugEnabled()) {<a name="line.6350"></a> |
| <span class="sourceLineNo">6351</span> LOG.debug(getRegionInfo().getEncodedName() + " : "<a name="line.6351"></a> |
| <span class="sourceLineNo">6352</span> + "Refreshing store files to see whether we can free up memstore");<a name="line.6352"></a> |
| <span class="sourceLineNo">6353</span> }<a name="line.6353"></a> |
| <span class="sourceLineNo">6354</span><a name="line.6354"></a> |
| <span class="sourceLineNo">6355</span> long totalFreedDataSize = 0;<a name="line.6355"></a> |
| <span class="sourceLineNo">6356</span><a name="line.6356"></a> |
| <span class="sourceLineNo">6357</span> long smallestSeqIdInStores = Long.MAX_VALUE;<a name="line.6357"></a> |
| <span class="sourceLineNo">6358</span><a name="line.6358"></a> |
| <span class="sourceLineNo">6359</span> startRegionOperation(); // obtain region close lock<a name="line.6359"></a> |
| <span class="sourceLineNo">6360</span> try {<a name="line.6360"></a> |
| <span class="sourceLineNo">6361</span> Map<HStore, Long> map = new HashMap<>();<a name="line.6361"></a> |
| <span class="sourceLineNo">6362</span> synchronized (writestate) {<a name="line.6362"></a> |
| <span class="sourceLineNo">6363</span> for (HStore store : stores.values()) {<a name="line.6363"></a> |
| <span class="sourceLineNo">6364</span> // TODO: some stores might see new data from flush, while others do not which<a name="line.6364"></a> |
| <span class="sourceLineNo">6365</span> // MIGHT break atomic edits across column families.<a name="line.6365"></a> |
| <span class="sourceLineNo">6366</span> long maxSeqIdBefore = store.getMaxSequenceId().orElse(0L);<a name="line.6366"></a> |
| <span class="sourceLineNo">6367</span><a name="line.6367"></a> |
| <span class="sourceLineNo">6368</span> // refresh the store files. This is similar to observing a region open wal marker.<a name="line.6368"></a> |
| <span class="sourceLineNo">6369</span> store.refreshStoreFiles();<a name="line.6369"></a> |
| <span class="sourceLineNo">6370</span><a name="line.6370"></a> |
| <span class="sourceLineNo">6371</span> long storeSeqId = store.getMaxSequenceId().orElse(0L);<a name="line.6371"></a> |
| <span class="sourceLineNo">6372</span> if (storeSeqId < smallestSeqIdInStores) {<a name="line.6372"></a> |
| <span class="sourceLineNo">6373</span> smallestSeqIdInStores = storeSeqId;<a name="line.6373"></a> |
| <span class="sourceLineNo">6374</span> }<a name="line.6374"></a> |
| <span class="sourceLineNo">6375</span><a name="line.6375"></a> |
| <span class="sourceLineNo">6376</span> // see whether we can drop the memstore or the snapshot<a name="line.6376"></a> |
| <span class="sourceLineNo">6377</span> if (storeSeqId > maxSeqIdBefore) {<a name="line.6377"></a> |
| <span class="sourceLineNo">6378</span> if (writestate.flushing) {<a name="line.6378"></a> |
| <span class="sourceLineNo">6379</span> // only drop memstore snapshots if they are smaller than last flush for the store<a name="line.6379"></a> |
| <span class="sourceLineNo">6380</span> if (this.prepareFlushResult.flushOpSeqId <= storeSeqId) {<a name="line.6380"></a> |
| <span class="sourceLineNo">6381</span> StoreFlushContext ctx = this.prepareFlushResult.storeFlushCtxs == null ?<a name="line.6381"></a> |
| <span class="sourceLineNo">6382</span> null : this.prepareFlushResult.storeFlushCtxs.get(<a name="line.6382"></a> |
| <span class="sourceLineNo">6383</span> store.getColumnFamilyDescriptor().getName());<a name="line.6383"></a> |
| <span class="sourceLineNo">6384</span> if (ctx != null) {<a name="line.6384"></a> |
| <span class="sourceLineNo">6385</span> MemStoreSize mss = store.getFlushableSize();<a name="line.6385"></a> |
| <span class="sourceLineNo">6386</span> ctx.abort();<a name="line.6386"></a> |
| <span class="sourceLineNo">6387</span> this.decrMemStoreSize(mss);<a name="line.6387"></a> |
| <span class="sourceLineNo">6388</span> this.prepareFlushResult.storeFlushCtxs.<a name="line.6388"></a> |
| <span class="sourceLineNo">6389</span> remove(store.getColumnFamilyDescriptor().getName());<a name="line.6389"></a> |
| <span class="sourceLineNo">6390</span> totalFreedDataSize += mss.getDataSize();<a name="line.6390"></a> |
| <span class="sourceLineNo">6391</span> }<a name="line.6391"></a> |
| <span class="sourceLineNo">6392</span> }<a name="line.6392"></a> |
| <span class="sourceLineNo">6393</span> }<a name="line.6393"></a> |
| <span class="sourceLineNo">6394</span><a name="line.6394"></a> |
| <span class="sourceLineNo">6395</span> map.put(store, storeSeqId);<a name="line.6395"></a> |
| <span class="sourceLineNo">6396</span> }<a name="line.6396"></a> |
| <span class="sourceLineNo">6397</span> }<a name="line.6397"></a> |
| <span class="sourceLineNo">6398</span><a name="line.6398"></a> |
| <span class="sourceLineNo">6399</span> // if all stores ended up dropping their snapshots, we can safely drop the<a name="line.6399"></a> |
| <span class="sourceLineNo">6400</span> // prepareFlushResult<a name="line.6400"></a> |
| <span class="sourceLineNo">6401</span> dropPrepareFlushIfPossible();<a name="line.6401"></a> |
| <span class="sourceLineNo">6402</span><a name="line.6402"></a> |
| <span class="sourceLineNo">6403</span> // advance the mvcc read point so that the new flushed files are visible.<a name="line.6403"></a> |
| <span class="sourceLineNo">6404</span> // either greater than flush seq number or they were already picked up via flush.<a name="line.6404"></a> |
| <span class="sourceLineNo">6405</span> for (HStore s : stores.values()) {<a name="line.6405"></a> |
| <span class="sourceLineNo">6406</span> mvcc.advanceTo(s.getMaxMemStoreTS().orElse(0L));<a name="line.6406"></a> |
| <span class="sourceLineNo">6407</span> }<a name="line.6407"></a> |
| <span class="sourceLineNo">6408</span><a name="line.6408"></a> |
| <span class="sourceLineNo">6409</span><a name="line.6409"></a> |
| <span class="sourceLineNo">6410</span> // smallestSeqIdInStores is the seqId that we have a corresponding hfile for. We can safely<a name="line.6410"></a> |
| <span class="sourceLineNo">6411</span> // skip all edits that are to be replayed in the future with that has a smaller seqId<a name="line.6411"></a> |
| <span class="sourceLineNo">6412</span> // than this. We are updating lastReplayedOpenRegionSeqId so that we can skip all edits<a name="line.6412"></a> |
| <span class="sourceLineNo">6413</span> // that we have picked the flush files for<a name="line.6413"></a> |
| <span class="sourceLineNo">6414</span> if (this.lastReplayedOpenRegionSeqId < smallestSeqIdInStores) {<a name="line.6414"></a> |
| <span class="sourceLineNo">6415</span> this.lastReplayedOpenRegionSeqId = smallestSeqIdInStores;<a name="line.6415"></a> |
| <span class="sourceLineNo">6416</span> }<a name="line.6416"></a> |
| <span class="sourceLineNo">6417</span> }<a name="line.6417"></a> |
| <span class="sourceLineNo">6418</span> if (!map.isEmpty()) {<a name="line.6418"></a> |
| <span class="sourceLineNo">6419</span> for (Map.Entry<HStore, Long> entry : map.entrySet()) {<a name="line.6419"></a> |
| <span class="sourceLineNo">6420</span> // Drop the memstore contents if they are now smaller than the latest seen flushed file<a name="line.6420"></a> |
| <span class="sourceLineNo">6421</span> totalFreedDataSize += dropMemStoreContentsForSeqId(entry.getValue(), entry.getKey())<a name="line.6421"></a> |
| <span class="sourceLineNo">6422</span> .getDataSize();<a name="line.6422"></a> |
| <span class="sourceLineNo">6423</span> }<a name="line.6423"></a> |
| <span class="sourceLineNo">6424</span> }<a name="line.6424"></a> |
| <span class="sourceLineNo">6425</span> // C. Finally notify anyone waiting on memstore to clear:<a name="line.6425"></a> |
| <span class="sourceLineNo">6426</span> // e.g. checkResources().<a name="line.6426"></a> |
| <span class="sourceLineNo">6427</span> synchronized (this) {<a name="line.6427"></a> |
| <span class="sourceLineNo">6428</span> notifyAll(); // FindBugs NN_NAKED_NOTIFY<a name="line.6428"></a> |
| <span class="sourceLineNo">6429</span> }<a name="line.6429"></a> |
| <span class="sourceLineNo">6430</span> return totalFreedDataSize > 0;<a name="line.6430"></a> |
| <span class="sourceLineNo">6431</span> } finally {<a name="line.6431"></a> |
| <span class="sourceLineNo">6432</span> closeRegionOperation();<a name="line.6432"></a> |
| <span class="sourceLineNo">6433</span> }<a name="line.6433"></a> |
| <span class="sourceLineNo">6434</span> }<a name="line.6434"></a> |
| <span class="sourceLineNo">6435</span><a name="line.6435"></a> |
| <span class="sourceLineNo">6436</span> private void logRegionFiles() {<a name="line.6436"></a> |
| <span class="sourceLineNo">6437</span> if (LOG.isTraceEnabled()) {<a name="line.6437"></a> |
| <span class="sourceLineNo">6438</span> LOG.trace(getRegionInfo().getEncodedName() + " : Store files for region: ");<a name="line.6438"></a> |
| <span class="sourceLineNo">6439</span> stores.values().stream().filter(s -> s.getStorefiles() != null)<a name="line.6439"></a> |
| <span class="sourceLineNo">6440</span> .flatMap(s -> s.getStorefiles().stream())<a name="line.6440"></a> |
| <span class="sourceLineNo">6441</span> .forEachOrdered(sf -> LOG.trace(getRegionInfo().getEncodedName() + " : " + sf));<a name="line.6441"></a> |
| <span class="sourceLineNo">6442</span> }<a name="line.6442"></a> |
| <span class="sourceLineNo">6443</span> }<a name="line.6443"></a> |
| <span class="sourceLineNo">6444</span><a name="line.6444"></a> |
| <span class="sourceLineNo">6445</span> /** Checks whether the given regionName is either equal to our region, or that<a name="line.6445"></a> |
| <span class="sourceLineNo">6446</span> * the regionName is the primary region to our corresponding range for the secondary replica.<a name="line.6446"></a> |
| <span class="sourceLineNo">6447</span> */<a name="line.6447"></a> |
| <span class="sourceLineNo">6448</span> private void checkTargetRegion(byte[] encodedRegionName, String exceptionMsg, Object payload)<a name="line.6448"></a> |
| <span class="sourceLineNo">6449</span> throws WrongRegionException {<a name="line.6449"></a> |
| <span class="sourceLineNo">6450</span> if (Bytes.equals(this.getRegionInfo().getEncodedNameAsBytes(), encodedRegionName)) {<a name="line.6450"></a> |
| <span class="sourceLineNo">6451</span> return;<a name="line.6451"></a> |
| <span class="sourceLineNo">6452</span> }<a name="line.6452"></a> |
| <span class="sourceLineNo">6453</span><a name="line.6453"></a> |
| <span class="sourceLineNo">6454</span> if (!RegionReplicaUtil.isDefaultReplica(this.getRegionInfo()) &&<a name="line.6454"></a> |
| <span class="sourceLineNo">6455</span> Bytes.equals(encodedRegionName,<a name="line.6455"></a> |
| <span class="sourceLineNo">6456</span> this.fs.getRegionInfoForFS().getEncodedNameAsBytes())) {<a name="line.6456"></a> |
| <span class="sourceLineNo">6457</span> return;<a name="line.6457"></a> |
| <span class="sourceLineNo">6458</span> }<a name="line.6458"></a> |
| <span class="sourceLineNo">6459</span><a name="line.6459"></a> |
| <span class="sourceLineNo">6460</span> throw new WrongRegionException(exceptionMsg + payload<a name="line.6460"></a> |
| <span class="sourceLineNo">6461</span> + " targetted for region " + Bytes.toStringBinary(encodedRegionName)<a name="line.6461"></a> |
| <span class="sourceLineNo">6462</span> + " does not match this region: " + this.getRegionInfo());<a name="line.6462"></a> |
| <span class="sourceLineNo">6463</span> }<a name="line.6463"></a> |
| <span class="sourceLineNo">6464</span><a name="line.6464"></a> |
| <span class="sourceLineNo">6465</span> /**<a name="line.6465"></a> |
| <span class="sourceLineNo">6466</span> * Used by tests<a name="line.6466"></a> |
| <span class="sourceLineNo">6467</span> * @param s Store to add edit too.<a name="line.6467"></a> |
| <span class="sourceLineNo">6468</span> * @param cell Cell to add.<a name="line.6468"></a> |
| <span class="sourceLineNo">6469</span> */<a name="line.6469"></a> |
| <span class="sourceLineNo">6470</span> protected void restoreEdit(HStore s, Cell cell, MemStoreSizing memstoreAccounting) {<a name="line.6470"></a> |
| <span class="sourceLineNo">6471</span> s.add(cell, memstoreAccounting);<a name="line.6471"></a> |
| <span class="sourceLineNo">6472</span> }<a name="line.6472"></a> |
| <span class="sourceLineNo">6473</span><a name="line.6473"></a> |
| <span class="sourceLineNo">6474</span> /**<a name="line.6474"></a> |
| <span class="sourceLineNo">6475</span> * make sure have been through lease recovery before get file status, so the file length can be<a name="line.6475"></a> |
| <span class="sourceLineNo">6476</span> * trusted.<a name="line.6476"></a> |
| <span class="sourceLineNo">6477</span> * @param p File to check.<a name="line.6477"></a> |
| <span class="sourceLineNo">6478</span> * @return True if file was zero-length (and if so, we'll delete it in here).<a name="line.6478"></a> |
| <span class="sourceLineNo">6479</span> * @throws IOException<a name="line.6479"></a> |
| <span class="sourceLineNo">6480</span> */<a name="line.6480"></a> |
| <span class="sourceLineNo">6481</span> private static boolean isZeroLengthThenDelete(final FileSystem fs, final FileStatus stat,<a name="line.6481"></a> |
| <span class="sourceLineNo">6482</span> final Path p) throws IOException {<a name="line.6482"></a> |
| <span class="sourceLineNo">6483</span> if (stat.getLen() > 0) {<a name="line.6483"></a> |
| <span class="sourceLineNo">6484</span> return false;<a name="line.6484"></a> |
| <span class="sourceLineNo">6485</span> }<a name="line.6485"></a> |
| <span class="sourceLineNo">6486</span> LOG.warn("File " + p + " is zero-length, deleting.");<a name="line.6486"></a> |
| <span class="sourceLineNo">6487</span> fs.delete(p, false);<a name="line.6487"></a> |
| <span class="sourceLineNo">6488</span> return true;<a name="line.6488"></a> |
| <span class="sourceLineNo">6489</span> }<a name="line.6489"></a> |
| <span class="sourceLineNo">6490</span><a name="line.6490"></a> |
| <span class="sourceLineNo">6491</span> protected HStore instantiateHStore(final ColumnFamilyDescriptor family, boolean warmup)<a name="line.6491"></a> |
| <span class="sourceLineNo">6492</span> throws IOException {<a name="line.6492"></a> |
| <span class="sourceLineNo">6493</span> if (family.isMobEnabled()) {<a name="line.6493"></a> |
| <span class="sourceLineNo">6494</span> if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) {<a name="line.6494"></a> |
| <span class="sourceLineNo">6495</span> throw new IOException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS +<a name="line.6495"></a> |
| <span class="sourceLineNo">6496</span> " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY +<a name="line.6496"></a> |
| <span class="sourceLineNo">6497</span> " accordingly.");<a name="line.6497"></a> |
| <span class="sourceLineNo">6498</span> }<a name="line.6498"></a> |
| <span class="sourceLineNo">6499</span> return new HMobStore(this, family, this.conf, warmup);<a name="line.6499"></a> |
| <span class="sourceLineNo">6500</span> }<a name="line.6500"></a> |
| <span class="sourceLineNo">6501</span> return new HStore(this, family, this.conf, warmup);<a name="line.6501"></a> |
| <span class="sourceLineNo">6502</span> }<a name="line.6502"></a> |
| <span class="sourceLineNo">6503</span><a name="line.6503"></a> |
| <span class="sourceLineNo">6504</span> @Override<a name="line.6504"></a> |
| <span class="sourceLineNo">6505</span> public HStore getStore(byte[] column) {<a name="line.6505"></a> |
| <span class="sourceLineNo">6506</span> return this.stores.get(column);<a name="line.6506"></a> |
| <span class="sourceLineNo">6507</span> }<a name="line.6507"></a> |
| <span class="sourceLineNo">6508</span><a name="line.6508"></a> |
| <span class="sourceLineNo">6509</span> /**<a name="line.6509"></a> |
| <span class="sourceLineNo">6510</span> * Return HStore instance. Does not do any copy: as the number of store is limited, we iterate on<a name="line.6510"></a> |
| <span class="sourceLineNo">6511</span> * the list.<a name="line.6511"></a> |
| <span class="sourceLineNo">6512</span> */<a name="line.6512"></a> |
| <span class="sourceLineNo">6513</span> private HStore getStore(Cell cell) {<a name="line.6513"></a> |
| <span class="sourceLineNo">6514</span> return stores.entrySet().stream().filter(e -> CellUtil.matchingFamily(cell, e.getKey()))<a name="line.6514"></a> |
| <span class="sourceLineNo">6515</span> .map(e -> e.getValue()).findFirst().orElse(null);<a name="line.6515"></a> |
| <span class="sourceLineNo">6516</span> }<a name="line.6516"></a> |
| <span class="sourceLineNo">6517</span><a name="line.6517"></a> |
| <span class="sourceLineNo">6518</span> @Override<a name="line.6518"></a> |
| <span class="sourceLineNo">6519</span> public List<HStore> getStores() {<a name="line.6519"></a> |
| <span class="sourceLineNo">6520</span> return new ArrayList<>(stores.values());<a name="line.6520"></a> |
| <span class="sourceLineNo">6521</span> }<a name="line.6521"></a> |
| <span class="sourceLineNo">6522</span><a name="line.6522"></a> |
| <span class="sourceLineNo">6523</span> @Override<a name="line.6523"></a> |
| <span class="sourceLineNo">6524</span> public List<String> getStoreFileList(byte[][] columns) throws IllegalArgumentException {<a name="line.6524"></a> |
| <span class="sourceLineNo">6525</span> List<String> storeFileNames = new ArrayList<>();<a name="line.6525"></a> |
| <span class="sourceLineNo">6526</span> synchronized (closeLock) {<a name="line.6526"></a> |
| <span class="sourceLineNo">6527</span> for (byte[] column : columns) {<a name="line.6527"></a> |
| <span class="sourceLineNo">6528</span> HStore store = this.stores.get(column);<a name="line.6528"></a> |
| <span class="sourceLineNo">6529</span> if (store == null) {<a name="line.6529"></a> |
| <span class="sourceLineNo">6530</span> throw new IllegalArgumentException(<a name="line.6530"></a> |
| <span class="sourceLineNo">6531</span> "No column family : " + new String(column, StandardCharsets.UTF_8) + " available");<a name="line.6531"></a> |
| <span class="sourceLineNo">6532</span> }<a name="line.6532"></a> |
| <span class="sourceLineNo">6533</span> Collection<HStoreFile> storeFiles = store.getStorefiles();<a name="line.6533"></a> |
| <span class="sourceLineNo">6534</span> if (storeFiles == null) {<a name="line.6534"></a> |
| <span class="sourceLineNo">6535</span> continue;<a name="line.6535"></a> |
| <span class="sourceLineNo">6536</span> }<a name="line.6536"></a> |
| <span class="sourceLineNo">6537</span> for (HStoreFile storeFile : storeFiles) {<a name="line.6537"></a> |
| <span class="sourceLineNo">6538</span> storeFileNames.add(storeFile.getPath().toString());<a name="line.6538"></a> |
| <span class="sourceLineNo">6539</span> }<a name="line.6539"></a> |
| <span class="sourceLineNo">6540</span><a name="line.6540"></a> |
| <span class="sourceLineNo">6541</span> logRegionFiles();<a name="line.6541"></a> |
| <span class="sourceLineNo">6542</span> }<a name="line.6542"></a> |
| <span class="sourceLineNo">6543</span> }<a name="line.6543"></a> |
| <span class="sourceLineNo">6544</span> return storeFileNames;<a name="line.6544"></a> |
| <span class="sourceLineNo">6545</span> }<a name="line.6545"></a> |
| <span class="sourceLineNo">6546</span><a name="line.6546"></a> |
| <span class="sourceLineNo">6547</span> //////////////////////////////////////////////////////////////////////////////<a name="line.6547"></a> |
| <span class="sourceLineNo">6548</span> // Support code<a name="line.6548"></a> |
| <span class="sourceLineNo">6549</span> //////////////////////////////////////////////////////////////////////////////<a name="line.6549"></a> |
| <span class="sourceLineNo">6550</span><a name="line.6550"></a> |
| <span class="sourceLineNo">6551</span> /** Make sure this is a valid row for the HRegion */<a name="line.6551"></a> |
| <span class="sourceLineNo">6552</span> void checkRow(byte[] row, String op) throws IOException {<a name="line.6552"></a> |
| <span class="sourceLineNo">6553</span> if (!rowIsInRange(getRegionInfo(), row)) {<a name="line.6553"></a> |
| <span class="sourceLineNo">6554</span> throw new WrongRegionException("Requested row out of range for " +<a name="line.6554"></a> |
| <span class="sourceLineNo">6555</span> op + " on HRegion " + this + ", startKey='" +<a name="line.6555"></a> |
| <span class="sourceLineNo">6556</span> Bytes.toStringBinary(getRegionInfo().getStartKey()) + "', getEndKey()='" +<a name="line.6556"></a> |
| <span class="sourceLineNo">6557</span> Bytes.toStringBinary(getRegionInfo().getEndKey()) + "', row='" +<a name="line.6557"></a> |
| <span class="sourceLineNo">6558</span> Bytes.toStringBinary(row) + "'");<a name="line.6558"></a> |
| <span class="sourceLineNo">6559</span> }<a name="line.6559"></a> |
| <span class="sourceLineNo">6560</span> }<a name="line.6560"></a> |
| <span class="sourceLineNo">6561</span><a name="line.6561"></a> |
| <span class="sourceLineNo">6562</span><a name="line.6562"></a> |
| <span class="sourceLineNo">6563</span> /**<a name="line.6563"></a> |
| <span class="sourceLineNo">6564</span> * Get an exclusive ( write lock ) lock on a given row.<a name="line.6564"></a> |
| <span class="sourceLineNo">6565</span> * @param row Which row to lock.<a name="line.6565"></a> |
| <span class="sourceLineNo">6566</span> * @return A locked RowLock. The lock is exclusive and already aqquired.<a name="line.6566"></a> |
| <span class="sourceLineNo">6567</span> * @throws IOException<a name="line.6567"></a> |
| <span class="sourceLineNo">6568</span> */<a name="line.6568"></a> |
| <span class="sourceLineNo">6569</span> public RowLock getRowLock(byte[] row) throws IOException {<a name="line.6569"></a> |
| <span class="sourceLineNo">6570</span> return getRowLock(row, false);<a name="line.6570"></a> |
| <span class="sourceLineNo">6571</span> }<a name="line.6571"></a> |
| <span class="sourceLineNo">6572</span><a name="line.6572"></a> |
| <span class="sourceLineNo">6573</span> @Override<a name="line.6573"></a> |
| <span class="sourceLineNo">6574</span> public RowLock getRowLock(byte[] row, boolean readLock) throws IOException {<a name="line.6574"></a> |
| <span class="sourceLineNo">6575</span> checkRow(row, "row lock");<a name="line.6575"></a> |
| <span class="sourceLineNo">6576</span> return getRowLockInternal(row, readLock, null);<a name="line.6576"></a> |
| <span class="sourceLineNo">6577</span> }<a name="line.6577"></a> |
| <span class="sourceLineNo">6578</span><a name="line.6578"></a> |
| <span class="sourceLineNo">6579</span> protected RowLock getRowLockInternal(byte[] row, boolean readLock, final RowLock prevRowLock)<a name="line.6579"></a> |
| <span class="sourceLineNo">6580</span> throws IOException {<a name="line.6580"></a> |
| <span class="sourceLineNo">6581</span> // create an object to use a a key in the row lock map<a name="line.6581"></a> |
| <span class="sourceLineNo">6582</span> HashedBytes rowKey = new HashedBytes(row);<a name="line.6582"></a> |
| <span class="sourceLineNo">6583</span><a name="line.6583"></a> |
| <span class="sourceLineNo">6584</span> RowLockContext rowLockContext = null;<a name="line.6584"></a> |
| <span class="sourceLineNo">6585</span> RowLockImpl result = null;<a name="line.6585"></a> |
| <span class="sourceLineNo">6586</span><a name="line.6586"></a> |
| <span class="sourceLineNo">6587</span> boolean success = false;<a name="line.6587"></a> |
| <span class="sourceLineNo">6588</span> try (TraceScope scope = TraceUtil.createTrace("HRegion.getRowLock")) {<a name="line.6588"></a> |
| <span class="sourceLineNo">6589</span> TraceUtil.addTimelineAnnotation("Getting a " + (readLock?"readLock":"writeLock"));<a name="line.6589"></a> |
| <span class="sourceLineNo">6590</span> // Keep trying until we have a lock or error out.<a name="line.6590"></a> |
| <span class="sourceLineNo">6591</span> // TODO: do we need to add a time component here?<a name="line.6591"></a> |
| <span class="sourceLineNo">6592</span> while (result == null) {<a name="line.6592"></a> |
| <span class="sourceLineNo">6593</span> rowLockContext = computeIfAbsent(lockedRows, rowKey, () -> new RowLockContext(rowKey));<a name="line.6593"></a> |
| <span class="sourceLineNo">6594</span> // Now try an get the lock.<a name="line.6594"></a> |
| <span class="sourceLineNo">6595</span> // This can fail as<a name="line.6595"></a> |
| <span class="sourceLineNo">6596</span> if (readLock) {<a name="line.6596"></a> |
| <span class="sourceLineNo">6597</span> // For read lock, if the caller has locked the same row previously, it will not try<a name="line.6597"></a> |
| <span class="sourceLineNo">6598</span> // to acquire the same read lock. It simply returns the previous row lock.<a name="line.6598"></a> |
| <span class="sourceLineNo">6599</span> RowLockImpl prevRowLockImpl = (RowLockImpl)prevRowLock;<a name="line.6599"></a> |
| <span class="sourceLineNo">6600</span> if ((prevRowLockImpl != null) && (prevRowLockImpl.getLock() ==<a name="line.6600"></a> |
| <span class="sourceLineNo">6601</span> rowLockContext.readWriteLock.readLock())) {<a name="line.6601"></a> |
| <span class="sourceLineNo">6602</span> success = true;<a name="line.6602"></a> |
| <span class="sourceLineNo">6603</span> return prevRowLock;<a name="line.6603"></a> |
| <span class="sourceLineNo">6604</span> }<a name="line.6604"></a> |
| <span class="sourceLineNo">6605</span> result = rowLockContext.newReadLock();<a name="line.6605"></a> |
| <span class="sourceLineNo">6606</span> } else {<a name="line.6606"></a> |
| <span class="sourceLineNo">6607</span> result = rowLockContext.newWriteLock();<a name="line.6607"></a> |
| <span class="sourceLineNo">6608</span> }<a name="line.6608"></a> |
| <span class="sourceLineNo">6609</span> }<a name="line.6609"></a> |
| <span class="sourceLineNo">6610</span><a name="line.6610"></a> |
| <span class="sourceLineNo">6611</span> int timeout = rowLockWaitDuration;<a name="line.6611"></a> |
| <span class="sourceLineNo">6612</span> boolean reachDeadlineFirst = false;<a name="line.6612"></a> |
| <span class="sourceLineNo">6613</span> Optional<RpcCall> call = RpcServer.getCurrentCall();<a name="line.6613"></a> |
| <span class="sourceLineNo">6614</span> if (call.isPresent()) {<a name="line.6614"></a> |
| <span class="sourceLineNo">6615</span> long deadline = call.get().getDeadline();<a name="line.6615"></a> |
| <span class="sourceLineNo">6616</span> if (deadline < Long.MAX_VALUE) {<a name="line.6616"></a> |
| <span class="sourceLineNo">6617</span> int timeToDeadline = (int) (deadline - System.currentTimeMillis());<a name="line.6617"></a> |
| <span class="sourceLineNo">6618</span> if (timeToDeadline <= this.rowLockWaitDuration) {<a name="line.6618"></a> |
| <span class="sourceLineNo">6619</span> reachDeadlineFirst = true;<a name="line.6619"></a> |
| <span class="sourceLineNo">6620</span> timeout = timeToDeadline;<a name="line.6620"></a> |
| <span class="sourceLineNo">6621</span> }<a name="line.6621"></a> |
| <span class="sourceLineNo">6622</span> }<a name="line.6622"></a> |
| <span class="sourceLineNo">6623</span> }<a name="line.6623"></a> |
| <span class="sourceLineNo">6624</span><a name="line.6624"></a> |
| <span class="sourceLineNo">6625</span> if (timeout <= 0 || !result.getLock().tryLock(timeout, TimeUnit.MILLISECONDS)) {<a name="line.6625"></a> |
| <span class="sourceLineNo">6626</span> TraceUtil.addTimelineAnnotation("Failed to get row lock");<a name="line.6626"></a> |
| <span class="sourceLineNo">6627</span> String message = "Timed out waiting for lock for row: " + rowKey + " in region "<a name="line.6627"></a> |
| <span class="sourceLineNo">6628</span> + getRegionInfo().getEncodedName();<a name="line.6628"></a> |
| <span class="sourceLineNo">6629</span> if (reachDeadlineFirst) {<a name="line.6629"></a> |
| <span class="sourceLineNo">6630</span> throw new TimeoutIOException(message);<a name="line.6630"></a> |
| <span class="sourceLineNo">6631</span> } else {<a name="line.6631"></a> |
| <span class="sourceLineNo">6632</span> // If timeToDeadline is larger than rowLockWaitDuration, we can not drop the request.<a name="line.6632"></a> |
| <span class="sourceLineNo">6633</span> throw new IOException(message);<a name="line.6633"></a> |
| <span class="sourceLineNo">6634</span> }<a name="line.6634"></a> |
| <span class="sourceLineNo">6635</span> }<a name="line.6635"></a> |
| <span class="sourceLineNo">6636</span> rowLockContext.setThreadName(Thread.currentThread().getName());<a name="line.6636"></a> |
| <span class="sourceLineNo">6637</span> success = true;<a name="line.6637"></a> |
| <span class="sourceLineNo">6638</span> return result;<a name="line.6638"></a> |
| <span class="sourceLineNo">6639</span> } catch (InterruptedException ie) {<a name="line.6639"></a> |
| <span class="sourceLineNo">6640</span> if (LOG.isDebugEnabled()) {<a name="line.6640"></a> |
| <span class="sourceLineNo">6641</span> LOG.debug("Thread interrupted waiting for lock on row: {}, in region {}", rowKey,<a name="line.6641"></a> |
| <span class="sourceLineNo">6642</span> getRegionInfo().getRegionNameAsString());<a name="line.6642"></a> |
| <span class="sourceLineNo">6643</span> }<a name="line.6643"></a> |
| <span class="sourceLineNo">6644</span> TraceUtil.addTimelineAnnotation("Interrupted exception getting row lock");<a name="line.6644"></a> |
| <span class="sourceLineNo">6645</span> throw throwOnInterrupt(ie);<a name="line.6645"></a> |
| <span class="sourceLineNo">6646</span> } catch (Error error) {<a name="line.6646"></a> |
| <span class="sourceLineNo">6647</span> // The maximum lock count for read lock is 64K (hardcoded), when this maximum count<a name="line.6647"></a> |
| <span class="sourceLineNo">6648</span> // is reached, it will throw out an Error. This Error needs to be caught so it can<a name="line.6648"></a> |
| <span class="sourceLineNo">6649</span> // go ahead to process the minibatch with lock acquired.<a name="line.6649"></a> |
| <span class="sourceLineNo">6650</span> LOG.warn("Error to get row lock for {}, in region {}, cause: {}", Bytes.toStringBinary(row),<a name="line.6650"></a> |
| <span class="sourceLineNo">6651</span> getRegionInfo().getRegionNameAsString(), error);<a name="line.6651"></a> |
| <span class="sourceLineNo">6652</span> IOException ioe = new IOException(error);<a name="line.6652"></a> |
| <span class="sourceLineNo">6653</span> TraceUtil.addTimelineAnnotation("Error getting row lock");<a name="line.6653"></a> |
| <span class="sourceLineNo">6654</span> throw ioe;<a name="line.6654"></a> |
| <span class="sourceLineNo">6655</span> } finally {<a name="line.6655"></a> |
| <span class="sourceLineNo">6656</span> // Clean up the counts just in case this was the thing keeping the context alive.<a name="line.6656"></a> |
| <span class="sourceLineNo">6657</span> if (!success && rowLockContext != null) {<a name="line.6657"></a> |
| <span class="sourceLineNo">6658</span> rowLockContext.cleanUp();<a name="line.6658"></a> |
| <span class="sourceLineNo">6659</span> }<a name="line.6659"></a> |
| <span class="sourceLineNo">6660</span> }<a name="line.6660"></a> |
| <span class="sourceLineNo">6661</span> }<a name="line.6661"></a> |
| <span class="sourceLineNo">6662</span><a name="line.6662"></a> |
| <span class="sourceLineNo">6663</span> private void releaseRowLocks(List<RowLock> rowLocks) {<a name="line.6663"></a> |
| <span class="sourceLineNo">6664</span> if (rowLocks != null) {<a name="line.6664"></a> |
| <span class="sourceLineNo">6665</span> for (RowLock rowLock : rowLocks) {<a name="line.6665"></a> |
| <span class="sourceLineNo">6666</span> rowLock.release();<a name="line.6666"></a> |
| <span class="sourceLineNo">6667</span> }<a name="line.6667"></a> |
| <span class="sourceLineNo">6668</span> rowLocks.clear();<a name="line.6668"></a> |
| <span class="sourceLineNo">6669</span> }<a name="line.6669"></a> |
| <span class="sourceLineNo">6670</span> }<a name="line.6670"></a> |
| <span class="sourceLineNo">6671</span><a name="line.6671"></a> |
| <span class="sourceLineNo">6672</span> public int getReadLockCount() {<a name="line.6672"></a> |
| <span class="sourceLineNo">6673</span> return lock.getReadLockCount();<a name="line.6673"></a> |
| <span class="sourceLineNo">6674</span> }<a name="line.6674"></a> |
| <span class="sourceLineNo">6675</span><a name="line.6675"></a> |
| <span class="sourceLineNo">6676</span> public ConcurrentHashMap<HashedBytes, RowLockContext> getLockedRows() {<a name="line.6676"></a> |
| <span class="sourceLineNo">6677</span> return lockedRows;<a name="line.6677"></a> |
| <span class="sourceLineNo">6678</span> }<a name="line.6678"></a> |
| <span class="sourceLineNo">6679</span><a name="line.6679"></a> |
| <span class="sourceLineNo">6680</span> class RowLockContext {<a name="line.6680"></a> |
| <span class="sourceLineNo">6681</span> private final HashedBytes row;<a name="line.6681"></a> |
| <span class="sourceLineNo">6682</span> final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(true);<a name="line.6682"></a> |
| <span class="sourceLineNo">6683</span> final AtomicBoolean usable = new AtomicBoolean(true);<a name="line.6683"></a> |
| <span class="sourceLineNo">6684</span> final AtomicInteger count = new AtomicInteger(0);<a name="line.6684"></a> |
| <span class="sourceLineNo">6685</span> final Object lock = new Object();<a name="line.6685"></a> |
| <span class="sourceLineNo">6686</span> private String threadName;<a name="line.6686"></a> |
| <span class="sourceLineNo">6687</span><a name="line.6687"></a> |
| <span class="sourceLineNo">6688</span> RowLockContext(HashedBytes row) {<a name="line.6688"></a> |
| <span class="sourceLineNo">6689</span> this.row = row;<a name="line.6689"></a> |
| <span class="sourceLineNo">6690</span> }<a name="line.6690"></a> |
| <span class="sourceLineNo">6691</span><a name="line.6691"></a> |
| <span class="sourceLineNo">6692</span> RowLockImpl newWriteLock() {<a name="line.6692"></a> |
| <span class="sourceLineNo">6693</span> Lock l = readWriteLock.writeLock();<a name="line.6693"></a> |
| <span class="sourceLineNo">6694</span> return getRowLock(l);<a name="line.6694"></a> |
| <span class="sourceLineNo">6695</span> }<a name="line.6695"></a> |
| <span class="sourceLineNo">6696</span> RowLockImpl newReadLock() {<a name="line.6696"></a> |
| <span class="sourceLineNo">6697</span> Lock l = readWriteLock.readLock();<a name="line.6697"></a> |
| <span class="sourceLineNo">6698</span> return getRowLock(l);<a name="line.6698"></a> |
| <span class="sourceLineNo">6699</span> }<a name="line.6699"></a> |
| <span class="sourceLineNo">6700</span><a name="line.6700"></a> |
| <span class="sourceLineNo">6701</span> private RowLockImpl getRowLock(Lock l) {<a name="line.6701"></a> |
| <span class="sourceLineNo">6702</span> count.incrementAndGet();<a name="line.6702"></a> |
| <span class="sourceLineNo">6703</span> synchronized (lock) {<a name="line.6703"></a> |
| <span class="sourceLineNo">6704</span> if (usable.get()) {<a name="line.6704"></a> |
| <span class="sourceLineNo">6705</span> return new RowLockImpl(this, l);<a name="line.6705"></a> |
| <span class="sourceLineNo">6706</span> } else {<a name="line.6706"></a> |
| <span class="sourceLineNo">6707</span> return null;<a name="line.6707"></a> |
| <span class="sourceLineNo">6708</span> }<a name="line.6708"></a> |
| <span class="sourceLineNo">6709</span> }<a name="line.6709"></a> |
| <span class="sourceLineNo">6710</span> }<a name="line.6710"></a> |
| <span class="sourceLineNo">6711</span><a name="line.6711"></a> |
| <span class="sourceLineNo">6712</span> void cleanUp() {<a name="line.6712"></a> |
| <span class="sourceLineNo">6713</span> long c = count.decrementAndGet();<a name="line.6713"></a> |
| <span class="sourceLineNo">6714</span> if (c <= 0) {<a name="line.6714"></a> |
| <span class="sourceLineNo">6715</span> synchronized (lock) {<a name="line.6715"></a> |
| <span class="sourceLineNo">6716</span> if (count.get() <= 0 && usable.get()){ // Don't attempt to remove row if already removed<a name="line.6716"></a> |
| <span class="sourceLineNo">6717</span> usable.set(false);<a name="line.6717"></a> |
| <span class="sourceLineNo">6718</span> RowLockContext removed = lockedRows.remove(row);<a name="line.6718"></a> |
| <span class="sourceLineNo">6719</span> assert removed == this: "we should never remove a different context";<a name="line.6719"></a> |
| <span class="sourceLineNo">6720</span> }<a name="line.6720"></a> |
| <span class="sourceLineNo">6721</span> }<a name="line.6721"></a> |
| <span class="sourceLineNo">6722</span> }<a name="line.6722"></a> |
| <span class="sourceLineNo">6723</span> }<a name="line.6723"></a> |
| <span class="sourceLineNo">6724</span><a name="line.6724"></a> |
| <span class="sourceLineNo">6725</span> public void setThreadName(String threadName) {<a name="line.6725"></a> |
| <span class="sourceLineNo">6726</span> this.threadName = threadName;<a name="line.6726"></a> |
| <span class="sourceLineNo">6727</span> }<a name="line.6727"></a> |
| <span class="sourceLineNo">6728</span><a name="line.6728"></a> |
| <span class="sourceLineNo">6729</span> @Override<a name="line.6729"></a> |
| <span class="sourceLineNo">6730</span> public String toString() {<a name="line.6730"></a> |
| <span class="sourceLineNo">6731</span> return "RowLockContext{" +<a name="line.6731"></a> |
| <span class="sourceLineNo">6732</span> "row=" + row +<a name="line.6732"></a> |
| <span class="sourceLineNo">6733</span> ", readWriteLock=" + readWriteLock +<a name="line.6733"></a> |
| <span class="sourceLineNo">6734</span> ", count=" + count +<a name="line.6734"></a> |
| <span class="sourceLineNo">6735</span> ", threadName=" + threadName +<a name="line.6735"></a> |
| <span class="sourceLineNo">6736</span> '}';<a name="line.6736"></a> |
| <span class="sourceLineNo">6737</span> }<a name="line.6737"></a> |
| <span class="sourceLineNo">6738</span> }<a name="line.6738"></a> |
| <span class="sourceLineNo">6739</span><a name="line.6739"></a> |
| <span class="sourceLineNo">6740</span> /**<a name="line.6740"></a> |
| <span class="sourceLineNo">6741</span> * Class used to represent a lock on a row.<a name="line.6741"></a> |
| <span class="sourceLineNo">6742</span> */<a name="line.6742"></a> |
| <span class="sourceLineNo">6743</span> public static class RowLockImpl implements RowLock {<a name="line.6743"></a> |
| <span class="sourceLineNo">6744</span> private final RowLockContext context;<a name="line.6744"></a> |
| <span class="sourceLineNo">6745</span> private final Lock lock;<a name="line.6745"></a> |
| <span class="sourceLineNo">6746</span><a name="line.6746"></a> |
| <span class="sourceLineNo">6747</span> public RowLockImpl(RowLockContext context, Lock lock) {<a name="line.6747"></a> |
| <span class="sourceLineNo">6748</span> this.context = context;<a name="line.6748"></a> |
| <span class="sourceLineNo">6749</span> this.lock = lock;<a name="line.6749"></a> |
| <span class="sourceLineNo">6750</span> }<a name="line.6750"></a> |
| <span class="sourceLineNo">6751</span><a name="line.6751"></a> |
| <span class="sourceLineNo">6752</span> public Lock getLock() {<a name="line.6752"></a> |
| <span class="sourceLineNo">6753</span> return lock;<a name="line.6753"></a> |
| <span class="sourceLineNo">6754</span> }<a name="line.6754"></a> |
| <span class="sourceLineNo">6755</span><a name="line.6755"></a> |
| <span class="sourceLineNo">6756</span> public RowLockContext getContext() {<a name="line.6756"></a> |
| <span class="sourceLineNo">6757</span> return context;<a name="line.6757"></a> |
| <span class="sourceLineNo">6758</span> }<a name="line.6758"></a> |
| <span class="sourceLineNo">6759</span><a name="line.6759"></a> |
| <span class="sourceLineNo">6760</span> @Override<a name="line.6760"></a> |
| <span class="sourceLineNo">6761</span> public void release() {<a name="line.6761"></a> |
| <span class="sourceLineNo">6762</span> lock.unlock();<a name="line.6762"></a> |
| <span class="sourceLineNo">6763</span> context.cleanUp();<a name="line.6763"></a> |
| <span class="sourceLineNo">6764</span> }<a name="line.6764"></a> |
| <span class="sourceLineNo">6765</span><a name="line.6765"></a> |
| <span class="sourceLineNo">6766</span> @Override<a name="line.6766"></a> |
| <span class="sourceLineNo">6767</span> public String toString() {<a name="line.6767"></a> |
| <span class="sourceLineNo">6768</span> return "RowLockImpl{" +<a name="line.6768"></a> |
| <span class="sourceLineNo">6769</span> "context=" + context +<a name="line.6769"></a> |
| <span class="sourceLineNo">6770</span> ", lock=" + lock +<a name="line.6770"></a> |
| <span class="sourceLineNo">6771</span> '}';<a name="line.6771"></a> |
| <span class="sourceLineNo">6772</span> }<a name="line.6772"></a> |
| <span class="sourceLineNo">6773</span> }<a name="line.6773"></a> |
| <span class="sourceLineNo">6774</span><a name="line.6774"></a> |
| <span class="sourceLineNo">6775</span> /**<a name="line.6775"></a> |
| <span class="sourceLineNo">6776</span> * Determines whether multiple column families are present<a name="line.6776"></a> |
| <span class="sourceLineNo">6777</span> * Precondition: familyPaths is not null<a name="line.6777"></a> |
| <span class="sourceLineNo">6778</span> *<a name="line.6778"></a> |
| <span class="sourceLineNo">6779</span> * @param familyPaths List of (column family, hfilePath)<a name="line.6779"></a> |
| <span class="sourceLineNo">6780</span> */<a name="line.6780"></a> |
| <span class="sourceLineNo">6781</span> private static boolean hasMultipleColumnFamilies(Collection<Pair<byte[], String>> familyPaths) {<a name="line.6781"></a> |
| <span class="sourceLineNo">6782</span> boolean multipleFamilies = false;<a name="line.6782"></a> |
| <span class="sourceLineNo">6783</span> byte[] family = null;<a name="line.6783"></a> |
| <span class="sourceLineNo">6784</span> for (Pair<byte[], String> pair : familyPaths) {<a name="line.6784"></a> |
| <span class="sourceLineNo">6785</span> byte[] fam = pair.getFirst();<a name="line.6785"></a> |
| <span class="sourceLineNo">6786</span> if (family == null) {<a name="line.6786"></a> |
| <span class="sourceLineNo">6787</span> family = fam;<a name="line.6787"></a> |
| <span class="sourceLineNo">6788</span> } else if (!Bytes.equals(family, fam)) {<a name="line.6788"></a> |
| <span class="sourceLineNo">6789</span> multipleFamilies = true;<a name="line.6789"></a> |
| <span class="sourceLineNo">6790</span> break;<a name="line.6790"></a> |
| <span class="sourceLineNo">6791</span> }<a name="line.6791"></a> |
| <span class="sourceLineNo">6792</span> }<a name="line.6792"></a> |
| <span class="sourceLineNo">6793</span> return multipleFamilies;<a name="line.6793"></a> |
| <span class="sourceLineNo">6794</span> }<a name="line.6794"></a> |
| <span class="sourceLineNo">6795</span><a name="line.6795"></a> |
| <span class="sourceLineNo">6796</span> /**<a name="line.6796"></a> |
| <span class="sourceLineNo">6797</span> * Attempts to atomically load a group of hfiles. This is critical for loading<a name="line.6797"></a> |
| <span class="sourceLineNo">6798</span> * rows with multiple column families atomically.<a name="line.6798"></a> |
| <span class="sourceLineNo">6799</span> *<a name="line.6799"></a> |
| <span class="sourceLineNo">6800</span> * @param familyPaths List of Pair&lt;byte[] column family, String hfilePath&gt;<a name="line.6800"></a> |
| <span class="sourceLineNo">6801</span> * @param bulkLoadListener Internal hooks enabling massaging/preparation of a<a name="line.6801"></a> |
| <span class="sourceLineNo">6802</span> * file about to be bulk loaded<a name="line.6802"></a> |
| <span class="sourceLineNo">6803</span> * @param assignSeqId<a name="line.6803"></a> |
| <span class="sourceLineNo">6804</span> * @return Map from family to List of store file paths if successful, null if failed recoverably<a name="line.6804"></a> |
| <span class="sourceLineNo">6805</span> * @throws IOException if failed unrecoverably.<a name="line.6805"></a> |
| <span class="sourceLineNo">6806</span> */<a name="line.6806"></a> |
| <span class="sourceLineNo">6807</span> public Map<byte[], List<Path>> bulkLoadHFiles(Collection<Pair<byte[], String>> familyPaths, boolean assignSeqId,<a name="line.6807"></a> |
| <span class="sourceLineNo">6808</span> BulkLoadListener bulkLoadListener) throws IOException {<a name="line.6808"></a> |
| <span class="sourceLineNo">6809</span> return bulkLoadHFiles(familyPaths, assignSeqId, bulkLoadListener, false,<a name="line.6809"></a> |
| <span class="sourceLineNo">6810</span> null, true);<a name="line.6810"></a> |
| <span class="sourceLineNo">6811</span> }<a name="line.6811"></a> |
| <span class="sourceLineNo">6812</span><a name="line.6812"></a> |
| <span class="sourceLineNo">6813</span> /**<a name="line.6813"></a> |
| <span class="sourceLineNo">6814</span> * Listener class to enable callers of<a name="line.6814"></a> |
| <span class="sourceLineNo">6815</span> * bulkLoadHFile() to perform any necessary<a name="line.6815"></a> |
| <span class="sourceLineNo">6816</span> * pre/post processing of a given bulkload call<a name="line.6816"></a> |
| <span class="sourceLineNo">6817</span> */<a name="line.6817"></a> |
| <span class="sourceLineNo">6818</span> public interface BulkLoadListener {<a name="line.6818"></a> |
| <span class="sourceLineNo">6819</span> /**<a name="line.6819"></a> |
| <span class="sourceLineNo">6820</span> * Called before an HFile is actually loaded<a name="line.6820"></a> |
| <span class="sourceLineNo">6821</span> * @param family family being loaded to<a name="line.6821"></a> |
| <span class="sourceLineNo">6822</span> * @param srcPath path of HFile<a name="line.6822"></a> |
| <span class="sourceLineNo">6823</span> * @return final path to be used for actual loading<a name="line.6823"></a> |
| <span class="sourceLineNo">6824</span> * @throws IOException<a name="line.6824"></a> |
| <span class="sourceLineNo">6825</span> */<a name="line.6825"></a> |
| <span class="sourceLineNo">6826</span> String prepareBulkLoad(byte[] family, String srcPath, boolean copyFile)<a name="line.6826"></a> |
| <span class="sourceLineNo">6827</span> throws IOException;<a name="line.6827"></a> |
| <span class="sourceLineNo">6828</span><a name="line.6828"></a> |
| <span class="sourceLineNo">6829</span> /**<a name="line.6829"></a> |
| <span class="sourceLineNo">6830</span> * Called after a successful HFile load<a name="line.6830"></a> |
| <span class="sourceLineNo">6831</span> * @param family family being loaded to<a name="line.6831"></a> |
| <span class="sourceLineNo">6832</span> * @param srcPath path of HFile<a name="line.6832"></a> |
| <span class="sourceLineNo">6833</span> * @throws IOException<a name="line.6833"></a> |
| <span class="sourceLineNo">6834</span> */<a name="line.6834"></a> |
| <span class="sourceLineNo">6835</span> void doneBulkLoad(byte[] family, String srcPath) throws IOException;<a name="line.6835"></a> |
| <span class="sourceLineNo">6836</span><a name="line.6836"></a> |
| <span class="sourceLineNo">6837</span> /**<a name="line.6837"></a> |
| <span class="sourceLineNo">6838</span> * Called after a failed HFile load<a name="line.6838"></a> |
| <span class="sourceLineNo">6839</span> * @param family family being loaded to<a name="line.6839"></a> |
| <span class="sourceLineNo">6840</span> * @param srcPath path of HFile<a name="line.6840"></a> |
| <span class="sourceLineNo">6841</span> * @throws IOException<a name="line.6841"></a> |
| <span class="sourceLineNo">6842</span> */<a name="line.6842"></a> |
| <span class="sourceLineNo">6843</span> void failedBulkLoad(byte[] family, String srcPath) throws IOException;<a name="line.6843"></a> |
| <span class="sourceLineNo">6844</span> }<a name="line.6844"></a> |
| <span class="sourceLineNo">6845</span><a name="line.6845"></a> |
| <span class="sourceLineNo">6846</span> /**<a name="line.6846"></a> |
| <span class="sourceLineNo">6847</span> * Attempts to atomically load a group of hfiles. This is critical for loading<a name="line.6847"></a> |
| <span class="sourceLineNo">6848</span> * rows with multiple column families atomically.<a name="line.6848"></a> |
| <span class="sourceLineNo">6849</span> *<a name="line.6849"></a> |
| <span class="sourceLineNo">6850</span> * @param familyPaths List of Pair&lt;byte[] column family, String hfilePath&gt;<a name="line.6850"></a> |
| <span class="sourceLineNo">6851</span> * @param assignSeqId<a name="line.6851"></a> |
| <span class="sourceLineNo">6852</span> * @param bulkLoadListener Internal hooks enabling massaging/preparation of a<a name="line.6852"></a> |
| <span class="sourceLineNo">6853</span> * file about to be bulk loaded<a name="line.6853"></a> |
| <span class="sourceLineNo">6854</span> * @param copyFile always copy hfiles if true<a name="line.6854"></a> |
| <span class="sourceLineNo">6855</span> * @param clusterIds ids from clusters that had already handled the given bulkload event.<a name="line.6855"></a> |
| <span class="sourceLineNo">6856</span> * @return Map from family to List of store file paths if successful, null if failed recoverably<a name="line.6856"></a> |
| <span class="sourceLineNo">6857</span> * @throws IOException if failed unrecoverably.<a name="line.6857"></a> |
| <span class="sourceLineNo">6858</span> */<a name="line.6858"></a> |
| <span class="sourceLineNo">6859</span> public Map<byte[], List<Path>> bulkLoadHFiles(Collection<Pair<byte[], String>> familyPaths,<a name="line.6859"></a> |
| <span class="sourceLineNo">6860</span> boolean assignSeqId, BulkLoadListener bulkLoadListener, boolean copyFile,<a name="line.6860"></a> |
| <span class="sourceLineNo">6861</span> List<String> clusterIds, boolean replicate) throws IOException {<a name="line.6861"></a> |
| <span class="sourceLineNo">6862</span> long seqId = -1;<a name="line.6862"></a> |
| <span class="sourceLineNo">6863</span> Map<byte[], List<Path>> storeFiles = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.6863"></a> |
| <span class="sourceLineNo">6864</span> Map<String, Long> storeFilesSizes = new HashMap<>();<a name="line.6864"></a> |
| <span class="sourceLineNo">6865</span> Preconditions.checkNotNull(familyPaths);<a name="line.6865"></a> |
| <span class="sourceLineNo">6866</span> // we need writeLock for multi-family bulk load<a name="line.6866"></a> |
| <span class="sourceLineNo">6867</span> startBulkRegionOperation(hasMultipleColumnFamilies(familyPaths));<a name="line.6867"></a> |
| <span class="sourceLineNo">6868</span> boolean isSuccessful = false;<a name="line.6868"></a> |
| <span class="sourceLineNo">6869</span> try {<a name="line.6869"></a> |
| <span class="sourceLineNo">6870</span> this.writeRequestsCount.increment();<a name="line.6870"></a> |
| <span class="sourceLineNo">6871</span><a name="line.6871"></a> |
| <span class="sourceLineNo">6872</span> // There possibly was a split that happened between when the split keys<a name="line.6872"></a> |
| <span class="sourceLineNo">6873</span> // were gathered and before the HRegion's write lock was taken. We need<a name="line.6873"></a> |
| <span class="sourceLineNo">6874</span> // to validate the HFile region before attempting to bulk load all of them<a name="line.6874"></a> |
| <span class="sourceLineNo">6875</span> IOException ioException = null;<a name="line.6875"></a> |
| <span class="sourceLineNo">6876</span> List<Pair<byte[], String>> failures = new ArrayList<>();<a name="line.6876"></a> |
| <span class="sourceLineNo">6877</span> for (Pair<byte[], String> p : familyPaths) {<a name="line.6877"></a> |
| <span class="sourceLineNo">6878</span> byte[] familyName = p.getFirst();<a name="line.6878"></a> |
| <span class="sourceLineNo">6879</span> String path = p.getSecond();<a name="line.6879"></a> |
| <span class="sourceLineNo">6880</span><a name="line.6880"></a> |
| <span class="sourceLineNo">6881</span> HStore store = getStore(familyName);<a name="line.6881"></a> |
| <span class="sourceLineNo">6882</span> if (store == null) {<a name="line.6882"></a> |
| <span class="sourceLineNo">6883</span> ioException = new org.apache.hadoop.hbase.DoNotRetryIOException(<a name="line.6883"></a> |
| <span class="sourceLineNo">6884</span> "No such column family " + Bytes.toStringBinary(familyName));<a name="line.6884"></a> |
| <span class="sourceLineNo">6885</span> } else {<a name="line.6885"></a> |
| <span class="sourceLineNo">6886</span> try {<a name="line.6886"></a> |
| <span class="sourceLineNo">6887</span> store.assertBulkLoadHFileOk(new Path(path));<a name="line.6887"></a> |
| <span class="sourceLineNo">6888</span> } catch (WrongRegionException wre) {<a name="line.6888"></a> |
| <span class="sourceLineNo">6889</span> // recoverable (file doesn't fit in region)<a name="line.6889"></a> |
| <span class="sourceLineNo">6890</span> failures.add(p);<a name="line.6890"></a> |
| <span class="sourceLineNo">6891</span> } catch (IOException ioe) {<a name="line.6891"></a> |
| <span class="sourceLineNo">6892</span> // unrecoverable (hdfs problem)<a name="line.6892"></a> |
| <span class="sourceLineNo">6893</span> ioException = ioe;<a name="line.6893"></a> |
| <span class="sourceLineNo">6894</span> }<a name="line.6894"></a> |
| <span class="sourceLineNo">6895</span> }<a name="line.6895"></a> |
| <span class="sourceLineNo">6896</span><a name="line.6896"></a> |
| <span class="sourceLineNo">6897</span> // validation failed because of some sort of IO problem.<a name="line.6897"></a> |
| <span class="sourceLineNo">6898</span> if (ioException != null) {<a name="line.6898"></a> |
| <span class="sourceLineNo">6899</span> LOG.error("There was IO error when checking if the bulk load is ok in region {}.", this,<a name="line.6899"></a> |
| <span class="sourceLineNo">6900</span> ioException);<a name="line.6900"></a> |
| <span class="sourceLineNo">6901</span> throw ioException;<a name="line.6901"></a> |
| <span class="sourceLineNo">6902</span> }<a name="line.6902"></a> |
| <span class="sourceLineNo">6903</span> }<a name="line.6903"></a> |
| <span class="sourceLineNo">6904</span> // validation failed, bail out before doing anything permanent.<a name="line.6904"></a> |
| <span class="sourceLineNo">6905</span> if (failures.size() != 0) {<a name="line.6905"></a> |
| <span class="sourceLineNo">6906</span> StringBuilder list = new StringBuilder();<a name="line.6906"></a> |
| <span class="sourceLineNo">6907</span> for (Pair<byte[], String> p : failures) {<a name="line.6907"></a> |
| <span class="sourceLineNo">6908</span> list.append("\n").append(Bytes.toString(p.getFirst())).append(" : ")<a name="line.6908"></a> |
| <span class="sourceLineNo">6909</span> .append(p.getSecond());<a name="line.6909"></a> |
| <span class="sourceLineNo">6910</span> }<a name="line.6910"></a> |
| <span class="sourceLineNo">6911</span> // problem when validating<a name="line.6911"></a> |
| <span class="sourceLineNo">6912</span> LOG.warn("There was a recoverable bulk load failure likely due to a split. These (family,"<a name="line.6912"></a> |
| <span class="sourceLineNo">6913</span> + " HFile) pairs were not loaded: {}, in region {}", list.toString(), this);<a name="line.6913"></a> |
| <span class="sourceLineNo">6914</span> return null;<a name="line.6914"></a> |
| <span class="sourceLineNo">6915</span> }<a name="line.6915"></a> |
| <span class="sourceLineNo">6916</span><a name="line.6916"></a> |
| <span class="sourceLineNo">6917</span> // We need to assign a sequential ID that's in between two memstores in order to preserve<a name="line.6917"></a> |
| <span class="sourceLineNo">6918</span> // the guarantee that all the edits lower than the highest sequential ID from all the<a name="line.6918"></a> |
| <span class="sourceLineNo">6919</span> // HFiles are flushed on disk. See HBASE-10958. The sequence id returned when we flush is<a name="line.6919"></a> |
| <span class="sourceLineNo">6920</span> // guaranteed to be one beyond the file made when we flushed (or if nothing to flush, it is<a name="line.6920"></a> |
| <span class="sourceLineNo">6921</span> // a sequence id that we can be sure is beyond the last hfile written).<a name="line.6921"></a> |
| <span class="sourceLineNo">6922</span> if (assignSeqId) {<a name="line.6922"></a> |
| <span class="sourceLineNo">6923</span> FlushResult fs = flushcache(true, false, FlushLifeCycleTracker.DUMMY);<a name="line.6923"></a> |
| <span class="sourceLineNo">6924</span> if (fs.isFlushSucceeded()) {<a name="line.6924"></a> |
| <span class="sourceLineNo">6925</span> seqId = ((FlushResultImpl)fs).flushSequenceId;<a name="line.6925"></a> |
| <span class="sourceLineNo">6926</span> } else if (fs.getResult() == FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY) {<a name="line.6926"></a> |
| <span class="sourceLineNo">6927</span> seqId = ((FlushResultImpl)fs).flushSequenceId;<a name="line.6927"></a> |
| <span class="sourceLineNo">6928</span> } else if (fs.getResult() == FlushResult.Result.CANNOT_FLUSH) {<a name="line.6928"></a> |
| <span class="sourceLineNo">6929</span> // CANNOT_FLUSH may mean that a flush is already on-going<a name="line.6929"></a> |
| <span class="sourceLineNo">6930</span> // we need to wait for that flush to complete<a name="line.6930"></a> |
| <span class="sourceLineNo">6931</span> waitForFlushes();<a name="line.6931"></a> |
| <span class="sourceLineNo">6932</span> } else {<a name="line.6932"></a> |
| <span class="sourceLineNo">6933</span> throw new IOException("Could not bulk load with an assigned sequential ID because the "+<a name="line.6933"></a> |
| <span class="sourceLineNo">6934</span> "flush didn't run. Reason for not flushing: " + ((FlushResultImpl)fs).failureReason);<a name="line.6934"></a> |
| <span class="sourceLineNo">6935</span> }<a name="line.6935"></a> |
| <span class="sourceLineNo">6936</span> }<a name="line.6936"></a> |
| <span class="sourceLineNo">6937</span><a name="line.6937"></a> |
| <span class="sourceLineNo">6938</span> Map<byte[], List<Pair<Path, Path>>> familyWithFinalPath =<a name="line.6938"></a> |
| <span class="sourceLineNo">6939</span> new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.6939"></a> |
| <span class="sourceLineNo">6940</span> for (Pair<byte[], String> p : familyPaths) {<a name="line.6940"></a> |
| <span class="sourceLineNo">6941</span> byte[] familyName = p.getFirst();<a name="line.6941"></a> |
| <span class="sourceLineNo">6942</span> String path = p.getSecond();<a name="line.6942"></a> |
| <span class="sourceLineNo">6943</span> HStore store = getStore(familyName);<a name="line.6943"></a> |
| <span class="sourceLineNo">6944</span> if (!familyWithFinalPath.containsKey(familyName)) {<a name="line.6944"></a> |
| <span class="sourceLineNo">6945</span> familyWithFinalPath.put(familyName, new ArrayList<>());<a name="line.6945"></a> |
| <span class="sourceLineNo">6946</span> }<a name="line.6946"></a> |
| <span class="sourceLineNo">6947</span> List<Pair<Path, Path>> lst = familyWithFinalPath.get(familyName);<a name="line.6947"></a> |
| <span class="sourceLineNo">6948</span> try {<a name="line.6948"></a> |
| <span class="sourceLineNo">6949</span> String finalPath = path;<a name="line.6949"></a> |
| <span class="sourceLineNo">6950</span> if (bulkLoadListener != null) {<a name="line.6950"></a> |
| <span class="sourceLineNo">6951</span> finalPath = bulkLoadListener.prepareBulkLoad(familyName, path, copyFile);<a name="line.6951"></a> |
| <span class="sourceLineNo">6952</span> }<a name="line.6952"></a> |
| <span class="sourceLineNo">6953</span> Pair<Path, Path> pair = store.preBulkLoadHFile(finalPath, seqId);<a name="line.6953"></a> |
| <span class="sourceLineNo">6954</span> lst.add(pair);<a name="line.6954"></a> |
| <span class="sourceLineNo">6955</span> } catch (IOException ioe) {<a name="line.6955"></a> |
| <span class="sourceLineNo">6956</span> // A failure here can cause an atomicity violation that we currently<a name="line.6956"></a> |
| <span class="sourceLineNo">6957</span> // cannot recover from since it is likely a failed HDFS operation.<a name="line.6957"></a> |
| <span class="sourceLineNo">6958</span><a name="line.6958"></a> |
| <span class="sourceLineNo">6959</span> LOG.error("There was a partial failure due to IO when attempting to" +<a name="line.6959"></a> |
| <span class="sourceLineNo">6960</span> " load " + Bytes.toString(p.getFirst()) + " : " + p.getSecond(), ioe);<a name="line.6960"></a> |
| <span class="sourceLineNo">6961</span> if (bulkLoadListener != null) {<a name="line.6961"></a> |
| <span class="sourceLineNo">6962</span> try {<a name="line.6962"></a> |
| <span class="sourceLineNo">6963</span> bulkLoadListener.failedBulkLoad(familyName, path);<a name="line.6963"></a> |
| <span class="sourceLineNo">6964</span> } catch (Exception ex) {<a name="line.6964"></a> |
| <span class="sourceLineNo">6965</span> LOG.error("Error while calling failedBulkLoad for family " +<a name="line.6965"></a> |
| <span class="sourceLineNo">6966</span> Bytes.toString(familyName) + " with path " + path, ex);<a name="line.6966"></a> |
| <span class="sourceLineNo">6967</span> }<a name="line.6967"></a> |
| <span class="sourceLineNo">6968</span> }<a name="line.6968"></a> |
| <span class="sourceLineNo">6969</span> throw ioe;<a name="line.6969"></a> |
| <span class="sourceLineNo">6970</span> }<a name="line.6970"></a> |
| <span class="sourceLineNo">6971</span> }<a name="line.6971"></a> |
| <span class="sourceLineNo">6972</span><a name="line.6972"></a> |
| <span class="sourceLineNo">6973</span> if (this.getCoprocessorHost() != null) {<a name="line.6973"></a> |
| <span class="sourceLineNo">6974</span> for (Map.Entry<byte[], List<Pair<Path, Path>>> entry : familyWithFinalPath.entrySet()) {<a name="line.6974"></a> |
| <span class="sourceLineNo">6975</span> this.getCoprocessorHost().preCommitStoreFile(entry.getKey(), entry.getValue());<a name="line.6975"></a> |
| <span class="sourceLineNo">6976</span> }<a name="line.6976"></a> |
| <span class="sourceLineNo">6977</span> }<a name="line.6977"></a> |
| <span class="sourceLineNo">6978</span> for (Map.Entry<byte[], List<Pair<Path, Path>>> entry : familyWithFinalPath.entrySet()) {<a name="line.6978"></a> |
| <span class="sourceLineNo">6979</span> byte[] familyName = entry.getKey();<a name="line.6979"></a> |
| <span class="sourceLineNo">6980</span> for (Pair<Path, Path> p : entry.getValue()) {<a name="line.6980"></a> |
| <span class="sourceLineNo">6981</span> String path = p.getFirst().toString();<a name="line.6981"></a> |
| <span class="sourceLineNo">6982</span> Path commitedStoreFile = p.getSecond();<a name="line.6982"></a> |
| <span class="sourceLineNo">6983</span> HStore store = getStore(familyName);<a name="line.6983"></a> |
| <span class="sourceLineNo">6984</span> try {<a name="line.6984"></a> |
| <span class="sourceLineNo">6985</span> store.bulkLoadHFile(familyName, path, commitedStoreFile);<a name="line.6985"></a> |
| <span class="sourceLineNo">6986</span> // Note the size of the store file<a name="line.6986"></a> |
| <span class="sourceLineNo">6987</span> try {<a name="line.6987"></a> |
| <span class="sourceLineNo">6988</span> FileSystem fs = commitedStoreFile.getFileSystem(baseConf);<a name="line.6988"></a> |
| <span class="sourceLineNo">6989</span> storeFilesSizes.put(commitedStoreFile.getName(), fs.getFileStatus(commitedStoreFile)<a name="line.6989"></a> |
| <span class="sourceLineNo">6990</span> .getLen());<a name="line.6990"></a> |
| <span class="sourceLineNo">6991</span> } catch (IOException e) {<a name="line.6991"></a> |
| <span class="sourceLineNo">6992</span> LOG.warn("Failed to find the size of hfile " + commitedStoreFile, e);<a name="line.6992"></a> |
| <span class="sourceLineNo">6993</span> storeFilesSizes.put(commitedStoreFile.getName(), 0L);<a name="line.6993"></a> |
| <span class="sourceLineNo">6994</span> }<a name="line.6994"></a> |
| <span class="sourceLineNo">6995</span><a name="line.6995"></a> |
| <span class="sourceLineNo">6996</span> if(storeFiles.containsKey(familyName)) {<a name="line.6996"></a> |
| <span class="sourceLineNo">6997</span> storeFiles.get(familyName).add(commitedStoreFile);<a name="line.6997"></a> |
| <span class="sourceLineNo">6998</span> } else {<a name="line.6998"></a> |
| <span class="sourceLineNo">6999</span> List<Path> storeFileNames = new ArrayList<>();<a name="line.6999"></a> |
| <span class="sourceLineNo">7000</span> storeFileNames.add(commitedStoreFile);<a name="line.7000"></a> |
| <span class="sourceLineNo">7001</span> storeFiles.put(familyName, storeFileNames);<a name="line.7001"></a> |
| <span class="sourceLineNo">7002</span> }<a name="line.7002"></a> |
| <span class="sourceLineNo">7003</span> if (bulkLoadListener != null) {<a name="line.7003"></a> |
| <span class="sourceLineNo">7004</span> bulkLoadListener.doneBulkLoad(familyName, path);<a name="line.7004"></a> |
| <span class="sourceLineNo">7005</span> }<a name="line.7005"></a> |
| <span class="sourceLineNo">7006</span> } catch (IOException ioe) {<a name="line.7006"></a> |
| <span class="sourceLineNo">7007</span> // A failure here can cause an atomicity violation that we currently<a name="line.7007"></a> |
| <span class="sourceLineNo">7008</span> // cannot recover from since it is likely a failed HDFS operation.<a name="line.7008"></a> |
| <span class="sourceLineNo">7009</span><a name="line.7009"></a> |
| <span class="sourceLineNo">7010</span> // TODO Need a better story for reverting partial failures due to HDFS.<a name="line.7010"></a> |
| <span class="sourceLineNo">7011</span> LOG.error("There was a partial failure due to IO when attempting to" +<a name="line.7011"></a> |
| <span class="sourceLineNo">7012</span> " load " + Bytes.toString(familyName) + " : " + p.getSecond(), ioe);<a name="line.7012"></a> |
| <span class="sourceLineNo">7013</span> if (bulkLoadListener != null) {<a name="line.7013"></a> |
| <span class="sourceLineNo">7014</span> try {<a name="line.7014"></a> |
| <span class="sourceLineNo">7015</span> bulkLoadListener.failedBulkLoad(familyName, path);<a name="line.7015"></a> |
| <span class="sourceLineNo">7016</span> } catch (Exception ex) {<a name="line.7016"></a> |
| <span class="sourceLineNo">7017</span> LOG.error("Error while calling failedBulkLoad for family " +<a name="line.7017"></a> |
| <span class="sourceLineNo">7018</span> Bytes.toString(familyName) + " with path " + path, ex);<a name="line.7018"></a> |
| <span class="sourceLineNo">7019</span> }<a name="line.7019"></a> |
| <span class="sourceLineNo">7020</span> }<a name="line.7020"></a> |
| <span class="sourceLineNo">7021</span> throw ioe;<a name="line.7021"></a> |
| <span class="sourceLineNo">7022</span> }<a name="line.7022"></a> |
| <span class="sourceLineNo">7023</span> }<a name="line.7023"></a> |
| <span class="sourceLineNo">7024</span> }<a name="line.7024"></a> |
| <span class="sourceLineNo">7025</span><a name="line.7025"></a> |
| <span class="sourceLineNo">7026</span> isSuccessful = true;<a name="line.7026"></a> |
| <span class="sourceLineNo">7027</span> //request compaction<a name="line.7027"></a> |
| <span class="sourceLineNo">7028</span> familyWithFinalPath.keySet().forEach(family -> {<a name="line.7028"></a> |
| <span class="sourceLineNo">7029</span> HStore store = getStore(family);<a name="line.7029"></a> |
| <span class="sourceLineNo">7030</span> try {<a name="line.7030"></a> |
| <span class="sourceLineNo">7031</span> if (this.rsServices != null && store.needsCompaction()) {<a name="line.7031"></a> |
| <span class="sourceLineNo">7032</span> this.rsServices.getCompactionRequestor().requestCompaction(this, store,<a name="line.7032"></a> |
| <span class="sourceLineNo">7033</span> "bulkload hfiles request compaction", Store.PRIORITY_USER + 1,<a name="line.7033"></a> |
| <span class="sourceLineNo">7034</span> CompactionLifeCycleTracker.DUMMY, null);<a name="line.7034"></a> |
| <span class="sourceLineNo">7035</span> }<a name="line.7035"></a> |
| <span class="sourceLineNo">7036</span> } catch (IOException e) {<a name="line.7036"></a> |
| <span class="sourceLineNo">7037</span> LOG.error("bulkload hfiles request compaction error ", e);<a name="line.7037"></a> |
| <span class="sourceLineNo">7038</span> }<a name="line.7038"></a> |
| <span class="sourceLineNo">7039</span> });<a name="line.7039"></a> |
| <span class="sourceLineNo">7040</span> } finally {<a name="line.7040"></a> |
| <span class="sourceLineNo">7041</span> if (wal != null && !storeFiles.isEmpty()) {<a name="line.7041"></a> |
| <span class="sourceLineNo">7042</span> // Write a bulk load event for hfiles that are loaded<a name="line.7042"></a> |
| <span class="sourceLineNo">7043</span> try {<a name="line.7043"></a> |
| <span class="sourceLineNo">7044</span> WALProtos.BulkLoadDescriptor loadDescriptor =<a name="line.7044"></a> |
| <span class="sourceLineNo">7045</span> ProtobufUtil.toBulkLoadDescriptor(this.getRegionInfo().getTable(),<a name="line.7045"></a> |
| <span class="sourceLineNo">7046</span> UnsafeByteOperations.unsafeWrap(this.getRegionInfo().getEncodedNameAsBytes()),<a name="line.7046"></a> |
| <span class="sourceLineNo">7047</span> storeFiles, storeFilesSizes, seqId, clusterIds, replicate);<a name="line.7047"></a> |
| <span class="sourceLineNo">7048</span> WALUtil.writeBulkLoadMarkerAndSync(this.wal, this.getReplicationScope(), getRegionInfo(),<a name="line.7048"></a> |
| <span class="sourceLineNo">7049</span> loadDescriptor, mvcc);<a name="line.7049"></a> |
| <span class="sourceLineNo">7050</span> } catch (IOException ioe) {<a name="line.7050"></a> |
| <span class="sourceLineNo">7051</span> if (this.rsServices != null) {<a name="line.7051"></a> |
| <span class="sourceLineNo">7052</span> // Have to abort region server because some hfiles has been loaded but we can't write<a name="line.7052"></a> |
| <span class="sourceLineNo">7053</span> // the event into WAL<a name="line.7053"></a> |
| <span class="sourceLineNo">7054</span> isSuccessful = false;<a name="line.7054"></a> |
| <span class="sourceLineNo">7055</span> this.rsServices.abort("Failed to write bulk load event into WAL.", ioe);<a name="line.7055"></a> |
| <span class="sourceLineNo">7056</span> }<a name="line.7056"></a> |
| <span class="sourceLineNo">7057</span> }<a name="line.7057"></a> |
| <span class="sourceLineNo">7058</span> }<a name="line.7058"></a> |
| <span class="sourceLineNo">7059</span><a name="line.7059"></a> |
| <span class="sourceLineNo">7060</span> closeBulkRegionOperation();<a name="line.7060"></a> |
| <span class="sourceLineNo">7061</span> }<a name="line.7061"></a> |
| <span class="sourceLineNo">7062</span> return isSuccessful ? storeFiles : null;<a name="line.7062"></a> |
| <span class="sourceLineNo">7063</span> }<a name="line.7063"></a> |
| <span class="sourceLineNo">7064</span><a name="line.7064"></a> |
| <span class="sourceLineNo">7065</span> @Override<a name="line.7065"></a> |
| <span class="sourceLineNo">7066</span> public boolean equals(Object o) {<a name="line.7066"></a> |
| <span class="sourceLineNo">7067</span> return o instanceof HRegion && Bytes.equals(getRegionInfo().getRegionName(),<a name="line.7067"></a> |
| <span class="sourceLineNo">7068</span> ((HRegion) o).getRegionInfo().getRegionName());<a name="line.7068"></a> |
| <span class="sourceLineNo">7069</span> }<a name="line.7069"></a> |
| <span class="sourceLineNo">7070</span><a name="line.7070"></a> |
| <span class="sourceLineNo">7071</span> @Override<a name="line.7071"></a> |
| <span class="sourceLineNo">7072</span> public int hashCode() {<a name="line.7072"></a> |
| <span class="sourceLineNo">7073</span> return Bytes.hashCode(getRegionInfo().getRegionName());<a name="line.7073"></a> |
| <span class="sourceLineNo">7074</span> }<a name="line.7074"></a> |
| <span class="sourceLineNo">7075</span><a name="line.7075"></a> |
| <span class="sourceLineNo">7076</span> @Override<a name="line.7076"></a> |
| <span class="sourceLineNo">7077</span> public String toString() {<a name="line.7077"></a> |
| <span class="sourceLineNo">7078</span> return getRegionInfo().getRegionNameAsString();<a name="line.7078"></a> |
| <span class="sourceLineNo">7079</span> }<a name="line.7079"></a> |
| <span class="sourceLineNo">7080</span><a name="line.7080"></a> |
| <span class="sourceLineNo">7081</span> // Utility methods<a name="line.7081"></a> |
| <span class="sourceLineNo">7082</span> /**<a name="line.7082"></a> |
| <span class="sourceLineNo">7083</span> * A utility method to create new instances of HRegion based on the {@link HConstants#REGION_IMPL}<a name="line.7083"></a> |
| <span class="sourceLineNo">7084</span> * configuration property.<a name="line.7084"></a> |
| <span class="sourceLineNo">7085</span> * @param tableDir qualified path of directory where region should be located, usually the table<a name="line.7085"></a> |
| <span class="sourceLineNo">7086</span> * directory.<a name="line.7086"></a> |
| <span class="sourceLineNo">7087</span> * @param wal The WAL is the outbound log for any updates to the HRegion The wal file is a logfile<a name="line.7087"></a> |
| <span class="sourceLineNo">7088</span> * from the previous execution that's custom-computed for this HRegion. The HRegionServer<a name="line.7088"></a> |
| <span class="sourceLineNo">7089</span> * computes and sorts the appropriate wal info for this HRegion. If there is a previous<a name="line.7089"></a> |
| <span class="sourceLineNo">7090</span> * file (implying that the HRegion has been written-to before), then read it from the<a name="line.7090"></a> |
| <span class="sourceLineNo">7091</span> * supplied path.<a name="line.7091"></a> |
| <span class="sourceLineNo">7092</span> * @param fs is the filesystem.<a name="line.7092"></a> |
| <span class="sourceLineNo">7093</span> * @param conf is global configuration settings.<a name="line.7093"></a> |
| <span class="sourceLineNo">7094</span> * @param regionInfo - RegionInfo that describes the region is new), then read them from the<a name="line.7094"></a> |
| <span class="sourceLineNo">7095</span> * supplied path.<a name="line.7095"></a> |
| <span class="sourceLineNo">7096</span> * @param htd the table descriptor<a name="line.7096"></a> |
| <span class="sourceLineNo">7097</span> * @return the new instance<a name="line.7097"></a> |
| <span class="sourceLineNo">7098</span> */<a name="line.7098"></a> |
| <span class="sourceLineNo">7099</span> public static HRegion newHRegion(Path tableDir, WAL wal, FileSystem fs,<a name="line.7099"></a> |
| <span class="sourceLineNo">7100</span> Configuration conf, RegionInfo regionInfo, final TableDescriptor htd,<a name="line.7100"></a> |
| <span class="sourceLineNo">7101</span> RegionServerServices rsServices) {<a name="line.7101"></a> |
| <span class="sourceLineNo">7102</span> try {<a name="line.7102"></a> |
| <span class="sourceLineNo">7103</span> @SuppressWarnings("unchecked")<a name="line.7103"></a> |
| <span class="sourceLineNo">7104</span> Class<? extends HRegion> regionClass =<a name="line.7104"></a> |
| <span class="sourceLineNo">7105</span> (Class<? extends HRegion>) conf.getClass(HConstants.REGION_IMPL, HRegion.class);<a name="line.7105"></a> |
| <span class="sourceLineNo">7106</span><a name="line.7106"></a> |
| <span class="sourceLineNo">7107</span> Constructor<? extends HRegion> c =<a name="line.7107"></a> |
| <span class="sourceLineNo">7108</span> regionClass.getConstructor(Path.class, WAL.class, FileSystem.class,<a name="line.7108"></a> |
| <span class="sourceLineNo">7109</span> Configuration.class, RegionInfo.class, TableDescriptor.class,<a name="line.7109"></a> |
| <span class="sourceLineNo">7110</span> RegionServerServices.class);<a name="line.7110"></a> |
| <span class="sourceLineNo">7111</span><a name="line.7111"></a> |
| <span class="sourceLineNo">7112</span> return c.newInstance(tableDir, wal, fs, conf, regionInfo, htd, rsServices);<a name="line.7112"></a> |
| <span class="sourceLineNo">7113</span> } catch (Throwable e) {<a name="line.7113"></a> |
| <span class="sourceLineNo">7114</span> // todo: what should I throw here?<a name="line.7114"></a> |
| <span class="sourceLineNo">7115</span> throw new IllegalStateException("Could not instantiate a region instance.", e);<a name="line.7115"></a> |
| <span class="sourceLineNo">7116</span> }<a name="line.7116"></a> |
| <span class="sourceLineNo">7117</span> }<a name="line.7117"></a> |
| <span class="sourceLineNo">7118</span><a name="line.7118"></a> |
| <span class="sourceLineNo">7119</span> /**<a name="line.7119"></a> |
| <span class="sourceLineNo">7120</span> * Convenience method creating new HRegions. Used by createTable.<a name="line.7120"></a> |
| <span class="sourceLineNo">7121</span> * @param info Info for region to create.<a name="line.7121"></a> |
| <span class="sourceLineNo">7122</span> * @param rootDir Root directory for HBase instance<a name="line.7122"></a> |
| <span class="sourceLineNo">7123</span> * @param wal shared WAL<a name="line.7123"></a> |
| <span class="sourceLineNo">7124</span> * @param initialize - true to initialize the region<a name="line.7124"></a> |
| <span class="sourceLineNo">7125</span> * @return new HRegion<a name="line.7125"></a> |
| <span class="sourceLineNo">7126</span> */<a name="line.7126"></a> |
| <span class="sourceLineNo">7127</span> public static HRegion createHRegion(final RegionInfo info, final Path rootDir,<a name="line.7127"></a> |
| <span class="sourceLineNo">7128</span> final Configuration conf, final TableDescriptor hTableDescriptor, final WAL wal,<a name="line.7128"></a> |
| <span class="sourceLineNo">7129</span> final boolean initialize) throws IOException {<a name="line.7129"></a> |
| <span class="sourceLineNo">7130</span> return createHRegion(info, rootDir, conf, hTableDescriptor, wal, initialize, null);<a name="line.7130"></a> |
| <span class="sourceLineNo">7131</span> }<a name="line.7131"></a> |
| <span class="sourceLineNo">7132</span><a name="line.7132"></a> |
| <span class="sourceLineNo">7133</span> /**<a name="line.7133"></a> |
| <span class="sourceLineNo">7134</span> * Convenience method creating new HRegions. Used by createTable.<a name="line.7134"></a> |
| <span class="sourceLineNo">7135</span> * @param info Info for region to create.<a name="line.7135"></a> |
| <span class="sourceLineNo">7136</span> * @param rootDir Root directory for HBase instance<a name="line.7136"></a> |
| <span class="sourceLineNo">7137</span> * @param wal shared WAL<a name="line.7137"></a> |
| <span class="sourceLineNo">7138</span> * @param initialize - true to initialize the region<a name="line.7138"></a> |
| <span class="sourceLineNo">7139</span> * @param rsRpcServices An interface we can request flushes against.<a name="line.7139"></a> |
| <span class="sourceLineNo">7140</span> * @return new HRegion<a name="line.7140"></a> |
| <span class="sourceLineNo">7141</span> */<a name="line.7141"></a> |
| <span class="sourceLineNo">7142</span> public static HRegion createHRegion(final RegionInfo info, final Path rootDir,<a name="line.7142"></a> |
| <span class="sourceLineNo">7143</span> final Configuration conf, final TableDescriptor hTableDescriptor, final WAL wal,<a name="line.7143"></a> |
| <span class="sourceLineNo">7144</span> final boolean initialize, RegionServerServices rsRpcServices) throws IOException {<a name="line.7144"></a> |
| <span class="sourceLineNo">7145</span> LOG.info("creating " + info + ", tableDescriptor="<a name="line.7145"></a> |
| <span class="sourceLineNo">7146</span> + (hTableDescriptor == null ? "null" : hTableDescriptor) + ", regionDir=" + rootDir);<a name="line.7146"></a> |
| <span class="sourceLineNo">7147</span> createRegionDir(conf, info, rootDir);<a name="line.7147"></a> |
| <span class="sourceLineNo">7148</span> FileSystem fs = rootDir.getFileSystem(conf);<a name="line.7148"></a> |
| <span class="sourceLineNo">7149</span> Path tableDir = CommonFSUtils.getTableDir(rootDir, info.getTable());<a name="line.7149"></a> |
| <span class="sourceLineNo">7150</span> HRegion region =<a name="line.7150"></a> |
| <span class="sourceLineNo">7151</span> HRegion.newHRegion(tableDir, wal, fs, conf, info, hTableDescriptor, rsRpcServices);<a name="line.7151"></a> |
| <span class="sourceLineNo">7152</span> if (initialize) {<a name="line.7152"></a> |
| <span class="sourceLineNo">7153</span> region.initialize(null);<a name="line.7153"></a> |
| <span class="sourceLineNo">7154</span> }<a name="line.7154"></a> |
| <span class="sourceLineNo">7155</span> return region;<a name="line.7155"></a> |
| <span class="sourceLineNo">7156</span> }<a name="line.7156"></a> |
| <span class="sourceLineNo">7157</span><a name="line.7157"></a> |
| <span class="sourceLineNo">7158</span> /**<a name="line.7158"></a> |
| <span class="sourceLineNo">7159</span> * Create a region under the given table directory.<a name="line.7159"></a> |
| <span class="sourceLineNo">7160</span> */<a name="line.7160"></a> |
| <span class="sourceLineNo">7161</span> public static HRegion createHRegion(Configuration conf, RegionInfo regionInfo, FileSystem fs,<a name="line.7161"></a> |
| <span class="sourceLineNo">7162</span> Path tableDir, TableDescriptor tableDesc) throws IOException {<a name="line.7162"></a> |
| <span class="sourceLineNo">7163</span> LOG.info("Creating {}, tableDescriptor={}, under table dir {}", regionInfo, tableDesc,<a name="line.7163"></a> |
| <span class="sourceLineNo">7164</span> tableDir);<a name="line.7164"></a> |
| <span class="sourceLineNo">7165</span> HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, regionInfo);<a name="line.7165"></a> |
| <span class="sourceLineNo">7166</span> HRegion region = HRegion.newHRegion(tableDir, null, fs, conf, regionInfo, tableDesc, null);<a name="line.7166"></a> |
| <span class="sourceLineNo">7167</span> return region;<a name="line.7167"></a> |
| <span class="sourceLineNo">7168</span> }<a name="line.7168"></a> |
| <span class="sourceLineNo">7169</span><a name="line.7169"></a> |
| <span class="sourceLineNo">7170</span> /**<a name="line.7170"></a> |
| <span class="sourceLineNo">7171</span> * Create the region directory in the filesystem.<a name="line.7171"></a> |
| <span class="sourceLineNo">7172</span> */<a name="line.7172"></a> |
| <span class="sourceLineNo">7173</span> public static HRegionFileSystem createRegionDir(Configuration configuration, RegionInfo ri,<a name="line.7173"></a> |
| <span class="sourceLineNo">7174</span> Path rootDir)<a name="line.7174"></a> |
| <span class="sourceLineNo">7175</span> throws IOException {<a name="line.7175"></a> |
| <span class="sourceLineNo">7176</span> FileSystem fs = rootDir.getFileSystem(configuration);<a name="line.7176"></a> |
| <span class="sourceLineNo">7177</span> Path tableDir = CommonFSUtils.getTableDir(rootDir, ri.getTable());<a name="line.7177"></a> |
| <span class="sourceLineNo">7178</span> // If directory already exists, will log warning and keep going. Will try to create<a name="line.7178"></a> |
| <span class="sourceLineNo">7179</span> // .regioninfo. If one exists, will overwrite.<a name="line.7179"></a> |
| <span class="sourceLineNo">7180</span> return HRegionFileSystem.createRegionOnFileSystem(configuration, fs, tableDir, ri);<a name="line.7180"></a> |
| <span class="sourceLineNo">7181</span> }<a name="line.7181"></a> |
| <span class="sourceLineNo">7182</span><a name="line.7182"></a> |
| <span class="sourceLineNo">7183</span> public static HRegion createHRegion(final RegionInfo info, final Path rootDir,<a name="line.7183"></a> |
| <span class="sourceLineNo">7184</span> final Configuration conf,<a name="line.7184"></a> |
| <span class="sourceLineNo">7185</span> final TableDescriptor hTableDescriptor,<a name="line.7185"></a> |
| <span class="sourceLineNo">7186</span> final WAL wal)<a name="line.7186"></a> |
| <span class="sourceLineNo">7187</span> throws IOException {<a name="line.7187"></a> |
| <span class="sourceLineNo">7188</span> return createHRegion(info, rootDir, conf, hTableDescriptor, wal, true);<a name="line.7188"></a> |
| <span class="sourceLineNo">7189</span> }<a name="line.7189"></a> |
| <span class="sourceLineNo">7190</span><a name="line.7190"></a> |
| <span class="sourceLineNo">7191</span><a name="line.7191"></a> |
| <span class="sourceLineNo">7192</span> /**<a name="line.7192"></a> |
| <span class="sourceLineNo">7193</span> * Open a Region.<a name="line.7193"></a> |
| <span class="sourceLineNo">7194</span> * @param info Info for region to be opened.<a name="line.7194"></a> |
| <span class="sourceLineNo">7195</span> * @param wal WAL for region to use. This method will call<a name="line.7195"></a> |
| <span class="sourceLineNo">7196</span> * WAL#setSequenceNumber(long) passing the result of the call to<a name="line.7196"></a> |
| <span class="sourceLineNo">7197</span> * HRegion#getMinSequenceId() to ensure the wal id is properly kept<a name="line.7197"></a> |
| <span class="sourceLineNo">7198</span> * up. HRegionStore does this every time it opens a new region.<a name="line.7198"></a> |
| <span class="sourceLineNo">7199</span> * @return new HRegion<a name="line.7199"></a> |
| <span class="sourceLineNo">7200</span> *<a name="line.7200"></a> |
| <span class="sourceLineNo">7201</span> * @throws IOException<a name="line.7201"></a> |
| <span class="sourceLineNo">7202</span> */<a name="line.7202"></a> |
| <span class="sourceLineNo">7203</span> public static HRegion openHRegion(final RegionInfo info,<a name="line.7203"></a> |
| <span class="sourceLineNo">7204</span> final TableDescriptor htd, final WAL wal,<a name="line.7204"></a> |
| <span class="sourceLineNo">7205</span> final Configuration conf)<a name="line.7205"></a> |
| <span class="sourceLineNo">7206</span> throws IOException {<a name="line.7206"></a> |
| <span class="sourceLineNo">7207</span> return openHRegion(info, htd, wal, conf, null, null);<a name="line.7207"></a> |
| <span class="sourceLineNo">7208</span> }<a name="line.7208"></a> |
| <span class="sourceLineNo">7209</span><a name="line.7209"></a> |
| <span class="sourceLineNo">7210</span> /**<a name="line.7210"></a> |
| <span class="sourceLineNo">7211</span> * Open a Region.<a name="line.7211"></a> |
| <span class="sourceLineNo">7212</span> * @param info Info for region to be opened<a name="line.7212"></a> |
| <span class="sourceLineNo">7213</span> * @param htd the table descriptor<a name="line.7213"></a> |
| <span class="sourceLineNo">7214</span> * @param wal WAL for region to use. This method will call<a name="line.7214"></a> |
| <span class="sourceLineNo">7215</span> * WAL#setSequenceNumber(long) passing the result of the call to<a name="line.7215"></a> |
| <span class="sourceLineNo">7216</span> * HRegion#getMinSequenceId() to ensure the wal id is properly kept<a name="line.7216"></a> |
| <span class="sourceLineNo">7217</span> * up. HRegionStore does this every time it opens a new region.<a name="line.7217"></a> |
| <span class="sourceLineNo">7218</span> * @param conf The Configuration object to use.<a name="line.7218"></a> |
| <span class="sourceLineNo">7219</span> * @param rsServices An interface we can request flushes against.<a name="line.7219"></a> |
| <span class="sourceLineNo">7220</span> * @param reporter An interface we can report progress against.<a name="line.7220"></a> |
| <span class="sourceLineNo">7221</span> * @return new HRegion<a name="line.7221"></a> |
| <span class="sourceLineNo">7222</span> *<a name="line.7222"></a> |
| <span class="sourceLineNo">7223</span> * @throws IOException<a name="line.7223"></a> |
| <span class="sourceLineNo">7224</span> */<a name="line.7224"></a> |
| <span class="sourceLineNo">7225</span> public static HRegion openHRegion(final RegionInfo info,<a name="line.7225"></a> |
| <span class="sourceLineNo">7226</span> final TableDescriptor htd, final WAL wal, final Configuration conf,<a name="line.7226"></a> |
| <span class="sourceLineNo">7227</span> final RegionServerServices rsServices,<a name="line.7227"></a> |
| <span class="sourceLineNo">7228</span> final CancelableProgressable reporter)<a name="line.7228"></a> |
| <span class="sourceLineNo">7229</span> throws IOException {<a name="line.7229"></a> |
| <span class="sourceLineNo">7230</span> return openHRegion(CommonFSUtils.getRootDir(conf), info, htd, wal, conf, rsServices, reporter);<a name="line.7230"></a> |
| <span class="sourceLineNo">7231</span> }<a name="line.7231"></a> |
| <span class="sourceLineNo">7232</span><a name="line.7232"></a> |
| <span class="sourceLineNo">7233</span> /**<a name="line.7233"></a> |
| <span class="sourceLineNo">7234</span> * Open a Region.<a name="line.7234"></a> |
| <span class="sourceLineNo">7235</span> * @param rootDir Root directory for HBase instance<a name="line.7235"></a> |
| <span class="sourceLineNo">7236</span> * @param info Info for region to be opened.<a name="line.7236"></a> |
| <span class="sourceLineNo">7237</span> * @param htd the table descriptor<a name="line.7237"></a> |
| <span class="sourceLineNo">7238</span> * @param wal WAL for region to use. This method will call<a name="line.7238"></a> |
| <span class="sourceLineNo">7239</span> * WAL#setSequenceNumber(long) passing the result of the call to<a name="line.7239"></a> |
| <span class="sourceLineNo">7240</span> * HRegion#getMinSequenceId() to ensure the wal id is properly kept<a name="line.7240"></a> |
| <span class="sourceLineNo">7241</span> * up. HRegionStore does this every time it opens a new region.<a name="line.7241"></a> |
| <span class="sourceLineNo">7242</span> * @param conf The Configuration object to use.<a name="line.7242"></a> |
| <span class="sourceLineNo">7243</span> * @return new HRegion<a name="line.7243"></a> |
| <span class="sourceLineNo">7244</span> * @throws IOException<a name="line.7244"></a> |
| <span class="sourceLineNo">7245</span> */<a name="line.7245"></a> |
| <span class="sourceLineNo">7246</span> public static HRegion openHRegion(Path rootDir, final RegionInfo info,<a name="line.7246"></a> |
| <span class="sourceLineNo">7247</span> final TableDescriptor htd, final WAL wal, final Configuration conf)<a name="line.7247"></a> |
| <span class="sourceLineNo">7248</span> throws IOException {<a name="line.7248"></a> |
| <span class="sourceLineNo">7249</span> return openHRegion(rootDir, info, htd, wal, conf, null, null);<a name="line.7249"></a> |
| <span class="sourceLineNo">7250</span> }<a name="line.7250"></a> |
| <span class="sourceLineNo">7251</span><a name="line.7251"></a> |
| <span class="sourceLineNo">7252</span> /**<a name="line.7252"></a> |
| <span class="sourceLineNo">7253</span> * Open a Region.<a name="line.7253"></a> |
| <span class="sourceLineNo">7254</span> * @param rootDir Root directory for HBase instance<a name="line.7254"></a> |
| <span class="sourceLineNo">7255</span> * @param info Info for region to be opened.<a name="line.7255"></a> |
| <span class="sourceLineNo">7256</span> * @param htd the table descriptor<a name="line.7256"></a> |
| <span class="sourceLineNo">7257</span> * @param wal WAL for region to use. This method will call<a name="line.7257"></a> |
| <span class="sourceLineNo">7258</span> * WAL#setSequenceNumber(long) passing the result of the call to<a name="line.7258"></a> |
| <span class="sourceLineNo">7259</span> * HRegion#getMinSequenceId() to ensure the wal id is properly kept<a name="line.7259"></a> |
| <span class="sourceLineNo">7260</span> * up. HRegionStore does this every time it opens a new region.<a name="line.7260"></a> |
| <span class="sourceLineNo">7261</span> * @param conf The Configuration object to use.<a name="line.7261"></a> |
| <span class="sourceLineNo">7262</span> * @param rsServices An interface we can request flushes against.<a name="line.7262"></a> |
| <span class="sourceLineNo">7263</span> * @param reporter An interface we can report progress against.<a name="line.7263"></a> |
| <span class="sourceLineNo">7264</span> * @return new HRegion<a name="line.7264"></a> |
| <span class="sourceLineNo">7265</span> * @throws IOException<a name="line.7265"></a> |
| <span class="sourceLineNo">7266</span> */<a name="line.7266"></a> |
| <span class="sourceLineNo">7267</span> public static HRegion openHRegion(final Path rootDir, final RegionInfo info,<a name="line.7267"></a> |
| <span class="sourceLineNo">7268</span> final TableDescriptor htd, final WAL wal, final Configuration conf,<a name="line.7268"></a> |
| <span class="sourceLineNo">7269</span> final RegionServerServices rsServices,<a name="line.7269"></a> |
| <span class="sourceLineNo">7270</span> final CancelableProgressable reporter)<a name="line.7270"></a> |
| <span class="sourceLineNo">7271</span> throws IOException {<a name="line.7271"></a> |
| <span class="sourceLineNo">7272</span> FileSystem fs = null;<a name="line.7272"></a> |
| <span class="sourceLineNo">7273</span> if (rsServices != null) {<a name="line.7273"></a> |
| <span class="sourceLineNo">7274</span> fs = rsServices.getFileSystem();<a name="line.7274"></a> |
| <span class="sourceLineNo">7275</span> }<a name="line.7275"></a> |
| <span class="sourceLineNo">7276</span> if (fs == null) {<a name="line.7276"></a> |
| <span class="sourceLineNo">7277</span> fs = rootDir.getFileSystem(conf);<a name="line.7277"></a> |
| <span class="sourceLineNo">7278</span> }<a name="line.7278"></a> |
| <span class="sourceLineNo">7279</span> return openHRegion(conf, fs, rootDir, info, htd, wal, rsServices, reporter);<a name="line.7279"></a> |
| <span class="sourceLineNo">7280</span> }<a name="line.7280"></a> |
| <span class="sourceLineNo">7281</span><a name="line.7281"></a> |
| <span class="sourceLineNo">7282</span> /**<a name="line.7282"></a> |
| <span class="sourceLineNo">7283</span> * Open a Region.<a name="line.7283"></a> |
| <span class="sourceLineNo">7284</span> * @param conf The Configuration object to use.<a name="line.7284"></a> |
| <span class="sourceLineNo">7285</span> * @param fs Filesystem to use<a name="line.7285"></a> |
| <span class="sourceLineNo">7286</span> * @param rootDir Root directory for HBase instance<a name="line.7286"></a> |
| <span class="sourceLineNo">7287</span> * @param info Info for region to be opened.<a name="line.7287"></a> |
| <span class="sourceLineNo">7288</span> * @param htd the table descriptor<a name="line.7288"></a> |
| <span class="sourceLineNo">7289</span> * @param wal WAL for region to use. This method will call<a name="line.7289"></a> |
| <span class="sourceLineNo">7290</span> * WAL#setSequenceNumber(long) passing the result of the call to<a name="line.7290"></a> |
| <span class="sourceLineNo">7291</span> * HRegion#getMinSequenceId() to ensure the wal id is properly kept<a name="line.7291"></a> |
| <span class="sourceLineNo">7292</span> * up. HRegionStore does this every time it opens a new region.<a name="line.7292"></a> |
| <span class="sourceLineNo">7293</span> * @return new HRegion<a name="line.7293"></a> |
| <span class="sourceLineNo">7294</span> */<a name="line.7294"></a> |
| <span class="sourceLineNo">7295</span> public static HRegion openHRegion(final Configuration conf, final FileSystem fs,<a name="line.7295"></a> |
| <span class="sourceLineNo">7296</span> final Path rootDir, final RegionInfo info, final TableDescriptor htd, final WAL wal)<a name="line.7296"></a> |
| <span class="sourceLineNo">7297</span> throws IOException {<a name="line.7297"></a> |
| <span class="sourceLineNo">7298</span> return openHRegion(conf, fs, rootDir, info, htd, wal, null, null);<a name="line.7298"></a> |
| <span class="sourceLineNo">7299</span> }<a name="line.7299"></a> |
| <span class="sourceLineNo">7300</span><a name="line.7300"></a> |
| <span class="sourceLineNo">7301</span> /**<a name="line.7301"></a> |
| <span class="sourceLineNo">7302</span> * Open a Region.<a name="line.7302"></a> |
| <span class="sourceLineNo">7303</span> * @param conf The Configuration object to use.<a name="line.7303"></a> |
| <span class="sourceLineNo">7304</span> * @param fs Filesystem to use<a name="line.7304"></a> |
| <span class="sourceLineNo">7305</span> * @param rootDir Root directory for HBase instance<a name="line.7305"></a> |
| <span class="sourceLineNo">7306</span> * @param info Info for region to be opened.<a name="line.7306"></a> |
| <span class="sourceLineNo">7307</span> * @param htd the table descriptor<a name="line.7307"></a> |
| <span class="sourceLineNo">7308</span> * @param wal WAL for region to use. This method will call<a name="line.7308"></a> |
| <span class="sourceLineNo">7309</span> * WAL#setSequenceNumber(long) passing the result of the call to<a name="line.7309"></a> |
| <span class="sourceLineNo">7310</span> * HRegion#getMinSequenceId() to ensure the wal id is properly kept<a name="line.7310"></a> |
| <span class="sourceLineNo">7311</span> * up. HRegionStore does this every time it opens a new region.<a name="line.7311"></a> |
| <span class="sourceLineNo">7312</span> * @param rsServices An interface we can request flushes against.<a name="line.7312"></a> |
| <span class="sourceLineNo">7313</span> * @param reporter An interface we can report progress against.<a name="line.7313"></a> |
| <span class="sourceLineNo">7314</span> * @return new HRegion<a name="line.7314"></a> |
| <span class="sourceLineNo">7315</span> */<a name="line.7315"></a> |
| <span class="sourceLineNo">7316</span> public static HRegion openHRegion(final Configuration conf, final FileSystem fs,<a name="line.7316"></a> |
| <span class="sourceLineNo">7317</span> final Path rootDir, final RegionInfo info, final TableDescriptor htd, final WAL wal,<a name="line.7317"></a> |
| <span class="sourceLineNo">7318</span> final RegionServerServices rsServices, final CancelableProgressable reporter)<a name="line.7318"></a> |
| <span class="sourceLineNo">7319</span> throws IOException {<a name="line.7319"></a> |
| <span class="sourceLineNo">7320</span> Path tableDir = CommonFSUtils.getTableDir(rootDir, info.getTable());<a name="line.7320"></a> |
| <span class="sourceLineNo">7321</span> return openHRegionFromTableDir(conf, fs, tableDir, info, htd, wal, rsServices, reporter);<a name="line.7321"></a> |
| <span class="sourceLineNo">7322</span> }<a name="line.7322"></a> |
| <span class="sourceLineNo">7323</span><a name="line.7323"></a> |
| <span class="sourceLineNo">7324</span> /**<a name="line.7324"></a> |
| <span class="sourceLineNo">7325</span> * Open a Region.<a name="line.7325"></a> |
| <span class="sourceLineNo">7326</span> * @param conf The Configuration object to use.<a name="line.7326"></a> |
| <span class="sourceLineNo">7327</span> * @param fs Filesystem to use<a name="line.7327"></a> |
| <span class="sourceLineNo">7328</span> * @param info Info for region to be opened.<a name="line.7328"></a> |
| <span class="sourceLineNo">7329</span> * @param htd the table descriptor<a name="line.7329"></a> |
| <span class="sourceLineNo">7330</span> * @param wal WAL for region to use. This method will call<a name="line.7330"></a> |
| <span class="sourceLineNo">7331</span> * WAL#setSequenceNumber(long) passing the result of the call to<a name="line.7331"></a> |
| <span class="sourceLineNo">7332</span> * HRegion#getMinSequenceId() to ensure the wal id is properly kept<a name="line.7332"></a> |
| <span class="sourceLineNo">7333</span> * up. HRegionStore does this every time it opens a new region.<a name="line.7333"></a> |
| <span class="sourceLineNo">7334</span> * @param rsServices An interface we can request flushes against.<a name="line.7334"></a> |
| <span class="sourceLineNo">7335</span> * @param reporter An interface we can report progress against.<a name="line.7335"></a> |
| <span class="sourceLineNo">7336</span> * @return new HRegion<a name="line.7336"></a> |
| <span class="sourceLineNo">7337</span> * @throws NullPointerException if {@code info} is {@code null}<a name="line.7337"></a> |
| <span class="sourceLineNo">7338</span> */<a name="line.7338"></a> |
| <span class="sourceLineNo">7339</span> public static HRegion openHRegionFromTableDir(final Configuration conf, final FileSystem fs,<a name="line.7339"></a> |
| <span class="sourceLineNo">7340</span> final Path tableDir, final RegionInfo info, final TableDescriptor htd, final WAL wal,<a name="line.7340"></a> |
| <span class="sourceLineNo">7341</span> final RegionServerServices rsServices, final CancelableProgressable reporter)<a name="line.7341"></a> |
| <span class="sourceLineNo">7342</span> throws IOException {<a name="line.7342"></a> |
| <span class="sourceLineNo">7343</span> Objects.requireNonNull(info, "RegionInfo cannot be null");<a name="line.7343"></a> |
| <span class="sourceLineNo">7344</span> LOG.debug("Opening region: {}", info);<a name="line.7344"></a> |
| <span class="sourceLineNo">7345</span> HRegion r = HRegion.newHRegion(tableDir, wal, fs, conf, info, htd, rsServices);<a name="line.7345"></a> |
| <span class="sourceLineNo">7346</span> return r.openHRegion(reporter);<a name="line.7346"></a> |
| <span class="sourceLineNo">7347</span> }<a name="line.7347"></a> |
| <span class="sourceLineNo">7348</span><a name="line.7348"></a> |
| <span class="sourceLineNo">7349</span> public NavigableMap<byte[], Integer> getReplicationScope() {<a name="line.7349"></a> |
| <span class="sourceLineNo">7350</span> return this.replicationScope;<a name="line.7350"></a> |
| <span class="sourceLineNo">7351</span> }<a name="line.7351"></a> |
| <span class="sourceLineNo">7352</span><a name="line.7352"></a> |
| <span class="sourceLineNo">7353</span> /**<a name="line.7353"></a> |
| <span class="sourceLineNo">7354</span> * Useful when reopening a closed region (normally for unit tests)<a name="line.7354"></a> |
| <span class="sourceLineNo">7355</span> * @param other original object<a name="line.7355"></a> |
| <span class="sourceLineNo">7356</span> * @param reporter An interface we can report progress against.<a name="line.7356"></a> |
| <span class="sourceLineNo">7357</span> * @return new HRegion<a name="line.7357"></a> |
| <span class="sourceLineNo">7358</span> */<a name="line.7358"></a> |
| <span class="sourceLineNo">7359</span> public static HRegion openHRegion(final HRegion other, final CancelableProgressable reporter)<a name="line.7359"></a> |
| <span class="sourceLineNo">7360</span> throws IOException {<a name="line.7360"></a> |
| <span class="sourceLineNo">7361</span> HRegionFileSystem regionFs = other.getRegionFileSystem();<a name="line.7361"></a> |
| <span class="sourceLineNo">7362</span> HRegion r = newHRegion(regionFs.getTableDir(), other.getWAL(), regionFs.getFileSystem(),<a name="line.7362"></a> |
| <span class="sourceLineNo">7363</span> other.baseConf, other.getRegionInfo(), other.getTableDescriptor(), null);<a name="line.7363"></a> |
| <span class="sourceLineNo">7364</span> return r.openHRegion(reporter);<a name="line.7364"></a> |
| <span class="sourceLineNo">7365</span> }<a name="line.7365"></a> |
| <span class="sourceLineNo">7366</span><a name="line.7366"></a> |
| <span class="sourceLineNo">7367</span> public static Region openHRegion(final Region other, final CancelableProgressable reporter)<a name="line.7367"></a> |
| <span class="sourceLineNo">7368</span> throws IOException {<a name="line.7368"></a> |
| <span class="sourceLineNo">7369</span> return openHRegion((HRegion)other, reporter);<a name="line.7369"></a> |
| <span class="sourceLineNo">7370</span> }<a name="line.7370"></a> |
| <span class="sourceLineNo">7371</span><a name="line.7371"></a> |
| <span class="sourceLineNo">7372</span> /**<a name="line.7372"></a> |
| <span class="sourceLineNo">7373</span> * Open HRegion.<a name="line.7373"></a> |
| <span class="sourceLineNo">7374</span> * <p/><a name="line.7374"></a> |
| <span class="sourceLineNo">7375</span> * Calls initialize and sets sequenceId.<a name="line.7375"></a> |
| <span class="sourceLineNo">7376</span> * @return Returns <code>this</code><a name="line.7376"></a> |
| <span class="sourceLineNo">7377</span> */<a name="line.7377"></a> |
| <span class="sourceLineNo">7378</span> private HRegion openHRegion(final CancelableProgressable reporter) throws IOException {<a name="line.7378"></a> |
| <span class="sourceLineNo">7379</span> try {<a name="line.7379"></a> |
| <span class="sourceLineNo">7380</span> // Refuse to open the region if we are missing local compression support<a name="line.7380"></a> |
| <span class="sourceLineNo">7381</span> TableDescriptorChecker.checkCompression(htableDescriptor);<a name="line.7381"></a> |
| <span class="sourceLineNo">7382</span> // Refuse to open the region if encryption configuration is incorrect or<a name="line.7382"></a> |
| <span class="sourceLineNo">7383</span> // codec support is missing<a name="line.7383"></a> |
| <span class="sourceLineNo">7384</span> LOG.debug("checking encryption for " + this.getRegionInfo().getEncodedName());<a name="line.7384"></a> |
| <span class="sourceLineNo">7385</span> TableDescriptorChecker.checkEncryption(conf, htableDescriptor);<a name="line.7385"></a> |
| <span class="sourceLineNo">7386</span> // Refuse to open the region if a required class cannot be loaded<a name="line.7386"></a> |
| <span class="sourceLineNo">7387</span> LOG.debug("checking classloading for " + this.getRegionInfo().getEncodedName());<a name="line.7387"></a> |
| <span class="sourceLineNo">7388</span> TableDescriptorChecker.checkClassLoading(conf, htableDescriptor);<a name="line.7388"></a> |
| <span class="sourceLineNo">7389</span> this.openSeqNum = initialize(reporter);<a name="line.7389"></a> |
| <span class="sourceLineNo">7390</span> this.mvcc.advanceTo(openSeqNum);<a name="line.7390"></a> |
| <span class="sourceLineNo">7391</span> // The openSeqNum must be increased every time when a region is assigned, as we rely on it to<a name="line.7391"></a> |
| <span class="sourceLineNo">7392</span> // determine whether a region has been successfully reopened. So here we always write open<a name="line.7392"></a> |
| <span class="sourceLineNo">7393</span> // marker, even if the table is read only.<a name="line.7393"></a> |
| <span class="sourceLineNo">7394</span> if (wal != null && getRegionServerServices() != null &&<a name="line.7394"></a> |
| <span class="sourceLineNo">7395</span> RegionReplicaUtil.isDefaultReplica(getRegionInfo())) {<a name="line.7395"></a> |
| <span class="sourceLineNo">7396</span> writeRegionOpenMarker(wal, openSeqNum);<a name="line.7396"></a> |
| <span class="sourceLineNo">7397</span> }<a name="line.7397"></a> |
| <span class="sourceLineNo">7398</span> } catch (Throwable t) {<a name="line.7398"></a> |
| <span class="sourceLineNo">7399</span> // By coprocessor path wrong region will open failed,<a name="line.7399"></a> |
| <span class="sourceLineNo">7400</span> // MetricsRegionWrapperImpl is already init and not close,<a name="line.7400"></a> |
| <span class="sourceLineNo">7401</span> // add region close when open failed<a name="line.7401"></a> |
| <span class="sourceLineNo">7402</span> try {<a name="line.7402"></a> |
| <span class="sourceLineNo">7403</span> // It is not required to write sequence id file when region open is failed.<a name="line.7403"></a> |
| <span class="sourceLineNo">7404</span> // Passing true to skip the sequence id file write.<a name="line.7404"></a> |
| <span class="sourceLineNo">7405</span> this.close(true);<a name="line.7405"></a> |
| <span class="sourceLineNo">7406</span> } catch (Throwable e) {<a name="line.7406"></a> |
| <span class="sourceLineNo">7407</span> LOG.warn("Open region: {} failed. Try close region but got exception ", this.getRegionInfo(),<a name="line.7407"></a> |
| <span class="sourceLineNo">7408</span> e);<a name="line.7408"></a> |
| <span class="sourceLineNo">7409</span> }<a name="line.7409"></a> |
| <span class="sourceLineNo">7410</span> throw t;<a name="line.7410"></a> |
| <span class="sourceLineNo">7411</span> }<a name="line.7411"></a> |
| <span class="sourceLineNo">7412</span> return this;<a name="line.7412"></a> |
| <span class="sourceLineNo">7413</span> }<a name="line.7413"></a> |
| <span class="sourceLineNo">7414</span><a name="line.7414"></a> |
| <span class="sourceLineNo">7415</span> /**<a name="line.7415"></a> |
| <span class="sourceLineNo">7416</span> * Open a Region on a read-only file-system (like hdfs snapshots)<a name="line.7416"></a> |
| <span class="sourceLineNo">7417</span> * @param conf The Configuration object to use.<a name="line.7417"></a> |
| <span class="sourceLineNo">7418</span> * @param fs Filesystem to use<a name="line.7418"></a> |
| <span class="sourceLineNo">7419</span> * @param info Info for region to be opened.<a name="line.7419"></a> |
| <span class="sourceLineNo">7420</span> * @param htd the table descriptor<a name="line.7420"></a> |
| <span class="sourceLineNo">7421</span> * @return new HRegion<a name="line.7421"></a> |
| <span class="sourceLineNo">7422</span> * @throws NullPointerException if {@code info} is {@code null}<a name="line.7422"></a> |
| <span class="sourceLineNo">7423</span> */<a name="line.7423"></a> |
| <span class="sourceLineNo">7424</span> public static HRegion openReadOnlyFileSystemHRegion(final Configuration conf, final FileSystem fs,<a name="line.7424"></a> |
| <span class="sourceLineNo">7425</span> final Path tableDir, RegionInfo info, final TableDescriptor htd) throws IOException {<a name="line.7425"></a> |
| <span class="sourceLineNo">7426</span> Objects.requireNonNull(info, "RegionInfo cannot be null");<a name="line.7426"></a> |
| <span class="sourceLineNo">7427</span> if (LOG.isDebugEnabled()) {<a name="line.7427"></a> |
| <span class="sourceLineNo">7428</span> LOG.debug("Opening region (readOnly filesystem): " + info);<a name="line.7428"></a> |
| <span class="sourceLineNo">7429</span> }<a name="line.7429"></a> |
| <span class="sourceLineNo">7430</span> if (info.getReplicaId() <= 0) {<a name="line.7430"></a> |
| <span class="sourceLineNo">7431</span> info = RegionReplicaUtil.getRegionInfoForReplica(info, 1);<a name="line.7431"></a> |
| <span class="sourceLineNo">7432</span> }<a name="line.7432"></a> |
| <span class="sourceLineNo">7433</span> HRegion r = HRegion.newHRegion(tableDir, null, fs, conf, info, htd, null);<a name="line.7433"></a> |
| <span class="sourceLineNo">7434</span> r.writestate.setReadOnly(true);<a name="line.7434"></a> |
| <span class="sourceLineNo">7435</span> return r.openHRegion(null);<a name="line.7435"></a> |
| <span class="sourceLineNo">7436</span> }<a name="line.7436"></a> |
| <span class="sourceLineNo">7437</span><a name="line.7437"></a> |
| <span class="sourceLineNo">7438</span> public static void warmupHRegion(final RegionInfo info,<a name="line.7438"></a> |
| <span class="sourceLineNo">7439</span> final TableDescriptor htd, final WAL wal, final Configuration conf,<a name="line.7439"></a> |
| <span class="sourceLineNo">7440</span> final RegionServerServices rsServices,<a name="line.7440"></a> |
| <span class="sourceLineNo">7441</span> final CancelableProgressable reporter)<a name="line.7441"></a> |
| <span class="sourceLineNo">7442</span> throws IOException {<a name="line.7442"></a> |
| <span class="sourceLineNo">7443</span><a name="line.7443"></a> |
| <span class="sourceLineNo">7444</span> Objects.requireNonNull(info, "RegionInfo cannot be null");<a name="line.7444"></a> |
| <span class="sourceLineNo">7445</span> LOG.debug("Warmup {}", info);<a name="line.7445"></a> |
| <span class="sourceLineNo">7446</span> Path rootDir = CommonFSUtils.getRootDir(conf);<a name="line.7446"></a> |
| <span class="sourceLineNo">7447</span> Path tableDir = CommonFSUtils.getTableDir(rootDir, info.getTable());<a name="line.7447"></a> |
| <span class="sourceLineNo">7448</span> FileSystem fs = null;<a name="line.7448"></a> |
| <span class="sourceLineNo">7449</span> if (rsServices != null) {<a name="line.7449"></a> |
| <span class="sourceLineNo">7450</span> fs = rsServices.getFileSystem();<a name="line.7450"></a> |
| <span class="sourceLineNo">7451</span> }<a name="line.7451"></a> |
| <span class="sourceLineNo">7452</span> if (fs == null) {<a name="line.7452"></a> |
| <span class="sourceLineNo">7453</span> fs = rootDir.getFileSystem(conf);<a name="line.7453"></a> |
| <span class="sourceLineNo">7454</span> }<a name="line.7454"></a> |
| <span class="sourceLineNo">7455</span> HRegion r = HRegion.newHRegion(tableDir, wal, fs, conf, info, htd, null);<a name="line.7455"></a> |
| <span class="sourceLineNo">7456</span> r.initializeWarmup(reporter);<a name="line.7456"></a> |
| <span class="sourceLineNo">7457</span> }<a name="line.7457"></a> |
| <span class="sourceLineNo">7458</span><a name="line.7458"></a> |
| <span class="sourceLineNo">7459</span> /**<a name="line.7459"></a> |
| <span class="sourceLineNo">7460</span> * Computes the Path of the HRegion<a name="line.7460"></a> |
| <span class="sourceLineNo">7461</span> *<a name="line.7461"></a> |
| <span class="sourceLineNo">7462</span> * @param tabledir qualified path for table<a name="line.7462"></a> |
| <span class="sourceLineNo">7463</span> * @param name ENCODED region name<a name="line.7463"></a> |
| <span class="sourceLineNo">7464</span> * @return Path of HRegion directory<a name="line.7464"></a> |
| <span class="sourceLineNo">7465</span> * @deprecated For tests only; to be removed.<a name="line.7465"></a> |
| <span class="sourceLineNo">7466</span> */<a name="line.7466"></a> |
| <span class="sourceLineNo">7467</span> @Deprecated<a name="line.7467"></a> |
| <span class="sourceLineNo">7468</span> public static Path getRegionDir(final Path tabledir, final String name) {<a name="line.7468"></a> |
| <span class="sourceLineNo">7469</span> return new Path(tabledir, name);<a name="line.7469"></a> |
| <span class="sourceLineNo">7470</span> }<a name="line.7470"></a> |
| <span class="sourceLineNo">7471</span><a name="line.7471"></a> |
| <span class="sourceLineNo">7472</span> /**<a name="line.7472"></a> |
| <span class="sourceLineNo">7473</span> * Determines if the specified row is within the row range specified by the<a name="line.7473"></a> |
| <span class="sourceLineNo">7474</span> * specified RegionInfo<a name="line.7474"></a> |
| <span class="sourceLineNo">7475</span> *<a name="line.7475"></a> |
| <span class="sourceLineNo">7476</span> * @param info RegionInfo that specifies the row range<a name="line.7476"></a> |
| <span class="sourceLineNo">7477</span> * @param row row to be checked<a name="line.7477"></a> |
| <span class="sourceLineNo">7478</span> * @return true if the row is within the range specified by the RegionInfo<a name="line.7478"></a> |
| <span class="sourceLineNo">7479</span> */<a name="line.7479"></a> |
| <span class="sourceLineNo">7480</span> public static boolean rowIsInRange(RegionInfo info, final byte [] row) {<a name="line.7480"></a> |
| <span class="sourceLineNo">7481</span> return ((info.getStartKey().length == 0) ||<a name="line.7481"></a> |
| <span class="sourceLineNo">7482</span> (Bytes.compareTo(info.getStartKey(), row) <= 0)) &&<a name="line.7482"></a> |
| <span class="sourceLineNo">7483</span> ((info.getEndKey().length == 0) ||<a name="line.7483"></a> |
| <span class="sourceLineNo">7484</span> (Bytes.compareTo(info.getEndKey(), row) > 0));<a name="line.7484"></a> |
| <span class="sourceLineNo">7485</span> }<a name="line.7485"></a> |
| <span class="sourceLineNo">7486</span><a name="line.7486"></a> |
| <span class="sourceLineNo">7487</span> public static boolean rowIsInRange(RegionInfo info, final byte [] row, final int offset,<a name="line.7487"></a> |
| <span class="sourceLineNo">7488</span> final short length) {<a name="line.7488"></a> |
| <span class="sourceLineNo">7489</span> return ((info.getStartKey().length == 0) ||<a name="line.7489"></a> |
| <span class="sourceLineNo">7490</span> (Bytes.compareTo(info.getStartKey(), 0, info.getStartKey().length,<a name="line.7490"></a> |
| <span class="sourceLineNo">7491</span> row, offset, length) <= 0)) &&<a name="line.7491"></a> |
| <span class="sourceLineNo">7492</span> ((info.getEndKey().length == 0) ||<a name="line.7492"></a> |
| <span class="sourceLineNo">7493</span> (Bytes.compareTo(info.getEndKey(), 0, info.getEndKey().length, row, offset, length) > 0));<a name="line.7493"></a> |
| <span class="sourceLineNo">7494</span> }<a name="line.7494"></a> |
| <span class="sourceLineNo">7495</span><a name="line.7495"></a> |
| <span class="sourceLineNo">7496</span> @Override<a name="line.7496"></a> |
| <span class="sourceLineNo">7497</span> public Result get(final Get get) throws IOException {<a name="line.7497"></a> |
| <span class="sourceLineNo">7498</span> prepareGet(get);<a name="line.7498"></a> |
| <span class="sourceLineNo">7499</span> List<Cell> results = get(get, true);<a name="line.7499"></a> |
| <span class="sourceLineNo">7500</span> boolean stale = this.getRegionInfo().getReplicaId() != 0;<a name="line.7500"></a> |
| <span class="sourceLineNo">7501</span> return Result.create(results, get.isCheckExistenceOnly() ? !results.isEmpty() : null, stale);<a name="line.7501"></a> |
| <span class="sourceLineNo">7502</span> }<a name="line.7502"></a> |
| <span class="sourceLineNo">7503</span><a name="line.7503"></a> |
| <span class="sourceLineNo">7504</span> void prepareGet(final Get get) throws IOException {<a name="line.7504"></a> |
| <span class="sourceLineNo">7505</span> checkRow(get.getRow(), "Get");<a name="line.7505"></a> |
| <span class="sourceLineNo">7506</span> // Verify families are all valid<a name="line.7506"></a> |
| <span class="sourceLineNo">7507</span> if (get.hasFamilies()) {<a name="line.7507"></a> |
| <span class="sourceLineNo">7508</span> for (byte[] family : get.familySet()) {<a name="line.7508"></a> |
| <span class="sourceLineNo">7509</span> checkFamily(family);<a name="line.7509"></a> |
| <span class="sourceLineNo">7510</span> }<a name="line.7510"></a> |
| <span class="sourceLineNo">7511</span> } else { // Adding all families to scanner<a name="line.7511"></a> |
| <span class="sourceLineNo">7512</span> for (byte[] family : this.htableDescriptor.getColumnFamilyNames()) {<a name="line.7512"></a> |
| <span class="sourceLineNo">7513</span> get.addFamily(family);<a name="line.7513"></a> |
| <span class="sourceLineNo">7514</span> }<a name="line.7514"></a> |
| <span class="sourceLineNo">7515</span> }<a name="line.7515"></a> |
| <span class="sourceLineNo">7516</span> }<a name="line.7516"></a> |
| <span class="sourceLineNo">7517</span><a name="line.7517"></a> |
| <span class="sourceLineNo">7518</span> @Override<a name="line.7518"></a> |
| <span class="sourceLineNo">7519</span> public List<Cell> get(Get get, boolean withCoprocessor) throws IOException {<a name="line.7519"></a> |
| <span class="sourceLineNo">7520</span> return get(get, withCoprocessor, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.7520"></a> |
| <span class="sourceLineNo">7521</span> }<a name="line.7521"></a> |
| <span class="sourceLineNo">7522</span><a name="line.7522"></a> |
| <span class="sourceLineNo">7523</span> private List<Cell> get(Get get, boolean withCoprocessor, long nonceGroup, long nonce)<a name="line.7523"></a> |
| <span class="sourceLineNo">7524</span> throws IOException {<a name="line.7524"></a> |
| <span class="sourceLineNo">7525</span> List<Cell> results = new ArrayList<>();<a name="line.7525"></a> |
| <span class="sourceLineNo">7526</span> long before = EnvironmentEdgeManager.currentTime();<a name="line.7526"></a> |
| <span class="sourceLineNo">7527</span><a name="line.7527"></a> |
| <span class="sourceLineNo">7528</span> // pre-get CP hook<a name="line.7528"></a> |
| <span class="sourceLineNo">7529</span> if (withCoprocessor && (coprocessorHost != null)) {<a name="line.7529"></a> |
| <span class="sourceLineNo">7530</span> if (coprocessorHost.preGet(get, results)) {<a name="line.7530"></a> |
| <span class="sourceLineNo">7531</span> metricsUpdateForGet(results, before);<a name="line.7531"></a> |
| <span class="sourceLineNo">7532</span> return results;<a name="line.7532"></a> |
| <span class="sourceLineNo">7533</span> }<a name="line.7533"></a> |
| <span class="sourceLineNo">7534</span> }<a name="line.7534"></a> |
| <span class="sourceLineNo">7535</span> Scan scan = new Scan(get);<a name="line.7535"></a> |
| <span class="sourceLineNo">7536</span> if (scan.getLoadColumnFamiliesOnDemandValue() == null) {<a name="line.7536"></a> |
| <span class="sourceLineNo">7537</span> scan.setLoadColumnFamiliesOnDemand(isLoadingCfsOnDemandDefault());<a name="line.7537"></a> |
| <span class="sourceLineNo">7538</span> }<a name="line.7538"></a> |
| <span class="sourceLineNo">7539</span> RegionScanner scanner = null;<a name="line.7539"></a> |
| <span class="sourceLineNo">7540</span> try {<a name="line.7540"></a> |
| <span class="sourceLineNo">7541</span> scanner = getScanner(scan, null, nonceGroup, nonce);<a name="line.7541"></a> |
| <span class="sourceLineNo">7542</span> scanner.next(results);<a name="line.7542"></a> |
| <span class="sourceLineNo">7543</span> } finally {<a name="line.7543"></a> |
| <span class="sourceLineNo">7544</span> if (scanner != null)<a name="line.7544"></a> |
| <span class="sourceLineNo">7545</span> scanner.close();<a name="line.7545"></a> |
| <span class="sourceLineNo">7546</span> }<a name="line.7546"></a> |
| <span class="sourceLineNo">7547</span><a name="line.7547"></a> |
| <span class="sourceLineNo">7548</span> // post-get CP hook<a name="line.7548"></a> |
| <span class="sourceLineNo">7549</span> if (withCoprocessor && (coprocessorHost != null)) {<a name="line.7549"></a> |
| <span class="sourceLineNo">7550</span> coprocessorHost.postGet(get, results);<a name="line.7550"></a> |
| <span class="sourceLineNo">7551</span> }<a name="line.7551"></a> |
| <span class="sourceLineNo">7552</span><a name="line.7552"></a> |
| <span class="sourceLineNo">7553</span> metricsUpdateForGet(results, before);<a name="line.7553"></a> |
| <span class="sourceLineNo">7554</span><a name="line.7554"></a> |
| <span class="sourceLineNo">7555</span> return results;<a name="line.7555"></a> |
| <span class="sourceLineNo">7556</span> }<a name="line.7556"></a> |
| <span class="sourceLineNo">7557</span><a name="line.7557"></a> |
| <span class="sourceLineNo">7558</span> void metricsUpdateForGet(List<Cell> results, long before) {<a name="line.7558"></a> |
| <span class="sourceLineNo">7559</span> if (this.metricsRegion != null) {<a name="line.7559"></a> |
| <span class="sourceLineNo">7560</span> this.metricsRegion.updateGet(EnvironmentEdgeManager.currentTime() - before);<a name="line.7560"></a> |
| <span class="sourceLineNo">7561</span> }<a name="line.7561"></a> |
| <span class="sourceLineNo">7562</span> }<a name="line.7562"></a> |
| <span class="sourceLineNo">7563</span><a name="line.7563"></a> |
| <span class="sourceLineNo">7564</span> @Override<a name="line.7564"></a> |
| <span class="sourceLineNo">7565</span> public Result mutateRow(RowMutations rm) throws IOException {<a name="line.7565"></a> |
| <span class="sourceLineNo">7566</span> final List<Mutation> m = rm.getMutations();<a name="line.7566"></a> |
| <span class="sourceLineNo">7567</span> OperationStatus[] statuses = batchMutate(m.toArray(new Mutation[0]), true,<a name="line.7567"></a> |
| <span class="sourceLineNo">7568</span> HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.7568"></a> |
| <span class="sourceLineNo">7569</span><a name="line.7569"></a> |
| <span class="sourceLineNo">7570</span> List<Result> results = new ArrayList<>();<a name="line.7570"></a> |
| <span class="sourceLineNo">7571</span> for (OperationStatus status : statuses) {<a name="line.7571"></a> |
| <span class="sourceLineNo">7572</span> if (status.getResult() != null) {<a name="line.7572"></a> |
| <span class="sourceLineNo">7573</span> results.add(status.getResult());<a name="line.7573"></a> |
| <span class="sourceLineNo">7574</span> }<a name="line.7574"></a> |
| <span class="sourceLineNo">7575</span> }<a name="line.7575"></a> |
| <span class="sourceLineNo">7576</span><a name="line.7576"></a> |
| <span class="sourceLineNo">7577</span> if (results.isEmpty()) {<a name="line.7577"></a> |
| <span class="sourceLineNo">7578</span> return null;<a name="line.7578"></a> |
| <span class="sourceLineNo">7579</span> }<a name="line.7579"></a> |
| <span class="sourceLineNo">7580</span><a name="line.7580"></a> |
| <span class="sourceLineNo">7581</span> // Merge the results of the Increment/Append operations<a name="line.7581"></a> |
| <span class="sourceLineNo">7582</span> List<Cell> cells = new ArrayList<>();<a name="line.7582"></a> |
| <span class="sourceLineNo">7583</span> for (Result result : results) {<a name="line.7583"></a> |
| <span class="sourceLineNo">7584</span> if (result.rawCells() != null) {<a name="line.7584"></a> |
| <span class="sourceLineNo">7585</span> cells.addAll(Arrays.asList(result.rawCells()));<a name="line.7585"></a> |
| <span class="sourceLineNo">7586</span> }<a name="line.7586"></a> |
| <span class="sourceLineNo">7587</span> }<a name="line.7587"></a> |
| <span class="sourceLineNo">7588</span> return Result.create(cells);<a name="line.7588"></a> |
| <span class="sourceLineNo">7589</span> }<a name="line.7589"></a> |
| <span class="sourceLineNo">7590</span><a name="line.7590"></a> |
| <span class="sourceLineNo">7591</span> /**<a name="line.7591"></a> |
| <span class="sourceLineNo">7592</span> * Perform atomic (all or none) mutations within the region.<a name="line.7592"></a> |
| <span class="sourceLineNo">7593</span> * @param mutations The list of mutations to perform.<a name="line.7593"></a> |
| <span class="sourceLineNo">7594</span> * <code>mutations</code> can contain operations for multiple rows.<a name="line.7594"></a> |
| <span class="sourceLineNo">7595</span> * Caller has to ensure that all rows are contained in this region.<a name="line.7595"></a> |
| <span class="sourceLineNo">7596</span> * @param rowsToLock Rows to lock<a name="line.7596"></a> |
| <span class="sourceLineNo">7597</span> * @param nonceGroup Optional nonce group of the operation (client Id)<a name="line.7597"></a> |
| <span class="sourceLineNo">7598</span> * @param nonce Optional nonce of the operation (unique random id to ensure "more idempotence")<a name="line.7598"></a> |
| <span class="sourceLineNo">7599</span> * If multiple rows are locked care should be taken that<a name="line.7599"></a> |
| <span class="sourceLineNo">7600</span> * <code>rowsToLock</code> is sorted in order to avoid deadlocks.<a name="line.7600"></a> |
| <span class="sourceLineNo">7601</span> * @throws IOException<a name="line.7601"></a> |
| <span class="sourceLineNo">7602</span> */<a name="line.7602"></a> |
| <span class="sourceLineNo">7603</span> @Override<a name="line.7603"></a> |
| <span class="sourceLineNo">7604</span> public void mutateRowsWithLocks(Collection<Mutation> mutations,<a name="line.7604"></a> |
| <span class="sourceLineNo">7605</span> Collection<byte[]> rowsToLock, long nonceGroup, long nonce) throws IOException {<a name="line.7605"></a> |
| <span class="sourceLineNo">7606</span> batchMutate(new MutationBatchOperation(this, mutations.toArray(new Mutation[mutations.size()]),<a name="line.7606"></a> |
| <span class="sourceLineNo">7607</span> true, nonceGroup, nonce) {<a name="line.7607"></a> |
| <span class="sourceLineNo">7608</span> @Override<a name="line.7608"></a> |
| <span class="sourceLineNo">7609</span> public MiniBatchOperationInProgress<Mutation> lockRowsAndBuildMiniBatch(<a name="line.7609"></a> |
| <span class="sourceLineNo">7610</span> List<RowLock> acquiredRowLocks) throws IOException {<a name="line.7610"></a> |
| <span class="sourceLineNo">7611</span> RowLock prevRowLock = null;<a name="line.7611"></a> |
| <span class="sourceLineNo">7612</span> for (byte[] row : rowsToLock) {<a name="line.7612"></a> |
| <span class="sourceLineNo">7613</span> try {<a name="line.7613"></a> |
| <span class="sourceLineNo">7614</span> RowLock rowLock = region.getRowLockInternal(row, false, prevRowLock); // write lock<a name="line.7614"></a> |
| <span class="sourceLineNo">7615</span> if (rowLock != prevRowLock) {<a name="line.7615"></a> |
| <span class="sourceLineNo">7616</span> acquiredRowLocks.add(rowLock);<a name="line.7616"></a> |
| <span class="sourceLineNo">7617</span> prevRowLock = rowLock;<a name="line.7617"></a> |
| <span class="sourceLineNo">7618</span> }<a name="line.7618"></a> |
| <span class="sourceLineNo">7619</span> } catch (IOException ioe) {<a name="line.7619"></a> |
| <span class="sourceLineNo">7620</span> LOG.warn("Failed getting lock, row={}, in region {}", Bytes.toStringBinary(row), this,<a name="line.7620"></a> |
| <span class="sourceLineNo">7621</span> ioe);<a name="line.7621"></a> |
| <span class="sourceLineNo">7622</span> throw ioe;<a name="line.7622"></a> |
| <span class="sourceLineNo">7623</span> }<a name="line.7623"></a> |
| <span class="sourceLineNo">7624</span> }<a name="line.7624"></a> |
| <span class="sourceLineNo">7625</span> return createMiniBatch(size(), size());<a name="line.7625"></a> |
| <span class="sourceLineNo">7626</span> }<a name="line.7626"></a> |
| <span class="sourceLineNo">7627</span> });<a name="line.7627"></a> |
| <span class="sourceLineNo">7628</span> }<a name="line.7628"></a> |
| <span class="sourceLineNo">7629</span><a name="line.7629"></a> |
| <span class="sourceLineNo">7630</span> /**<a name="line.7630"></a> |
| <span class="sourceLineNo">7631</span> * @return statistics about the current load of the region<a name="line.7631"></a> |
| <span class="sourceLineNo">7632</span> */<a name="line.7632"></a> |
| <span class="sourceLineNo">7633</span> public ClientProtos.RegionLoadStats getLoadStatistics() {<a name="line.7633"></a> |
| <span class="sourceLineNo">7634</span> if (!regionStatsEnabled) {<a name="line.7634"></a> |
| <span class="sourceLineNo">7635</span> return null;<a name="line.7635"></a> |
| <span class="sourceLineNo">7636</span> }<a name="line.7636"></a> |
| <span class="sourceLineNo">7637</span> ClientProtos.RegionLoadStats.Builder stats = ClientProtos.RegionLoadStats.newBuilder();<a name="line.7637"></a> |
| <span class="sourceLineNo">7638</span> stats.setMemStoreLoad((int) (Math.min(100,<a name="line.7638"></a> |
| <span class="sourceLineNo">7639</span> (this.memStoreSizing.getMemStoreSize().getHeapSize() * 100) / this.memstoreFlushSize)));<a name="line.7639"></a> |
| <span class="sourceLineNo">7640</span> if (rsServices.getHeapMemoryManager() != null) {<a name="line.7640"></a> |
| <span class="sourceLineNo">7641</span> // the HeapMemoryManager uses -0.0 to signal a problem asking the JVM,<a name="line.7641"></a> |
| <span class="sourceLineNo">7642</span> // so we could just do the calculation below and we'll get a 0.<a name="line.7642"></a> |
| <span class="sourceLineNo">7643</span> // treating it as a special case analogous to no HMM instead so that it can be<a name="line.7643"></a> |
| <span class="sourceLineNo">7644</span> // programatically treated different from using <1% of heap.<a name="line.7644"></a> |
| <span class="sourceLineNo">7645</span> final float occupancy = rsServices.getHeapMemoryManager().getHeapOccupancyPercent();<a name="line.7645"></a> |
| <span class="sourceLineNo">7646</span> if (occupancy != HeapMemoryManager.HEAP_OCCUPANCY_ERROR_VALUE) {<a name="line.7646"></a> |
| <span class="sourceLineNo">7647</span> stats.setHeapOccupancy((int)(occupancy * 100));<a name="line.7647"></a> |
| <span class="sourceLineNo">7648</span> }<a name="line.7648"></a> |
| <span class="sourceLineNo">7649</span> }<a name="line.7649"></a> |
| <span class="sourceLineNo">7650</span> stats.setCompactionPressure((int) (rsServices.getCompactionPressure() * 100 > 100 ? 100<a name="line.7650"></a> |
| <span class="sourceLineNo">7651</span> : rsServices.getCompactionPressure() * 100));<a name="line.7651"></a> |
| <span class="sourceLineNo">7652</span> return stats.build();<a name="line.7652"></a> |
| <span class="sourceLineNo">7653</span> }<a name="line.7653"></a> |
| <span class="sourceLineNo">7654</span><a name="line.7654"></a> |
| <span class="sourceLineNo">7655</span> @Override<a name="line.7655"></a> |
| <span class="sourceLineNo">7656</span> public void processRowsWithLocks(RowProcessor<?,?> processor) throws IOException {<a name="line.7656"></a> |
| <span class="sourceLineNo">7657</span> processRowsWithLocks(processor, rowProcessorTimeout, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.7657"></a> |
| <span class="sourceLineNo">7658</span> }<a name="line.7658"></a> |
| <span class="sourceLineNo">7659</span><a name="line.7659"></a> |
| <span class="sourceLineNo">7660</span> @Override<a name="line.7660"></a> |
| <span class="sourceLineNo">7661</span> public void processRowsWithLocks(RowProcessor<?,?> processor, long nonceGroup, long nonce)<a name="line.7661"></a> |
| <span class="sourceLineNo">7662</span> throws IOException {<a name="line.7662"></a> |
| <span class="sourceLineNo">7663</span> processRowsWithLocks(processor, rowProcessorTimeout, nonceGroup, nonce);<a name="line.7663"></a> |
| <span class="sourceLineNo">7664</span> }<a name="line.7664"></a> |
| <span class="sourceLineNo">7665</span><a name="line.7665"></a> |
| <span class="sourceLineNo">7666</span> @Override<a name="line.7666"></a> |
| <span class="sourceLineNo">7667</span> public void processRowsWithLocks(RowProcessor<?,?> processor, long timeout,<a name="line.7667"></a> |
| <span class="sourceLineNo">7668</span> long nonceGroup, long nonce) throws IOException {<a name="line.7668"></a> |
| <span class="sourceLineNo">7669</span> for (byte[] row : processor.getRowsToLock()) {<a name="line.7669"></a> |
| <span class="sourceLineNo">7670</span> checkRow(row, "processRowsWithLocks");<a name="line.7670"></a> |
| <span class="sourceLineNo">7671</span> }<a name="line.7671"></a> |
| <span class="sourceLineNo">7672</span> if (!processor.readOnly()) {<a name="line.7672"></a> |
| <span class="sourceLineNo">7673</span> checkReadOnly();<a name="line.7673"></a> |
| <span class="sourceLineNo">7674</span> }<a name="line.7674"></a> |
| <span class="sourceLineNo">7675</span> checkResources();<a name="line.7675"></a> |
| <span class="sourceLineNo">7676</span> startRegionOperation();<a name="line.7676"></a> |
| <span class="sourceLineNo">7677</span> WALEdit walEdit = new WALEdit();<a name="line.7677"></a> |
| <span class="sourceLineNo">7678</span><a name="line.7678"></a> |
| <span class="sourceLineNo">7679</span> // STEP 1. Run pre-process hook<a name="line.7679"></a> |
| <span class="sourceLineNo">7680</span> preProcess(processor, walEdit);<a name="line.7680"></a> |
| <span class="sourceLineNo">7681</span> // Short circuit the read only case<a name="line.7681"></a> |
| <span class="sourceLineNo">7682</span> if (processor.readOnly()) {<a name="line.7682"></a> |
| <span class="sourceLineNo">7683</span> try {<a name="line.7683"></a> |
| <span class="sourceLineNo">7684</span> long now = EnvironmentEdgeManager.currentTime();<a name="line.7684"></a> |
| <span class="sourceLineNo">7685</span> doProcessRowWithTimeout(processor, now, this, null, null, timeout);<a name="line.7685"></a> |
| <span class="sourceLineNo">7686</span> processor.postProcess(this, walEdit, true);<a name="line.7686"></a> |
| <span class="sourceLineNo">7687</span> } finally {<a name="line.7687"></a> |
| <span class="sourceLineNo">7688</span> closeRegionOperation();<a name="line.7688"></a> |
| <span class="sourceLineNo">7689</span> }<a name="line.7689"></a> |
| <span class="sourceLineNo">7690</span> return;<a name="line.7690"></a> |
| <span class="sourceLineNo">7691</span> }<a name="line.7691"></a> |
| <span class="sourceLineNo">7692</span><a name="line.7692"></a> |
| <span class="sourceLineNo">7693</span> boolean locked = false;<a name="line.7693"></a> |
| <span class="sourceLineNo">7694</span> List<RowLock> acquiredRowLocks = null;<a name="line.7694"></a> |
| <span class="sourceLineNo">7695</span> List<Mutation> mutations = new ArrayList<>();<a name="line.7695"></a> |
| <span class="sourceLineNo">7696</span> Collection<byte[]> rowsToLock = processor.getRowsToLock();<a name="line.7696"></a> |
| <span class="sourceLineNo">7697</span> // This is assigned by mvcc either explicity in the below or in the guts of the WAL append<a name="line.7697"></a> |
| <span class="sourceLineNo">7698</span> // when it assigns the edit a sequencedid (A.K.A the mvcc write number).<a name="line.7698"></a> |
| <span class="sourceLineNo">7699</span> WriteEntry writeEntry = null;<a name="line.7699"></a> |
| <span class="sourceLineNo">7700</span> MemStoreSizing memstoreAccounting = new NonThreadSafeMemStoreSizing();<a name="line.7700"></a> |
| <span class="sourceLineNo">7701</span><a name="line.7701"></a> |
| <span class="sourceLineNo">7702</span> // Check for thread interrupt status in case we have been signaled from<a name="line.7702"></a> |
| <span class="sourceLineNo">7703</span> // #interruptRegionOperation.<a name="line.7703"></a> |
| <span class="sourceLineNo">7704</span> checkInterrupt();<a name="line.7704"></a> |
| <span class="sourceLineNo">7705</span><a name="line.7705"></a> |
| <span class="sourceLineNo">7706</span> try {<a name="line.7706"></a> |
| <span class="sourceLineNo">7707</span> boolean success = false;<a name="line.7707"></a> |
| <span class="sourceLineNo">7708</span> try {<a name="line.7708"></a> |
| <span class="sourceLineNo">7709</span> // STEP 2. Acquire the row lock(s)<a name="line.7709"></a> |
| <span class="sourceLineNo">7710</span> acquiredRowLocks = new ArrayList<>(rowsToLock.size());<a name="line.7710"></a> |
| <span class="sourceLineNo">7711</span> RowLock prevRowLock = null;<a name="line.7711"></a> |
| <span class="sourceLineNo">7712</span> for (byte[] row : rowsToLock) {<a name="line.7712"></a> |
| <span class="sourceLineNo">7713</span> // Attempt to lock all involved rows, throw if any lock times out<a name="line.7713"></a> |
| <span class="sourceLineNo">7714</span> // use a writer lock for mixed reads and writes<a name="line.7714"></a> |
| <span class="sourceLineNo">7715</span> RowLock rowLock = getRowLockInternal(row, false, prevRowLock);<a name="line.7715"></a> |
| <span class="sourceLineNo">7716</span> if (rowLock != prevRowLock) {<a name="line.7716"></a> |
| <span class="sourceLineNo">7717</span> acquiredRowLocks.add(rowLock);<a name="line.7717"></a> |
| <span class="sourceLineNo">7718</span> prevRowLock = rowLock;<a name="line.7718"></a> |
| <span class="sourceLineNo">7719</span> }<a name="line.7719"></a> |
| <span class="sourceLineNo">7720</span> }<a name="line.7720"></a> |
| <span class="sourceLineNo">7721</span><a name="line.7721"></a> |
| <span class="sourceLineNo">7722</span> // Check for thread interrupt status in case we have been signaled from<a name="line.7722"></a> |
| <span class="sourceLineNo">7723</span> // #interruptRegionOperation. Do it before we take the lock and disable interrupts for<a name="line.7723"></a> |
| <span class="sourceLineNo">7724</span> // the WAL append.<a name="line.7724"></a> |
| <span class="sourceLineNo">7725</span> checkInterrupt();<a name="line.7725"></a> |
| <span class="sourceLineNo">7726</span><a name="line.7726"></a> |
| <span class="sourceLineNo">7727</span> // STEP 3. Region lock<a name="line.7727"></a> |
| <span class="sourceLineNo">7728</span> lock(this.updatesLock.readLock(), acquiredRowLocks.isEmpty() ? 1 : acquiredRowLocks.size());<a name="line.7728"></a> |
| <span class="sourceLineNo">7729</span> locked = true;<a name="line.7729"></a> |
| <span class="sourceLineNo">7730</span><a name="line.7730"></a> |
| <span class="sourceLineNo">7731</span> // From this point until memstore update this operation should not be interrupted.<a name="line.7731"></a> |
| <span class="sourceLineNo">7732</span> disableInterrupts();<a name="line.7732"></a> |
| <span class="sourceLineNo">7733</span><a name="line.7733"></a> |
| <span class="sourceLineNo">7734</span> long now = EnvironmentEdgeManager.currentTime();<a name="line.7734"></a> |
| <span class="sourceLineNo">7735</span> // STEP 4. Let the processor scan the rows, generate mutations and add waledits<a name="line.7735"></a> |
| <span class="sourceLineNo">7736</span> doProcessRowWithTimeout(processor, now, this, mutations, walEdit, timeout);<a name="line.7736"></a> |
| <span class="sourceLineNo">7737</span> if (!mutations.isEmpty()) {<a name="line.7737"></a> |
| <span class="sourceLineNo">7738</span> writeRequestsCount.add(mutations.size());<a name="line.7738"></a> |
| <span class="sourceLineNo">7739</span> // STEP 5. Call the preBatchMutate hook<a name="line.7739"></a> |
| <span class="sourceLineNo">7740</span> processor.preBatchMutate(this, walEdit);<a name="line.7740"></a> |
| <span class="sourceLineNo">7741</span><a name="line.7741"></a> |
| <span class="sourceLineNo">7742</span> // STEP 6. Append and sync if walEdit has data to write out.<a name="line.7742"></a> |
| <span class="sourceLineNo">7743</span> if (!walEdit.isEmpty()) {<a name="line.7743"></a> |
| <span class="sourceLineNo">7744</span> writeEntry = doWALAppend(walEdit, getEffectiveDurability(processor.useDurability()),<a name="line.7744"></a> |
| <span class="sourceLineNo">7745</span> processor.getClusterIds(), now, nonceGroup, nonce);<a name="line.7745"></a> |
| <span class="sourceLineNo">7746</span> } else {<a name="line.7746"></a> |
| <span class="sourceLineNo">7747</span> // We are here if WAL is being skipped.<a name="line.7747"></a> |
| <span class="sourceLineNo">7748</span> writeEntry = this.mvcc.begin();<a name="line.7748"></a> |
| <span class="sourceLineNo">7749</span> }<a name="line.7749"></a> |
| <span class="sourceLineNo">7750</span><a name="line.7750"></a> |
| <span class="sourceLineNo">7751</span> // STEP 7. Apply to memstore<a name="line.7751"></a> |
| <span class="sourceLineNo">7752</span> long sequenceId = writeEntry.getWriteNumber();<a name="line.7752"></a> |
| <span class="sourceLineNo">7753</span> for (Mutation m : mutations) {<a name="line.7753"></a> |
| <span class="sourceLineNo">7754</span> // Handle any tag based cell features.<a name="line.7754"></a> |
| <span class="sourceLineNo">7755</span> // TODO: Do we need to call rewriteCellTags down in applyToMemStore()? Why not before<a name="line.7755"></a> |
| <span class="sourceLineNo">7756</span> // so tags go into WAL?<a name="line.7756"></a> |
| <span class="sourceLineNo">7757</span> rewriteCellTags(m.getFamilyCellMap(), m);<a name="line.7757"></a> |
| <span class="sourceLineNo">7758</span> for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) {<a name="line.7758"></a> |
| <span class="sourceLineNo">7759</span> Cell cell = cellScanner.current();<a name="line.7759"></a> |
| <span class="sourceLineNo">7760</span> if (walEdit.isEmpty()) {<a name="line.7760"></a> |
| <span class="sourceLineNo">7761</span> // If walEdit is empty, we put nothing in WAL. WAL stamps Cells with sequence id.<a name="line.7761"></a> |
| <span class="sourceLineNo">7762</span> // If no WAL, need to stamp it here.<a name="line.7762"></a> |
| <span class="sourceLineNo">7763</span> PrivateCellUtil.setSequenceId(cell, sequenceId);<a name="line.7763"></a> |
| <span class="sourceLineNo">7764</span> }<a name="line.7764"></a> |
| <span class="sourceLineNo">7765</span> applyToMemStore(getStore(cell), cell, memstoreAccounting);<a name="line.7765"></a> |
| <span class="sourceLineNo">7766</span> }<a name="line.7766"></a> |
| <span class="sourceLineNo">7767</span> }<a name="line.7767"></a> |
| <span class="sourceLineNo">7768</span><a name="line.7768"></a> |
| <span class="sourceLineNo">7769</span> // STEP 8. call postBatchMutate hook<a name="line.7769"></a> |
| <span class="sourceLineNo">7770</span> processor.postBatchMutate(this);<a name="line.7770"></a> |
| <span class="sourceLineNo">7771</span><a name="line.7771"></a> |
| <span class="sourceLineNo">7772</span> // STEP 9. Complete mvcc.<a name="line.7772"></a> |
| <span class="sourceLineNo">7773</span> mvcc.completeAndWait(writeEntry);<a name="line.7773"></a> |
| <span class="sourceLineNo">7774</span> writeEntry = null;<a name="line.7774"></a> |
| <span class="sourceLineNo">7775</span><a name="line.7775"></a> |
| <span class="sourceLineNo">7776</span> // STEP 10. Release region lock<a name="line.7776"></a> |
| <span class="sourceLineNo">7777</span> if (locked) {<a name="line.7777"></a> |
| <span class="sourceLineNo">7778</span> this.updatesLock.readLock().unlock();<a name="line.7778"></a> |
| <span class="sourceLineNo">7779</span> locked = false;<a name="line.7779"></a> |
| <span class="sourceLineNo">7780</span> }<a name="line.7780"></a> |
| <span class="sourceLineNo">7781</span><a name="line.7781"></a> |
| <span class="sourceLineNo">7782</span> // STEP 11. Release row lock(s)<a name="line.7782"></a> |
| <span class="sourceLineNo">7783</span> releaseRowLocks(acquiredRowLocks);<a name="line.7783"></a> |
| <span class="sourceLineNo">7784</span><a name="line.7784"></a> |
| <span class="sourceLineNo">7785</span> if (rsServices != null && rsServices.getMetrics() != null) {<a name="line.7785"></a> |
| <span class="sourceLineNo">7786</span> rsServices.getMetrics().updateWriteQueryMeter(this.htableDescriptor.<a name="line.7786"></a> |
| <span class="sourceLineNo">7787</span> getTableName(), mutations.size());<a name="line.7787"></a> |
| <span class="sourceLineNo">7788</span> }<a name="line.7788"></a> |
| <span class="sourceLineNo">7789</span> }<a name="line.7789"></a> |
| <span class="sourceLineNo">7790</span> success = true;<a name="line.7790"></a> |
| <span class="sourceLineNo">7791</span> } finally {<a name="line.7791"></a> |
| <span class="sourceLineNo">7792</span> // Call complete rather than completeAndWait because we probably had error if walKey != null<a name="line.7792"></a> |
| <span class="sourceLineNo">7793</span> if (writeEntry != null) mvcc.complete(writeEntry);<a name="line.7793"></a> |
| <span class="sourceLineNo">7794</span> if (locked) {<a name="line.7794"></a> |
| <span class="sourceLineNo">7795</span> this.updatesLock.readLock().unlock();<a name="line.7795"></a> |
| <span class="sourceLineNo">7796</span> }<a name="line.7796"></a> |
| <span class="sourceLineNo">7797</span> // release locks if some were acquired but another timed out<a name="line.7797"></a> |
| <span class="sourceLineNo">7798</span> releaseRowLocks(acquiredRowLocks);<a name="line.7798"></a> |
| <span class="sourceLineNo">7799</span><a name="line.7799"></a> |
| <span class="sourceLineNo">7800</span> enableInterrupts();<a name="line.7800"></a> |
| <span class="sourceLineNo">7801</span> }<a name="line.7801"></a> |
| <span class="sourceLineNo">7802</span><a name="line.7802"></a> |
| <span class="sourceLineNo">7803</span> // 12. Run post-process hook<a name="line.7803"></a> |
| <span class="sourceLineNo">7804</span> processor.postProcess(this, walEdit, success);<a name="line.7804"></a> |
| <span class="sourceLineNo">7805</span> } finally {<a name="line.7805"></a> |
| <span class="sourceLineNo">7806</span> closeRegionOperation();<a name="line.7806"></a> |
| <span class="sourceLineNo">7807</span> if (!mutations.isEmpty()) {<a name="line.7807"></a> |
| <span class="sourceLineNo">7808</span> this.incMemStoreSize(memstoreAccounting.getMemStoreSize());<a name="line.7808"></a> |
| <span class="sourceLineNo">7809</span> requestFlushIfNeeded();<a name="line.7809"></a> |
| <span class="sourceLineNo">7810</span> }<a name="line.7810"></a> |
| <span class="sourceLineNo">7811</span> }<a name="line.7811"></a> |
| <span class="sourceLineNo">7812</span> }<a name="line.7812"></a> |
| <span class="sourceLineNo">7813</span><a name="line.7813"></a> |
| <span class="sourceLineNo">7814</span> private void preProcess(final RowProcessor<?,?> processor, final WALEdit walEdit)<a name="line.7814"></a> |
| <span class="sourceLineNo">7815</span> throws IOException {<a name="line.7815"></a> |
| <span class="sourceLineNo">7816</span> try {<a name="line.7816"></a> |
| <span class="sourceLineNo">7817</span> processor.preProcess(this, walEdit);<a name="line.7817"></a> |
| <span class="sourceLineNo">7818</span> } catch (IOException e) {<a name="line.7818"></a> |
| <span class="sourceLineNo">7819</span> closeRegionOperation();<a name="line.7819"></a> |
| <span class="sourceLineNo">7820</span> throw e;<a name="line.7820"></a> |
| <span class="sourceLineNo">7821</span> }<a name="line.7821"></a> |
| <span class="sourceLineNo">7822</span> }<a name="line.7822"></a> |
| <span class="sourceLineNo">7823</span><a name="line.7823"></a> |
| <span class="sourceLineNo">7824</span> private void doProcessRowWithTimeout(final RowProcessor<?,?> processor,<a name="line.7824"></a> |
| <span class="sourceLineNo">7825</span> final long now,<a name="line.7825"></a> |
| <span class="sourceLineNo">7826</span> final HRegion region,<a name="line.7826"></a> |
| <span class="sourceLineNo">7827</span> final List<Mutation> mutations,<a name="line.7827"></a> |
| <span class="sourceLineNo">7828</span> final WALEdit walEdit,<a name="line.7828"></a> |
| <span class="sourceLineNo">7829</span> final long timeout) throws IOException {<a name="line.7829"></a> |
| <span class="sourceLineNo">7830</span> // Short circuit the no time bound case.<a name="line.7830"></a> |
| <span class="sourceLineNo">7831</span> if (timeout < 0) {<a name="line.7831"></a> |
| <span class="sourceLineNo">7832</span> try {<a name="line.7832"></a> |
| <span class="sourceLineNo">7833</span> processor.process(now, region, mutations, walEdit);<a name="line.7833"></a> |
| <span class="sourceLineNo">7834</span> } catch (IOException e) {<a name="line.7834"></a> |
| <span class="sourceLineNo">7835</span> String row = processor.getRowsToLock().isEmpty() ? "" :<a name="line.7835"></a> |
| <span class="sourceLineNo">7836</span> " on row(s):" + Bytes.toStringBinary(processor.getRowsToLock().iterator().next()) + "...";<a name="line.7836"></a> |
| <span class="sourceLineNo">7837</span> LOG.warn("RowProcessor: {}, in region {}, throws Exception {}",<a name="line.7837"></a> |
| <span class="sourceLineNo">7838</span> processor.getClass().getName(), getRegionInfo().getRegionNameAsString(), row, e);<a name="line.7838"></a> |
| <span class="sourceLineNo">7839</span> throw e;<a name="line.7839"></a> |
| <span class="sourceLineNo">7840</span> }<a name="line.7840"></a> |
| <span class="sourceLineNo">7841</span> return;<a name="line.7841"></a> |
| <span class="sourceLineNo">7842</span> }<a name="line.7842"></a> |
| <span class="sourceLineNo">7843</span><a name="line.7843"></a> |
| <span class="sourceLineNo">7844</span> // Case with time bound<a name="line.7844"></a> |
| <span class="sourceLineNo">7845</span> FutureTask<Void> task = new FutureTask<>(new Callable<Void>() {<a name="line.7845"></a> |
| <span class="sourceLineNo">7846</span> @Override<a name="line.7846"></a> |
| <span class="sourceLineNo">7847</span> public Void call() throws IOException {<a name="line.7847"></a> |
| <span class="sourceLineNo">7848</span> try {<a name="line.7848"></a> |
| <span class="sourceLineNo">7849</span> processor.process(now, region, mutations, walEdit);<a name="line.7849"></a> |
| <span class="sourceLineNo">7850</span> return null;<a name="line.7850"></a> |
| <span class="sourceLineNo">7851</span> } catch (IOException e) {<a name="line.7851"></a> |
| <span class="sourceLineNo">7852</span> String row = processor.getRowsToLock().isEmpty() ? "" :<a name="line.7852"></a> |
| <span class="sourceLineNo">7853</span> " on row(s):" + Bytes.toStringBinary(processor.getRowsToLock().iterator().next()) + "...";<a name="line.7853"></a> |
| <span class="sourceLineNo">7854</span> LOG.warn("RowProcessor: {}, in region {}, throws Exception {}",<a name="line.7854"></a> |
| <span class="sourceLineNo">7855</span> processor.getClass().getName(), getRegionInfo().getRegionNameAsString(), row, e);<a name="line.7855"></a> |
| <span class="sourceLineNo">7856</span> throw e;<a name="line.7856"></a> |
| <span class="sourceLineNo">7857</span> }<a name="line.7857"></a> |
| <span class="sourceLineNo">7858</span> }<a name="line.7858"></a> |
| <span class="sourceLineNo">7859</span> });<a name="line.7859"></a> |
| <span class="sourceLineNo">7860</span> rowProcessorExecutor.execute(task);<a name="line.7860"></a> |
| <span class="sourceLineNo">7861</span> try {<a name="line.7861"></a> |
| <span class="sourceLineNo">7862</span> task.get(timeout, TimeUnit.MILLISECONDS);<a name="line.7862"></a> |
| <span class="sourceLineNo">7863</span> } catch (InterruptedException ie) {<a name="line.7863"></a> |
| <span class="sourceLineNo">7864</span> throw throwOnInterrupt(ie);<a name="line.7864"></a> |
| <span class="sourceLineNo">7865</span> } catch (TimeoutException te) {<a name="line.7865"></a> |
| <span class="sourceLineNo">7866</span> String row = processor.getRowsToLock().isEmpty() ? "" :<a name="line.7866"></a> |
| <span class="sourceLineNo">7867</span> " on row(s):" + Bytes.toStringBinary(processor.getRowsToLock().iterator().next()) + "...";<a name="line.7867"></a> |
| <span class="sourceLineNo">7868</span> LOG.error("RowProcessor timeout: {} ms, in region {}, {}", timeout,<a name="line.7868"></a> |
| <span class="sourceLineNo">7869</span> getRegionInfo().getRegionNameAsString(), row);<a name="line.7869"></a> |
| <span class="sourceLineNo">7870</span> throw new IOException(te);<a name="line.7870"></a> |
| <span class="sourceLineNo">7871</span> } catch (Exception e) {<a name="line.7871"></a> |
| <span class="sourceLineNo">7872</span> throw new IOException(e);<a name="line.7872"></a> |
| <span class="sourceLineNo">7873</span> }<a name="line.7873"></a> |
| <span class="sourceLineNo">7874</span> }<a name="line.7874"></a> |
| <span class="sourceLineNo">7875</span><a name="line.7875"></a> |
| <span class="sourceLineNo">7876</span> @Override<a name="line.7876"></a> |
| <span class="sourceLineNo">7877</span> public Result append(Append append) throws IOException {<a name="line.7877"></a> |
| <span class="sourceLineNo">7878</span> return append(append, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.7878"></a> |
| <span class="sourceLineNo">7879</span> }<a name="line.7879"></a> |
| <span class="sourceLineNo">7880</span><a name="line.7880"></a> |
| <span class="sourceLineNo">7881</span> public Result append(Append append, long nonceGroup, long nonce) throws IOException {<a name="line.7881"></a> |
| <span class="sourceLineNo">7882</span> checkReadOnly();<a name="line.7882"></a> |
| <span class="sourceLineNo">7883</span> checkResources();<a name="line.7883"></a> |
| <span class="sourceLineNo">7884</span> startRegionOperation(Operation.APPEND);<a name="line.7884"></a> |
| <span class="sourceLineNo">7885</span> try {<a name="line.7885"></a> |
| <span class="sourceLineNo">7886</span> // All edits for the given row (across all column families) must happen atomically.<a name="line.7886"></a> |
| <span class="sourceLineNo">7887</span> return mutate(append, true, nonceGroup, nonce).getResult();<a name="line.7887"></a> |
| <span class="sourceLineNo">7888</span> } finally {<a name="line.7888"></a> |
| <span class="sourceLineNo">7889</span> closeRegionOperation(Operation.APPEND);<a name="line.7889"></a> |
| <span class="sourceLineNo">7890</span> }<a name="line.7890"></a> |
| <span class="sourceLineNo">7891</span> }<a name="line.7891"></a> |
| <span class="sourceLineNo">7892</span><a name="line.7892"></a> |
| <span class="sourceLineNo">7893</span> @Override<a name="line.7893"></a> |
| <span class="sourceLineNo">7894</span> public Result increment(Increment increment) throws IOException {<a name="line.7894"></a> |
| <span class="sourceLineNo">7895</span> return increment(increment, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.7895"></a> |
| <span class="sourceLineNo">7896</span> }<a name="line.7896"></a> |
| <span class="sourceLineNo">7897</span><a name="line.7897"></a> |
| <span class="sourceLineNo">7898</span> public Result increment(Increment increment, long nonceGroup, long nonce) throws IOException {<a name="line.7898"></a> |
| <span class="sourceLineNo">7899</span> checkReadOnly();<a name="line.7899"></a> |
| <span class="sourceLineNo">7900</span> checkResources();<a name="line.7900"></a> |
| <span class="sourceLineNo">7901</span> startRegionOperation(Operation.INCREMENT);<a name="line.7901"></a> |
| <span class="sourceLineNo">7902</span> try {<a name="line.7902"></a> |
| <span class="sourceLineNo">7903</span> // All edits for the given row (across all column families) must happen atomically.<a name="line.7903"></a> |
| <span class="sourceLineNo">7904</span> return mutate(increment, true, nonceGroup, nonce).getResult();<a name="line.7904"></a> |
| <span class="sourceLineNo">7905</span> } finally {<a name="line.7905"></a> |
| <span class="sourceLineNo">7906</span> closeRegionOperation(Operation.INCREMENT);<a name="line.7906"></a> |
| <span class="sourceLineNo">7907</span> }<a name="line.7907"></a> |
| <span class="sourceLineNo">7908</span> }<a name="line.7908"></a> |
| <span class="sourceLineNo">7909</span><a name="line.7909"></a> |
| <span class="sourceLineNo">7910</span> private WriteEntry doWALAppend(WALEdit walEdit, Durability durability, List<UUID> clusterIds,<a name="line.7910"></a> |
| <span class="sourceLineNo">7911</span> long now, long nonceGroup, long nonce) throws IOException {<a name="line.7911"></a> |
| <span class="sourceLineNo">7912</span> return doWALAppend(walEdit, durability, clusterIds, now, nonceGroup, nonce,<a name="line.7912"></a> |
| <span class="sourceLineNo">7913</span> SequenceId.NO_SEQUENCE_ID);<a name="line.7913"></a> |
| <span class="sourceLineNo">7914</span> }<a name="line.7914"></a> |
| <span class="sourceLineNo">7915</span><a name="line.7915"></a> |
| <span class="sourceLineNo">7916</span> /**<a name="line.7916"></a> |
| <span class="sourceLineNo">7917</span> * @return writeEntry associated with this append<a name="line.7917"></a> |
| <span class="sourceLineNo">7918</span> */<a name="line.7918"></a> |
| <span class="sourceLineNo">7919</span> private WriteEntry doWALAppend(WALEdit walEdit, Durability durability, List<UUID> clusterIds,<a name="line.7919"></a> |
| <span class="sourceLineNo">7920</span> long now, long nonceGroup, long nonce, long origLogSeqNum) throws IOException {<a name="line.7920"></a> |
| <span class="sourceLineNo">7921</span> Preconditions.checkArgument(walEdit != null && !walEdit.isEmpty(),<a name="line.7921"></a> |
| <span class="sourceLineNo">7922</span> "WALEdit is null or empty!");<a name="line.7922"></a> |
| <span class="sourceLineNo">7923</span> Preconditions.checkArgument(!walEdit.isReplay() || origLogSeqNum != SequenceId.NO_SEQUENCE_ID,<a name="line.7923"></a> |
| <span class="sourceLineNo">7924</span> "Invalid replay sequence Id for replay WALEdit!");<a name="line.7924"></a> |
| <span class="sourceLineNo">7925</span> // Using default cluster id, as this can only happen in the originating cluster.<a name="line.7925"></a> |
| <span class="sourceLineNo">7926</span> // A slave cluster receives the final value (not the delta) as a Put. We use HLogKey<a name="line.7926"></a> |
| <span class="sourceLineNo">7927</span> // here instead of WALKeyImpl directly to support legacy coprocessors.<a name="line.7927"></a> |
| <span class="sourceLineNo">7928</span> WALKeyImpl walKey = walEdit.isReplay()?<a name="line.7928"></a> |
| <span class="sourceLineNo">7929</span> new WALKeyImpl(this.getRegionInfo().getEncodedNameAsBytes(),<a name="line.7929"></a> |
| <span class="sourceLineNo">7930</span> this.htableDescriptor.getTableName(), SequenceId.NO_SEQUENCE_ID, now, clusterIds,<a name="line.7930"></a> |
| <span class="sourceLineNo">7931</span> nonceGroup, nonce, mvcc) :<a name="line.7931"></a> |
| <span class="sourceLineNo">7932</span> new WALKeyImpl(this.getRegionInfo().getEncodedNameAsBytes(),<a name="line.7932"></a> |
| <span class="sourceLineNo">7933</span> this.htableDescriptor.getTableName(), SequenceId.NO_SEQUENCE_ID, now, clusterIds,<a name="line.7933"></a> |
| <span class="sourceLineNo">7934</span> nonceGroup, nonce, mvcc, this.getReplicationScope());<a name="line.7934"></a> |
| <span class="sourceLineNo">7935</span> if (walEdit.isReplay()) {<a name="line.7935"></a> |
| <span class="sourceLineNo">7936</span> walKey.setOrigLogSeqNum(origLogSeqNum);<a name="line.7936"></a> |
| <span class="sourceLineNo">7937</span> }<a name="line.7937"></a> |
| <span class="sourceLineNo">7938</span> //don't call the coproc hook for writes to the WAL caused by<a name="line.7938"></a> |
| <span class="sourceLineNo">7939</span> //system lifecycle events like flushes or compactions<a name="line.7939"></a> |
| <span class="sourceLineNo">7940</span> if (this.coprocessorHost != null && !walEdit.isMetaEdit()) {<a name="line.7940"></a> |
| <span class="sourceLineNo">7941</span> this.coprocessorHost.preWALAppend(walKey, walEdit);<a name="line.7941"></a> |
| <span class="sourceLineNo">7942</span> }<a name="line.7942"></a> |
| <span class="sourceLineNo">7943</span> WriteEntry writeEntry = null;<a name="line.7943"></a> |
| <span class="sourceLineNo">7944</span> try {<a name="line.7944"></a> |
| <span class="sourceLineNo">7945</span> long txid = this.wal.appendData(this.getRegionInfo(), walKey, walEdit);<a name="line.7945"></a> |
| <span class="sourceLineNo">7946</span> // Call sync on our edit.<a name="line.7946"></a> |
| <span class="sourceLineNo">7947</span> if (txid != 0) {<a name="line.7947"></a> |
| <span class="sourceLineNo">7948</span> sync(txid, durability);<a name="line.7948"></a> |
| <span class="sourceLineNo">7949</span> }<a name="line.7949"></a> |
| <span class="sourceLineNo">7950</span> writeEntry = walKey.getWriteEntry();<a name="line.7950"></a> |
| <span class="sourceLineNo">7951</span> } catch (IOException ioe) {<a name="line.7951"></a> |
| <span class="sourceLineNo">7952</span> if (walKey != null && walKey.getWriteEntry() != null) {<a name="line.7952"></a> |
| <span class="sourceLineNo">7953</span> mvcc.complete(walKey.getWriteEntry());<a name="line.7953"></a> |
| <span class="sourceLineNo">7954</span> }<a name="line.7954"></a> |
| <span class="sourceLineNo">7955</span> throw ioe;<a name="line.7955"></a> |
| <span class="sourceLineNo">7956</span> }<a name="line.7956"></a> |
| <span class="sourceLineNo">7957</span> return writeEntry;<a name="line.7957"></a> |
| <span class="sourceLineNo">7958</span> }<a name="line.7958"></a> |
| <span class="sourceLineNo">7959</span><a name="line.7959"></a> |
| <span class="sourceLineNo">7960</span> public static final long FIXED_OVERHEAD = ClassSize.estimateBase(HRegion.class, false);<a name="line.7960"></a> |
| <span class="sourceLineNo">7961</span><a name="line.7961"></a> |
| <span class="sourceLineNo">7962</span> // woefully out of date - currently missing:<a name="line.7962"></a> |
| <span class="sourceLineNo">7963</span> // 1 x HashMap - coprocessorServiceHandlers<a name="line.7963"></a> |
| <span class="sourceLineNo">7964</span> // 6 x LongAdder - numMutationsWithoutWAL, dataInMemoryWithoutWAL,<a name="line.7964"></a> |
| <span class="sourceLineNo">7965</span> // checkAndMutateChecksPassed, checkAndMutateChecksFailed, readRequestsCount,<a name="line.7965"></a> |
| <span class="sourceLineNo">7966</span> // writeRequestsCount, cpRequestsCount<a name="line.7966"></a> |
| <span class="sourceLineNo">7967</span> // 1 x HRegion$WriteState - writestate<a name="line.7967"></a> |
| <span class="sourceLineNo">7968</span> // 1 x RegionCoprocessorHost - coprocessorHost<a name="line.7968"></a> |
| <span class="sourceLineNo">7969</span> // 1 x RegionSplitPolicy - splitPolicy<a name="line.7969"></a> |
| <span class="sourceLineNo">7970</span> // 1 x MetricsRegion - metricsRegion<a name="line.7970"></a> |
| <span class="sourceLineNo">7971</span> // 1 x MetricsRegionWrapperImpl - metricsRegionWrapper<a name="line.7971"></a> |
| <span class="sourceLineNo">7972</span> public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +<a name="line.7972"></a> |
| <span class="sourceLineNo">7973</span> ClassSize.OBJECT + // closeLock<a name="line.7973"></a> |
| <span class="sourceLineNo">7974</span> (2 * ClassSize.ATOMIC_BOOLEAN) + // closed, closing<a name="line.7974"></a> |
| <span class="sourceLineNo">7975</span> (3 * ClassSize.ATOMIC_LONG) + // numPutsWithoutWAL, dataInMemoryWithoutWAL,<a name="line.7975"></a> |
| <span class="sourceLineNo">7976</span> // compactionsFailed<a name="line.7976"></a> |
| <span class="sourceLineNo">7977</span> (3 * ClassSize.CONCURRENT_HASHMAP) + // lockedRows, scannerReadPoints, regionLockHolders<a name="line.7977"></a> |
| <span class="sourceLineNo">7978</span> WriteState.HEAP_SIZE + // writestate<a name="line.7978"></a> |
| <span class="sourceLineNo">7979</span> ClassSize.CONCURRENT_SKIPLISTMAP + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + // stores<a name="line.7979"></a> |
| <span class="sourceLineNo">7980</span> (2 * ClassSize.REENTRANT_LOCK) + // lock, updatesLock<a name="line.7980"></a> |
| <span class="sourceLineNo">7981</span> MultiVersionConcurrencyControl.FIXED_SIZE // mvcc<a name="line.7981"></a> |
| <span class="sourceLineNo">7982</span> + 2 * ClassSize.TREEMAP // maxSeqIdInStores, replicationScopes<a name="line.7982"></a> |
| <span class="sourceLineNo">7983</span> + 2 * ClassSize.ATOMIC_INTEGER // majorInProgress, minorInProgress<a name="line.7983"></a> |
| <span class="sourceLineNo">7984</span> + ClassSize.STORE_SERVICES // store services<a name="line.7984"></a> |
| <span class="sourceLineNo">7985</span> + StoreHotnessProtector.FIXED_SIZE<a name="line.7985"></a> |
| <span class="sourceLineNo">7986</span> ;<a name="line.7986"></a> |
| <span class="sourceLineNo">7987</span><a name="line.7987"></a> |
| <span class="sourceLineNo">7988</span> @Override<a name="line.7988"></a> |
| <span class="sourceLineNo">7989</span> public long heapSize() {<a name="line.7989"></a> |
| <span class="sourceLineNo">7990</span> // this does not take into account row locks, recent flushes, mvcc entries, and more<a name="line.7990"></a> |
| <span class="sourceLineNo">7991</span> return DEEP_OVERHEAD + stores.values().stream().mapToLong(HStore::heapSize).sum();<a name="line.7991"></a> |
| <span class="sourceLineNo">7992</span> }<a name="line.7992"></a> |
| <span class="sourceLineNo">7993</span><a name="line.7993"></a> |
| <span class="sourceLineNo">7994</span> /**<a name="line.7994"></a> |
| <span class="sourceLineNo">7995</span> * Registers a new protocol buffer {@link Service} subclass as a coprocessor endpoint to be<a name="line.7995"></a> |
| <span class="sourceLineNo">7996</span> * available for handling {@link #execService(RpcController, CoprocessorServiceCall)} calls.<a name="line.7996"></a> |
| <span class="sourceLineNo">7997</span> * <p/><a name="line.7997"></a> |
| <span class="sourceLineNo">7998</span> * Only a single instance may be registered per region for a given {@link Service} subclass (the<a name="line.7998"></a> |
| <span class="sourceLineNo">7999</span> * instances are keyed on {@link ServiceDescriptor#getFullName()}.. After the first registration,<a name="line.7999"></a> |
| <span class="sourceLineNo">8000</span> * subsequent calls with the same service name will fail with a return value of {@code false}.<a name="line.8000"></a> |
| <span class="sourceLineNo">8001</span> * @param instance the {@code Service} subclass instance to expose as a coprocessor endpoint<a name="line.8001"></a> |
| <span class="sourceLineNo">8002</span> * @return {@code true} if the registration was successful, {@code false} otherwise<a name="line.8002"></a> |
| <span class="sourceLineNo">8003</span> */<a name="line.8003"></a> |
| <span class="sourceLineNo">8004</span> public boolean registerService(Service instance) {<a name="line.8004"></a> |
| <span class="sourceLineNo">8005</span> // No stacking of instances is allowed for a single service name<a name="line.8005"></a> |
| <span class="sourceLineNo">8006</span> ServiceDescriptor serviceDesc = instance.getDescriptorForType();<a name="line.8006"></a> |
| <span class="sourceLineNo">8007</span> String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc);<a name="line.8007"></a> |
| <span class="sourceLineNo">8008</span> if (coprocessorServiceHandlers.containsKey(serviceName)) {<a name="line.8008"></a> |
| <span class="sourceLineNo">8009</span> LOG.error("Coprocessor service {} already registered, rejecting request from {} in region {}",<a name="line.8009"></a> |
| <span class="sourceLineNo">8010</span> serviceName, instance, this);<a name="line.8010"></a> |
| <span class="sourceLineNo">8011</span> return false;<a name="line.8011"></a> |
| <span class="sourceLineNo">8012</span> }<a name="line.8012"></a> |
| <span class="sourceLineNo">8013</span><a name="line.8013"></a> |
| <span class="sourceLineNo">8014</span> coprocessorServiceHandlers.put(serviceName, instance);<a name="line.8014"></a> |
| <span class="sourceLineNo">8015</span> if (LOG.isDebugEnabled()) {<a name="line.8015"></a> |
| <span class="sourceLineNo">8016</span> LOG.debug("Registered coprocessor service: region=" +<a name="line.8016"></a> |
| <span class="sourceLineNo">8017</span> Bytes.toStringBinary(getRegionInfo().getRegionName()) + " service=" + serviceName);<a name="line.8017"></a> |
| <span class="sourceLineNo">8018</span> }<a name="line.8018"></a> |
| <span class="sourceLineNo">8019</span> return true;<a name="line.8019"></a> |
| <span class="sourceLineNo">8020</span> }<a name="line.8020"></a> |
| <span class="sourceLineNo">8021</span><a name="line.8021"></a> |
| <span class="sourceLineNo">8022</span> /**<a name="line.8022"></a> |
| <span class="sourceLineNo">8023</span> * Executes a single protocol buffer coprocessor endpoint {@link Service} method using<a name="line.8023"></a> |
| <span class="sourceLineNo">8024</span> * the registered protocol handlers. {@link Service} implementations must be registered via the<a name="line.8024"></a> |
| <span class="sourceLineNo">8025</span> * {@link #registerService(Service)}<a name="line.8025"></a> |
| <span class="sourceLineNo">8026</span> * method before they are available.<a name="line.8026"></a> |
| <span class="sourceLineNo">8027</span> *<a name="line.8027"></a> |
| <span class="sourceLineNo">8028</span> * @param controller an {@code RpcContoller} implementation to pass to the invoked service<a name="line.8028"></a> |
| <span class="sourceLineNo">8029</span> * @param call a {@code CoprocessorServiceCall} instance identifying the service, method,<a name="line.8029"></a> |
| <span class="sourceLineNo">8030</span> * and parameters for the method invocation<a name="line.8030"></a> |
| <span class="sourceLineNo">8031</span> * @return a protocol buffer {@code Message} instance containing the method's result<a name="line.8031"></a> |
| <span class="sourceLineNo">8032</span> * @throws IOException if no registered service handler is found or an error<a name="line.8032"></a> |
| <span class="sourceLineNo">8033</span> * occurs during the invocation<a name="line.8033"></a> |
| <span class="sourceLineNo">8034</span> * @see #registerService(Service)<a name="line.8034"></a> |
| <span class="sourceLineNo">8035</span> */<a name="line.8035"></a> |
| <span class="sourceLineNo">8036</span> public Message execService(RpcController controller, CoprocessorServiceCall call)<a name="line.8036"></a> |
| <span class="sourceLineNo">8037</span> throws IOException {<a name="line.8037"></a> |
| <span class="sourceLineNo">8038</span> String serviceName = call.getServiceName();<a name="line.8038"></a> |
| <span class="sourceLineNo">8039</span> Service service = coprocessorServiceHandlers.get(serviceName);<a name="line.8039"></a> |
| <span class="sourceLineNo">8040</span> if (service == null) {<a name="line.8040"></a> |
| <span class="sourceLineNo">8041</span> throw new UnknownProtocolException(null, "No registered coprocessor service found for " +<a name="line.8041"></a> |
| <span class="sourceLineNo">8042</span> serviceName + " in region " + Bytes.toStringBinary(getRegionInfo().getRegionName()));<a name="line.8042"></a> |
| <span class="sourceLineNo">8043</span> }<a name="line.8043"></a> |
| <span class="sourceLineNo">8044</span> ServiceDescriptor serviceDesc = service.getDescriptorForType();<a name="line.8044"></a> |
| <span class="sourceLineNo">8045</span><a name="line.8045"></a> |
| <span class="sourceLineNo">8046</span> cpRequestsCount.increment();<a name="line.8046"></a> |
| <span class="sourceLineNo">8047</span> String methodName = call.getMethodName();<a name="line.8047"></a> |
| <span class="sourceLineNo">8048</span> MethodDescriptor methodDesc =<a name="line.8048"></a> |
| <span class="sourceLineNo">8049</span> CoprocessorRpcUtils.getMethodDescriptor(methodName, serviceDesc);<a name="line.8049"></a> |
| <span class="sourceLineNo">8050</span><a name="line.8050"></a> |
| <span class="sourceLineNo">8051</span> Message.Builder builder =<a name="line.8051"></a> |
| <span class="sourceLineNo">8052</span> service.getRequestPrototype(methodDesc).newBuilderForType();<a name="line.8052"></a> |
| <span class="sourceLineNo">8053</span><a name="line.8053"></a> |
| <span class="sourceLineNo">8054</span> ProtobufUtil.mergeFrom(builder,<a name="line.8054"></a> |
| <span class="sourceLineNo">8055</span> call.getRequest().toByteArray());<a name="line.8055"></a> |
| <span class="sourceLineNo">8056</span> Message request =<a name="line.8056"></a> |
| <span class="sourceLineNo">8057</span> CoprocessorRpcUtils.getRequest(service, methodDesc, call.getRequest());<a name="line.8057"></a> |
| <span class="sourceLineNo">8058</span><a name="line.8058"></a> |
| <span class="sourceLineNo">8059</span> if (coprocessorHost != null) {<a name="line.8059"></a> |
| <span class="sourceLineNo">8060</span> request = coprocessorHost.preEndpointInvocation(service, methodName, request);<a name="line.8060"></a> |
| <span class="sourceLineNo">8061</span> }<a name="line.8061"></a> |
| <span class="sourceLineNo">8062</span><a name="line.8062"></a> |
| <span class="sourceLineNo">8063</span> final Message.Builder responseBuilder =<a name="line.8063"></a> |
| <span class="sourceLineNo">8064</span> service.getResponsePrototype(methodDesc).newBuilderForType();<a name="line.8064"></a> |
| <span class="sourceLineNo">8065</span> service.callMethod(methodDesc, controller, request, new RpcCallback<Message>() {<a name="line.8065"></a> |
| <span class="sourceLineNo">8066</span> @Override<a name="line.8066"></a> |
| <span class="sourceLineNo">8067</span> public void run(Message message) {<a name="line.8067"></a> |
| <span class="sourceLineNo">8068</span> if (message != null) {<a name="line.8068"></a> |
| <span class="sourceLineNo">8069</span> responseBuilder.mergeFrom(message);<a name="line.8069"></a> |
| <span class="sourceLineNo">8070</span> }<a name="line.8070"></a> |
| <span class="sourceLineNo">8071</span> }<a name="line.8071"></a> |
| <span class="sourceLineNo">8072</span> });<a name="line.8072"></a> |
| <span class="sourceLineNo">8073</span><a name="line.8073"></a> |
| <span class="sourceLineNo">8074</span> if (coprocessorHost != null) {<a name="line.8074"></a> |
| <span class="sourceLineNo">8075</span> coprocessorHost.postEndpointInvocation(service, methodName, request, responseBuilder);<a name="line.8075"></a> |
| <span class="sourceLineNo">8076</span> }<a name="line.8076"></a> |
| <span class="sourceLineNo">8077</span> IOException exception =<a name="line.8077"></a> |
| <span class="sourceLineNo">8078</span> org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils.getControllerException(controller);<a name="line.8078"></a> |
| <span class="sourceLineNo">8079</span> if (exception != null) {<a name="line.8079"></a> |
| <span class="sourceLineNo">8080</span> throw exception;<a name="line.8080"></a> |
| <span class="sourceLineNo">8081</span> }<a name="line.8081"></a> |
| <span class="sourceLineNo">8082</span><a name="line.8082"></a> |
| <span class="sourceLineNo">8083</span> return responseBuilder.build();<a name="line.8083"></a> |
| <span class="sourceLineNo">8084</span> }<a name="line.8084"></a> |
| <span class="sourceLineNo">8085</span><a name="line.8085"></a> |
| <span class="sourceLineNo">8086</span> public Optional<byte[]> checkSplit() {<a name="line.8086"></a> |
| <span class="sourceLineNo">8087</span> return checkSplit(false);<a name="line.8087"></a> |
| <span class="sourceLineNo">8088</span> }<a name="line.8088"></a> |
| <span class="sourceLineNo">8089</span><a name="line.8089"></a> |
| <span class="sourceLineNo">8090</span> /**<a name="line.8090"></a> |
| <span class="sourceLineNo">8091</span> * Return the split point. An empty result indicates the region isn't splittable.<a name="line.8091"></a> |
| <span class="sourceLineNo">8092</span> */<a name="line.8092"></a> |
| <span class="sourceLineNo">8093</span> public Optional<byte[]> checkSplit(boolean force) {<a name="line.8093"></a> |
| <span class="sourceLineNo">8094</span> // Can't split META<a name="line.8094"></a> |
| <span class="sourceLineNo">8095</span> if (this.getRegionInfo().isMetaRegion()) {<a name="line.8095"></a> |
| <span class="sourceLineNo">8096</span> return Optional.empty();<a name="line.8096"></a> |
| <span class="sourceLineNo">8097</span> }<a name="line.8097"></a> |
| <span class="sourceLineNo">8098</span><a name="line.8098"></a> |
| <span class="sourceLineNo">8099</span> // Can't split a region that is closing.<a name="line.8099"></a> |
| <span class="sourceLineNo">8100</span> if (this.isClosing()) {<a name="line.8100"></a> |
| <span class="sourceLineNo">8101</span> return Optional.empty();<a name="line.8101"></a> |
| <span class="sourceLineNo">8102</span> }<a name="line.8102"></a> |
| <span class="sourceLineNo">8103</span><a name="line.8103"></a> |
| <span class="sourceLineNo">8104</span> if (!force && !splitPolicy.shouldSplit()) {<a name="line.8104"></a> |
| <span class="sourceLineNo">8105</span> return Optional.empty();<a name="line.8105"></a> |
| <span class="sourceLineNo">8106</span> }<a name="line.8106"></a> |
| <span class="sourceLineNo">8107</span><a name="line.8107"></a> |
| <span class="sourceLineNo">8108</span> byte[] ret = splitPolicy.getSplitPoint();<a name="line.8108"></a> |
| <span class="sourceLineNo">8109</span><a name="line.8109"></a> |
| <span class="sourceLineNo">8110</span> if (ret != null) {<a name="line.8110"></a> |
| <span class="sourceLineNo">8111</span> try {<a name="line.8111"></a> |
| <span class="sourceLineNo">8112</span> checkRow(ret, "calculated split");<a name="line.8112"></a> |
| <span class="sourceLineNo">8113</span> } catch (IOException e) {<a name="line.8113"></a> |
| <span class="sourceLineNo">8114</span> LOG.error("Ignoring invalid split for region {}", this, e);<a name="line.8114"></a> |
| <span class="sourceLineNo">8115</span> return Optional.empty();<a name="line.8115"></a> |
| <span class="sourceLineNo">8116</span> }<a name="line.8116"></a> |
| <span class="sourceLineNo">8117</span> return Optional.of(ret);<a name="line.8117"></a> |
| <span class="sourceLineNo">8118</span> } else {<a name="line.8118"></a> |
| <span class="sourceLineNo">8119</span> return Optional.empty();<a name="line.8119"></a> |
| <span class="sourceLineNo">8120</span> }<a name="line.8120"></a> |
| <span class="sourceLineNo">8121</span> }<a name="line.8121"></a> |
| <span class="sourceLineNo">8122</span><a name="line.8122"></a> |
| <span class="sourceLineNo">8123</span> /**<a name="line.8123"></a> |
| <span class="sourceLineNo">8124</span> * @return The priority that this region should have in the compaction queue<a name="line.8124"></a> |
| <span class="sourceLineNo">8125</span> */<a name="line.8125"></a> |
| <span class="sourceLineNo">8126</span> public int getCompactPriority() {<a name="line.8126"></a> |
| <span class="sourceLineNo">8127</span> return stores.values().stream().mapToInt(HStore::getCompactPriority).min()<a name="line.8127"></a> |
| <span class="sourceLineNo">8128</span> .orElse(Store.NO_PRIORITY);<a name="line.8128"></a> |
| <span class="sourceLineNo">8129</span> }<a name="line.8129"></a> |
| <span class="sourceLineNo">8130</span><a name="line.8130"></a> |
| <span class="sourceLineNo">8131</span> /** @return the coprocessor host */<a name="line.8131"></a> |
| <span class="sourceLineNo">8132</span> public RegionCoprocessorHost getCoprocessorHost() {<a name="line.8132"></a> |
| <span class="sourceLineNo">8133</span> return coprocessorHost;<a name="line.8133"></a> |
| <span class="sourceLineNo">8134</span> }<a name="line.8134"></a> |
| <span class="sourceLineNo">8135</span><a name="line.8135"></a> |
| <span class="sourceLineNo">8136</span> /** @param coprocessorHost the new coprocessor host */<a name="line.8136"></a> |
| <span class="sourceLineNo">8137</span> public void setCoprocessorHost(final RegionCoprocessorHost coprocessorHost) {<a name="line.8137"></a> |
| <span class="sourceLineNo">8138</span> this.coprocessorHost = coprocessorHost;<a name="line.8138"></a> |
| <span class="sourceLineNo">8139</span> }<a name="line.8139"></a> |
| <span class="sourceLineNo">8140</span><a name="line.8140"></a> |
| <span class="sourceLineNo">8141</span> @Override<a name="line.8141"></a> |
| <span class="sourceLineNo">8142</span> public void startRegionOperation() throws IOException {<a name="line.8142"></a> |
| <span class="sourceLineNo">8143</span> startRegionOperation(Operation.ANY);<a name="line.8143"></a> |
| <span class="sourceLineNo">8144</span> }<a name="line.8144"></a> |
| <span class="sourceLineNo">8145</span><a name="line.8145"></a> |
| <span class="sourceLineNo">8146</span> @Override<a name="line.8146"></a> |
| <span class="sourceLineNo">8147</span> public void startRegionOperation(Operation op) throws IOException {<a name="line.8147"></a> |
| <span class="sourceLineNo">8148</span> boolean isInterruptableOp = false;<a name="line.8148"></a> |
| <span class="sourceLineNo">8149</span> switch (op) {<a name="line.8149"></a> |
| <span class="sourceLineNo">8150</span> case GET: // interruptible read operations<a name="line.8150"></a> |
| <span class="sourceLineNo">8151</span> case SCAN:<a name="line.8151"></a> |
| <span class="sourceLineNo">8152</span> isInterruptableOp = true;<a name="line.8152"></a> |
| <span class="sourceLineNo">8153</span> checkReadsEnabled();<a name="line.8153"></a> |
| <span class="sourceLineNo">8154</span> break;<a name="line.8154"></a> |
| <span class="sourceLineNo">8155</span> case INCREMENT: // interruptible write operations<a name="line.8155"></a> |
| <span class="sourceLineNo">8156</span> case APPEND:<a name="line.8156"></a> |
| <span class="sourceLineNo">8157</span> case PUT:<a name="line.8157"></a> |
| <span class="sourceLineNo">8158</span> case DELETE:<a name="line.8158"></a> |
| <span class="sourceLineNo">8159</span> case BATCH_MUTATE:<a name="line.8159"></a> |
| <span class="sourceLineNo">8160</span> case CHECK_AND_MUTATE:<a name="line.8160"></a> |
| <span class="sourceLineNo">8161</span> isInterruptableOp = true;<a name="line.8161"></a> |
| <span class="sourceLineNo">8162</span> break;<a name="line.8162"></a> |
| <span class="sourceLineNo">8163</span> default: // all others<a name="line.8163"></a> |
| <span class="sourceLineNo">8164</span> break;<a name="line.8164"></a> |
| <span class="sourceLineNo">8165</span> }<a name="line.8165"></a> |
| <span class="sourceLineNo">8166</span> if (op == Operation.MERGE_REGION || op == Operation.SPLIT_REGION<a name="line.8166"></a> |
| <span class="sourceLineNo">8167</span> || op == Operation.COMPACT_REGION || op == Operation.COMPACT_SWITCH) {<a name="line.8167"></a> |
| <span class="sourceLineNo">8168</span> // split, merge or compact region doesn't need to check the closing/closed state or lock the<a name="line.8168"></a> |
| <span class="sourceLineNo">8169</span> // region<a name="line.8169"></a> |
| <span class="sourceLineNo">8170</span> return;<a name="line.8170"></a> |
| <span class="sourceLineNo">8171</span> }<a name="line.8171"></a> |
| <span class="sourceLineNo">8172</span> if (this.closing.get()) {<a name="line.8172"></a> |
| <span class="sourceLineNo">8173</span> throw new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " is closing");<a name="line.8173"></a> |
| <span class="sourceLineNo">8174</span> }<a name="line.8174"></a> |
| <span class="sourceLineNo">8175</span> lock(lock.readLock());<a name="line.8175"></a> |
| <span class="sourceLineNo">8176</span> // Update regionLockHolders ONLY for any startRegionOperation call that is invoked from<a name="line.8176"></a> |
| <span class="sourceLineNo">8177</span> // an RPC handler<a name="line.8177"></a> |
| <span class="sourceLineNo">8178</span> Thread thisThread = Thread.currentThread();<a name="line.8178"></a> |
| <span class="sourceLineNo">8179</span> if (isInterruptableOp) {<a name="line.8179"></a> |
| <span class="sourceLineNo">8180</span> regionLockHolders.put(thisThread, true);<a name="line.8180"></a> |
| <span class="sourceLineNo">8181</span> }<a name="line.8181"></a> |
| <span class="sourceLineNo">8182</span> if (this.closed.get()) {<a name="line.8182"></a> |
| <span class="sourceLineNo">8183</span> lock.readLock().unlock();<a name="line.8183"></a> |
| <span class="sourceLineNo">8184</span> throw new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " is closed");<a name="line.8184"></a> |
| <span class="sourceLineNo">8185</span> }<a name="line.8185"></a> |
| <span class="sourceLineNo">8186</span> // The unit for snapshot is a region. So, all stores for this region must be<a name="line.8186"></a> |
| <span class="sourceLineNo">8187</span> // prepared for snapshot operation before proceeding.<a name="line.8187"></a> |
| <span class="sourceLineNo">8188</span> if (op == Operation.SNAPSHOT) {<a name="line.8188"></a> |
| <span class="sourceLineNo">8189</span> stores.values().forEach(HStore::preSnapshotOperation);<a name="line.8189"></a> |
| <span class="sourceLineNo">8190</span> }<a name="line.8190"></a> |
| <span class="sourceLineNo">8191</span> try {<a name="line.8191"></a> |
| <span class="sourceLineNo">8192</span> if (coprocessorHost != null) {<a name="line.8192"></a> |
| <span class="sourceLineNo">8193</span> coprocessorHost.postStartRegionOperation(op);<a name="line.8193"></a> |
| <span class="sourceLineNo">8194</span> }<a name="line.8194"></a> |
| <span class="sourceLineNo">8195</span> } catch (Exception e) {<a name="line.8195"></a> |
| <span class="sourceLineNo">8196</span> if (isInterruptableOp) {<a name="line.8196"></a> |
| <span class="sourceLineNo">8197</span> // would be harmless to remove what we didn't add but we know by 'isInterruptableOp'<a name="line.8197"></a> |
| <span class="sourceLineNo">8198</span> // if we added this thread to regionLockHolders<a name="line.8198"></a> |
| <span class="sourceLineNo">8199</span> regionLockHolders.remove(thisThread);<a name="line.8199"></a> |
| <span class="sourceLineNo">8200</span> }<a name="line.8200"></a> |
| <span class="sourceLineNo">8201</span> lock.readLock().unlock();<a name="line.8201"></a> |
| <span class="sourceLineNo">8202</span> throw new IOException(e);<a name="line.8202"></a> |
| <span class="sourceLineNo">8203</span> }<a name="line.8203"></a> |
| <span class="sourceLineNo">8204</span> }<a name="line.8204"></a> |
| <span class="sourceLineNo">8205</span><a name="line.8205"></a> |
| <span class="sourceLineNo">8206</span> @Override<a name="line.8206"></a> |
| <span class="sourceLineNo">8207</span> public void closeRegionOperation() throws IOException {<a name="line.8207"></a> |
| <span class="sourceLineNo">8208</span> closeRegionOperation(Operation.ANY);<a name="line.8208"></a> |
| <span class="sourceLineNo">8209</span> }<a name="line.8209"></a> |
| <span class="sourceLineNo">8210</span><a name="line.8210"></a> |
| <span class="sourceLineNo">8211</span> @Override<a name="line.8211"></a> |
| <span class="sourceLineNo">8212</span> public void closeRegionOperation(Operation operation) throws IOException {<a name="line.8212"></a> |
| <span class="sourceLineNo">8213</span> if (operation == Operation.SNAPSHOT) {<a name="line.8213"></a> |
| <span class="sourceLineNo">8214</span> stores.values().forEach(HStore::postSnapshotOperation);<a name="line.8214"></a> |
| <span class="sourceLineNo">8215</span> }<a name="line.8215"></a> |
| <span class="sourceLineNo">8216</span> Thread thisThread = Thread.currentThread();<a name="line.8216"></a> |
| <span class="sourceLineNo">8217</span> regionLockHolders.remove(thisThread);<a name="line.8217"></a> |
| <span class="sourceLineNo">8218</span> lock.readLock().unlock();<a name="line.8218"></a> |
| <span class="sourceLineNo">8219</span> if (coprocessorHost != null) {<a name="line.8219"></a> |
| <span class="sourceLineNo">8220</span> coprocessorHost.postCloseRegionOperation(operation);<a name="line.8220"></a> |
| <span class="sourceLineNo">8221</span> }<a name="line.8221"></a> |
| <span class="sourceLineNo">8222</span> }<a name="line.8222"></a> |
| <span class="sourceLineNo">8223</span><a name="line.8223"></a> |
| <span class="sourceLineNo">8224</span> /**<a name="line.8224"></a> |
| <span class="sourceLineNo">8225</span> * This method needs to be called before any public call that reads or<a name="line.8225"></a> |
| <span class="sourceLineNo">8226</span> * modifies stores in bulk. It has to be called just before a try.<a name="line.8226"></a> |
| <span class="sourceLineNo">8227</span> * #closeBulkRegionOperation needs to be called in the try's finally block<a name="line.8227"></a> |
| <span class="sourceLineNo">8228</span> * Acquires a writelock and checks if the region is closing or closed.<a name="line.8228"></a> |
| <span class="sourceLineNo">8229</span> * @throws NotServingRegionException when the region is closing or closed<a name="line.8229"></a> |
| <span class="sourceLineNo">8230</span> * @throws RegionTooBusyException if failed to get the lock in time<a name="line.8230"></a> |
| <span class="sourceLineNo">8231</span> * @throws InterruptedIOException if interrupted while waiting for a lock<a name="line.8231"></a> |
| <span class="sourceLineNo">8232</span> */<a name="line.8232"></a> |
| <span class="sourceLineNo">8233</span> private void startBulkRegionOperation(boolean writeLockNeeded) throws IOException {<a name="line.8233"></a> |
| <span class="sourceLineNo">8234</span> if (this.closing.get()) {<a name="line.8234"></a> |
| <span class="sourceLineNo">8235</span> throw new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " is closing");<a name="line.8235"></a> |
| <span class="sourceLineNo">8236</span> }<a name="line.8236"></a> |
| <span class="sourceLineNo">8237</span> if (writeLockNeeded) lock(lock.writeLock());<a name="line.8237"></a> |
| <span class="sourceLineNo">8238</span> else lock(lock.readLock());<a name="line.8238"></a> |
| <span class="sourceLineNo">8239</span> if (this.closed.get()) {<a name="line.8239"></a> |
| <span class="sourceLineNo">8240</span> if (writeLockNeeded) lock.writeLock().unlock();<a name="line.8240"></a> |
| <span class="sourceLineNo">8241</span> else lock.readLock().unlock();<a name="line.8241"></a> |
| <span class="sourceLineNo">8242</span> throw new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " is closed");<a name="line.8242"></a> |
| <span class="sourceLineNo">8243</span> }<a name="line.8243"></a> |
| <span class="sourceLineNo">8244</span> regionLockHolders.put(Thread.currentThread(), true);<a name="line.8244"></a> |
| <span class="sourceLineNo">8245</span> }<a name="line.8245"></a> |
| <span class="sourceLineNo">8246</span><a name="line.8246"></a> |
| <span class="sourceLineNo">8247</span> /**<a name="line.8247"></a> |
| <span class="sourceLineNo">8248</span> * Closes the lock. This needs to be called in the finally block corresponding<a name="line.8248"></a> |
| <span class="sourceLineNo">8249</span> * to the try block of #startRegionOperation<a name="line.8249"></a> |
| <span class="sourceLineNo">8250</span> */<a name="line.8250"></a> |
| <span class="sourceLineNo">8251</span> private void closeBulkRegionOperation(){<a name="line.8251"></a> |
| <span class="sourceLineNo">8252</span> regionLockHolders.remove(Thread.currentThread());<a name="line.8252"></a> |
| <span class="sourceLineNo">8253</span> if (lock.writeLock().isHeldByCurrentThread()) lock.writeLock().unlock();<a name="line.8253"></a> |
| <span class="sourceLineNo">8254</span> else lock.readLock().unlock();<a name="line.8254"></a> |
| <span class="sourceLineNo">8255</span> }<a name="line.8255"></a> |
| <span class="sourceLineNo">8256</span><a name="line.8256"></a> |
| <span class="sourceLineNo">8257</span> /**<a name="line.8257"></a> |
| <span class="sourceLineNo">8258</span> * Update LongAdders for number of puts without wal and the size of possible data loss.<a name="line.8258"></a> |
| <span class="sourceLineNo">8259</span> * These information are exposed by the region server metrics.<a name="line.8259"></a> |
| <span class="sourceLineNo">8260</span> */<a name="line.8260"></a> |
| <span class="sourceLineNo">8261</span> private void recordMutationWithoutWal(final Map<byte [], List<Cell>> familyMap) {<a name="line.8261"></a> |
| <span class="sourceLineNo">8262</span> numMutationsWithoutWAL.increment();<a name="line.8262"></a> |
| <span class="sourceLineNo">8263</span> if (numMutationsWithoutWAL.sum() <= 1) {<a name="line.8263"></a> |
| <span class="sourceLineNo">8264</span> LOG.info("writing data to region " + this +<a name="line.8264"></a> |
| <span class="sourceLineNo">8265</span> " with WAL disabled. Data may be lost in the event of a crash.");<a name="line.8265"></a> |
| <span class="sourceLineNo">8266</span> }<a name="line.8266"></a> |
| <span class="sourceLineNo">8267</span><a name="line.8267"></a> |
| <span class="sourceLineNo">8268</span> long mutationSize = 0;<a name="line.8268"></a> |
| <span class="sourceLineNo">8269</span> for (List<Cell> cells: familyMap.values()) {<a name="line.8269"></a> |
| <span class="sourceLineNo">8270</span> // Optimization: 'foreach' loop is not used. See:<a name="line.8270"></a> |
| <span class="sourceLineNo">8271</span> // HBASE-12023 HRegion.applyFamilyMapToMemstore creates too many iterator objects<a name="line.8271"></a> |
| <span class="sourceLineNo">8272</span> assert cells instanceof RandomAccess;<a name="line.8272"></a> |
| <span class="sourceLineNo">8273</span> int listSize = cells.size();<a name="line.8273"></a> |
| <span class="sourceLineNo">8274</span> for (int i=0; i < listSize; i++) {<a name="line.8274"></a> |
| <span class="sourceLineNo">8275</span> Cell cell = cells.get(i);<a name="line.8275"></a> |
| <span class="sourceLineNo">8276</span> mutationSize += cell.getSerializedSize();<a name="line.8276"></a> |
| <span class="sourceLineNo">8277</span> }<a name="line.8277"></a> |
| <span class="sourceLineNo">8278</span> }<a name="line.8278"></a> |
| <span class="sourceLineNo">8279</span><a name="line.8279"></a> |
| <span class="sourceLineNo">8280</span> dataInMemoryWithoutWAL.add(mutationSize);<a name="line.8280"></a> |
| <span class="sourceLineNo">8281</span> }<a name="line.8281"></a> |
| <span class="sourceLineNo">8282</span><a name="line.8282"></a> |
| <span class="sourceLineNo">8283</span> private void lock(final Lock lock) throws IOException {<a name="line.8283"></a> |
| <span class="sourceLineNo">8284</span> lock(lock, 1);<a name="line.8284"></a> |
| <span class="sourceLineNo">8285</span> }<a name="line.8285"></a> |
| <span class="sourceLineNo">8286</span><a name="line.8286"></a> |
| <span class="sourceLineNo">8287</span> /**<a name="line.8287"></a> |
| <span class="sourceLineNo">8288</span> * Try to acquire a lock. Throw RegionTooBusyException<a name="line.8288"></a> |
| <span class="sourceLineNo">8289</span> * if failed to get the lock in time. Throw InterruptedIOException<a name="line.8289"></a> |
| <span class="sourceLineNo">8290</span> * if interrupted while waiting for the lock.<a name="line.8290"></a> |
| <span class="sourceLineNo">8291</span> */<a name="line.8291"></a> |
| <span class="sourceLineNo">8292</span> private void lock(final Lock lock, final int multiplier) throws IOException {<a name="line.8292"></a> |
| <span class="sourceLineNo">8293</span> try {<a name="line.8293"></a> |
| <span class="sourceLineNo">8294</span> final long waitTime = Math.min(maxBusyWaitDuration,<a name="line.8294"></a> |
| <span class="sourceLineNo">8295</span> busyWaitDuration * Math.min(multiplier, maxBusyWaitMultiplier));<a name="line.8295"></a> |
| <span class="sourceLineNo">8296</span> if (!lock.tryLock(waitTime, TimeUnit.MILLISECONDS)) {<a name="line.8296"></a> |
| <span class="sourceLineNo">8297</span> // Don't print millis. Message is used as a key over in<a name="line.8297"></a> |
| <span class="sourceLineNo">8298</span> // RetriesExhaustedWithDetailsException processing.<a name="line.8298"></a> |
| <span class="sourceLineNo">8299</span> final String regionName =<a name="line.8299"></a> |
| <span class="sourceLineNo">8300</span> this.getRegionInfo() == null ? "unknown" : this.getRegionInfo().getRegionNameAsString();<a name="line.8300"></a> |
| <span class="sourceLineNo">8301</span> final String serverName = this.getRegionServerServices() == null ?<a name="line.8301"></a> |
| <span class="sourceLineNo">8302</span> "unknown" : (this.getRegionServerServices().getServerName() == null ?<a name="line.8302"></a> |
| <span class="sourceLineNo">8303</span> "unknown" : this.getRegionServerServices().getServerName().toString());<a name="line.8303"></a> |
| <span class="sourceLineNo">8304</span> RegionTooBusyException rtbe = new RegionTooBusyException(<a name="line.8304"></a> |
| <span class="sourceLineNo">8305</span> "Failed to obtain lock; regionName=" + regionName + ", server=" + serverName);<a name="line.8305"></a> |
| <span class="sourceLineNo">8306</span> LOG.warn("Region is too busy to allow lock acquisition.", rtbe);<a name="line.8306"></a> |
| <span class="sourceLineNo">8307</span> throw rtbe;<a name="line.8307"></a> |
| <span class="sourceLineNo">8308</span> }<a name="line.8308"></a> |
| <span class="sourceLineNo">8309</span> } catch (InterruptedException ie) {<a name="line.8309"></a> |
| <span class="sourceLineNo">8310</span> if (LOG.isDebugEnabled()) {<a name="line.8310"></a> |
| <span class="sourceLineNo">8311</span> LOG.debug("Interrupted while waiting for a lock in region {}", this);<a name="line.8311"></a> |
| <span class="sourceLineNo">8312</span> }<a name="line.8312"></a> |
| <span class="sourceLineNo">8313</span> throw throwOnInterrupt(ie);<a name="line.8313"></a> |
| <span class="sourceLineNo">8314</span> }<a name="line.8314"></a> |
| <span class="sourceLineNo">8315</span> }<a name="line.8315"></a> |
| <span class="sourceLineNo">8316</span><a name="line.8316"></a> |
| <span class="sourceLineNo">8317</span> /**<a name="line.8317"></a> |
| <span class="sourceLineNo">8318</span> * Calls sync with the given transaction ID<a name="line.8318"></a> |
| <span class="sourceLineNo">8319</span> * @param txid should sync up to which transaction<a name="line.8319"></a> |
| <span class="sourceLineNo">8320</span> * @throws IOException If anything goes wrong with DFS<a name="line.8320"></a> |
| <span class="sourceLineNo">8321</span> */<a name="line.8321"></a> |
| <span class="sourceLineNo">8322</span> private void sync(long txid, Durability durability) throws IOException {<a name="line.8322"></a> |
| <span class="sourceLineNo">8323</span> if (this.getRegionInfo().isMetaRegion()) {<a name="line.8323"></a> |
| <span class="sourceLineNo">8324</span> this.wal.sync(txid);<a name="line.8324"></a> |
| <span class="sourceLineNo">8325</span> } else {<a name="line.8325"></a> |
| <span class="sourceLineNo">8326</span> switch(durability) {<a name="line.8326"></a> |
| <span class="sourceLineNo">8327</span> case USE_DEFAULT:<a name="line.8327"></a> |
| <span class="sourceLineNo">8328</span> // do what table defaults to<a name="line.8328"></a> |
| <span class="sourceLineNo">8329</span> if (shouldSyncWAL()) {<a name="line.8329"></a> |
| <span class="sourceLineNo">8330</span> this.wal.sync(txid);<a name="line.8330"></a> |
| <span class="sourceLineNo">8331</span> }<a name="line.8331"></a> |
| <span class="sourceLineNo">8332</span> break;<a name="line.8332"></a> |
| <span class="sourceLineNo">8333</span> case SKIP_WAL:<a name="line.8333"></a> |
| <span class="sourceLineNo">8334</span> // nothing do to<a name="line.8334"></a> |
| <span class="sourceLineNo">8335</span> break;<a name="line.8335"></a> |
| <span class="sourceLineNo">8336</span> case ASYNC_WAL:<a name="line.8336"></a> |
| <span class="sourceLineNo">8337</span> // nothing do to<a name="line.8337"></a> |
| <span class="sourceLineNo">8338</span> break;<a name="line.8338"></a> |
| <span class="sourceLineNo">8339</span> case SYNC_WAL:<a name="line.8339"></a> |
| <span class="sourceLineNo">8340</span> this.wal.sync(txid, false);<a name="line.8340"></a> |
| <span class="sourceLineNo">8341</span> break;<a name="line.8341"></a> |
| <span class="sourceLineNo">8342</span> case FSYNC_WAL:<a name="line.8342"></a> |
| <span class="sourceLineNo">8343</span> this.wal.sync(txid, true);<a name="line.8343"></a> |
| <span class="sourceLineNo">8344</span> break;<a name="line.8344"></a> |
| <span class="sourceLineNo">8345</span> default:<a name="line.8345"></a> |
| <span class="sourceLineNo">8346</span> throw new RuntimeException("Unknown durability " + durability);<a name="line.8346"></a> |
| <span class="sourceLineNo">8347</span> }<a name="line.8347"></a> |
| <span class="sourceLineNo">8348</span> }<a name="line.8348"></a> |
| <span class="sourceLineNo">8349</span> }<a name="line.8349"></a> |
| <span class="sourceLineNo">8350</span><a name="line.8350"></a> |
| <span class="sourceLineNo">8351</span> /**<a name="line.8351"></a> |
| <span class="sourceLineNo">8352</span> * Check whether we should sync the wal from the table's durability settings<a name="line.8352"></a> |
| <span class="sourceLineNo">8353</span> */<a name="line.8353"></a> |
| <span class="sourceLineNo">8354</span> private boolean shouldSyncWAL() {<a name="line.8354"></a> |
| <span class="sourceLineNo">8355</span> return regionDurability.ordinal() > Durability.ASYNC_WAL.ordinal();<a name="line.8355"></a> |
| <span class="sourceLineNo">8356</span> }<a name="line.8356"></a> |
| <span class="sourceLineNo">8357</span><a name="line.8357"></a> |
| <span class="sourceLineNo">8358</span> /** @return the latest sequence number that was read from storage when this region was opened */<a name="line.8358"></a> |
| <span class="sourceLineNo">8359</span> public long getOpenSeqNum() {<a name="line.8359"></a> |
| <span class="sourceLineNo">8360</span> return this.openSeqNum;<a name="line.8360"></a> |
| <span class="sourceLineNo">8361</span> }<a name="line.8361"></a> |
| <span class="sourceLineNo">8362</span><a name="line.8362"></a> |
| <span class="sourceLineNo">8363</span> @Override<a name="line.8363"></a> |
| <span class="sourceLineNo">8364</span> public Map<byte[], Long> getMaxStoreSeqId() {<a name="line.8364"></a> |
| <span class="sourceLineNo">8365</span> return this.maxSeqIdInStores;<a name="line.8365"></a> |
| <span class="sourceLineNo">8366</span> }<a name="line.8366"></a> |
| <span class="sourceLineNo">8367</span><a name="line.8367"></a> |
| <span class="sourceLineNo">8368</span> public long getOldestSeqIdOfStore(byte[] familyName) {<a name="line.8368"></a> |
| <span class="sourceLineNo">8369</span> return wal.getEarliestMemStoreSeqNum(getRegionInfo().getEncodedNameAsBytes(), familyName);<a name="line.8369"></a> |
| <span class="sourceLineNo">8370</span> }<a name="line.8370"></a> |
| <span class="sourceLineNo">8371</span><a name="line.8371"></a> |
| <span class="sourceLineNo">8372</span> @Override<a name="line.8372"></a> |
| <span class="sourceLineNo">8373</span> public CompactionState getCompactionState() {<a name="line.8373"></a> |
| <span class="sourceLineNo">8374</span> boolean hasMajor = majorInProgress.get() > 0, hasMinor = minorInProgress.get() > 0;<a name="line.8374"></a> |
| <span class="sourceLineNo">8375</span> return (hasMajor ? (hasMinor ? CompactionState.MAJOR_AND_MINOR : CompactionState.MAJOR)<a name="line.8375"></a> |
| <span class="sourceLineNo">8376</span> : (hasMinor ? CompactionState.MINOR : CompactionState.NONE));<a name="line.8376"></a> |
| <span class="sourceLineNo">8377</span> }<a name="line.8377"></a> |
| <span class="sourceLineNo">8378</span><a name="line.8378"></a> |
| <span class="sourceLineNo">8379</span> public void reportCompactionRequestStart(boolean isMajor){<a name="line.8379"></a> |
| <span class="sourceLineNo">8380</span> (isMajor ? majorInProgress : minorInProgress).incrementAndGet();<a name="line.8380"></a> |
| <span class="sourceLineNo">8381</span> }<a name="line.8381"></a> |
| <span class="sourceLineNo">8382</span><a name="line.8382"></a> |
| <span class="sourceLineNo">8383</span> public void reportCompactionRequestEnd(boolean isMajor, int numFiles, long filesSizeCompacted) {<a name="line.8383"></a> |
| <span class="sourceLineNo">8384</span> int newValue = (isMajor ? majorInProgress : minorInProgress).decrementAndGet();<a name="line.8384"></a> |
| <span class="sourceLineNo">8385</span><a name="line.8385"></a> |
| <span class="sourceLineNo">8386</span> // metrics<a name="line.8386"></a> |
| <span class="sourceLineNo">8387</span> compactionsFinished.increment();<a name="line.8387"></a> |
| <span class="sourceLineNo">8388</span> compactionNumFilesCompacted.add(numFiles);<a name="line.8388"></a> |
| <span class="sourceLineNo">8389</span> compactionNumBytesCompacted.add(filesSizeCompacted);<a name="line.8389"></a> |
| <span class="sourceLineNo">8390</span><a name="line.8390"></a> |
| <span class="sourceLineNo">8391</span> assert newValue >= 0;<a name="line.8391"></a> |
| <span class="sourceLineNo">8392</span> }<a name="line.8392"></a> |
| <span class="sourceLineNo">8393</span><a name="line.8393"></a> |
| <span class="sourceLineNo">8394</span> public void reportCompactionRequestFailure() {<a name="line.8394"></a> |
| <span class="sourceLineNo">8395</span> compactionsFailed.increment();<a name="line.8395"></a> |
| <span class="sourceLineNo">8396</span> }<a name="line.8396"></a> |
| <span class="sourceLineNo">8397</span><a name="line.8397"></a> |
| <span class="sourceLineNo">8398</span> public void incrementCompactionsQueuedCount() {<a name="line.8398"></a> |
| <span class="sourceLineNo">8399</span> compactionsQueued.increment();<a name="line.8399"></a> |
| <span class="sourceLineNo">8400</span> }<a name="line.8400"></a> |
| <span class="sourceLineNo">8401</span><a name="line.8401"></a> |
| <span class="sourceLineNo">8402</span> public void decrementCompactionsQueuedCount() {<a name="line.8402"></a> |
| <span class="sourceLineNo">8403</span> compactionsQueued.decrement();<a name="line.8403"></a> |
| <span class="sourceLineNo">8404</span> }<a name="line.8404"></a> |
| <span class="sourceLineNo">8405</span><a name="line.8405"></a> |
| <span class="sourceLineNo">8406</span> public void incrementFlushesQueuedCount() {<a name="line.8406"></a> |
| <span class="sourceLineNo">8407</span> flushesQueued.increment();<a name="line.8407"></a> |
| <span class="sourceLineNo">8408</span> }<a name="line.8408"></a> |
| <span class="sourceLineNo">8409</span><a name="line.8409"></a> |
| <span class="sourceLineNo">8410</span> /**<a name="line.8410"></a> |
| <span class="sourceLineNo">8411</span> * If a handler thread is eligible for interrupt, make it ineligible. Should be paired<a name="line.8411"></a> |
| <span class="sourceLineNo">8412</span> * with {{@link #enableInterrupts()}.<a name="line.8412"></a> |
| <span class="sourceLineNo">8413</span> */<a name="line.8413"></a> |
| <span class="sourceLineNo">8414</span> void disableInterrupts() {<a name="line.8414"></a> |
| <span class="sourceLineNo">8415</span> regionLockHolders.computeIfPresent(Thread.currentThread(), (t,b) -> false);<a name="line.8415"></a> |
| <span class="sourceLineNo">8416</span> }<a name="line.8416"></a> |
| <span class="sourceLineNo">8417</span><a name="line.8417"></a> |
| <span class="sourceLineNo">8418</span> /**<a name="line.8418"></a> |
| <span class="sourceLineNo">8419</span> * If a handler thread was made ineligible for interrupt via {{@link #disableInterrupts()},<a name="line.8419"></a> |
| <span class="sourceLineNo">8420</span> * make it eligible again. No-op if interrupts are already enabled.<a name="line.8420"></a> |
| <span class="sourceLineNo">8421</span> */<a name="line.8421"></a> |
| <span class="sourceLineNo">8422</span> void enableInterrupts() {<a name="line.8422"></a> |
| <span class="sourceLineNo">8423</span> regionLockHolders.computeIfPresent(Thread.currentThread(), (t,b) -> true);<a name="line.8423"></a> |
| <span class="sourceLineNo">8424</span> }<a name="line.8424"></a> |
| <span class="sourceLineNo">8425</span><a name="line.8425"></a> |
| <span class="sourceLineNo">8426</span> /**<a name="line.8426"></a> |
| <span class="sourceLineNo">8427</span> * Interrupt any region options that have acquired the region lock via<a name="line.8427"></a> |
| <span class="sourceLineNo">8428</span> * {@link #startRegionOperation(org.apache.hadoop.hbase.regionserver.Region.Operation)},<a name="line.8428"></a> |
| <span class="sourceLineNo">8429</span> * or {@link #startBulkRegionOperation(boolean)}.<a name="line.8429"></a> |
| <span class="sourceLineNo">8430</span> */<a name="line.8430"></a> |
| <span class="sourceLineNo">8431</span> private void interruptRegionOperations() {<a name="line.8431"></a> |
| <span class="sourceLineNo">8432</span> for (Map.Entry<Thread, Boolean> entry: regionLockHolders.entrySet()) {<a name="line.8432"></a> |
| <span class="sourceLineNo">8433</span> // An entry in this map will have a boolean value indicating if it is currently<a name="line.8433"></a> |
| <span class="sourceLineNo">8434</span> // eligible for interrupt; if so, we should interrupt it.<a name="line.8434"></a> |
| <span class="sourceLineNo">8435</span> if (entry.getValue().booleanValue()) {<a name="line.8435"></a> |
| <span class="sourceLineNo">8436</span> entry.getKey().interrupt();<a name="line.8436"></a> |
| <span class="sourceLineNo">8437</span> }<a name="line.8437"></a> |
| <span class="sourceLineNo">8438</span> }<a name="line.8438"></a> |
| <span class="sourceLineNo">8439</span> }<a name="line.8439"></a> |
| <span class="sourceLineNo">8440</span><a name="line.8440"></a> |
| <span class="sourceLineNo">8441</span> /**<a name="line.8441"></a> |
| <span class="sourceLineNo">8442</span> * Check thread interrupt status and throw an exception if interrupted.<a name="line.8442"></a> |
| <span class="sourceLineNo">8443</span> * @throws NotServingRegionException if region is closing<a name="line.8443"></a> |
| <span class="sourceLineNo">8444</span> * @throws InterruptedIOException if interrupted but region is not closing<a name="line.8444"></a> |
| <span class="sourceLineNo">8445</span> */<a name="line.8445"></a> |
| <span class="sourceLineNo">8446</span> // Package scope for tests<a name="line.8446"></a> |
| <span class="sourceLineNo">8447</span> void checkInterrupt() throws NotServingRegionException, InterruptedIOException {<a name="line.8447"></a> |
| <span class="sourceLineNo">8448</span> if (Thread.interrupted()) {<a name="line.8448"></a> |
| <span class="sourceLineNo">8449</span> if (this.closing.get()) {<a name="line.8449"></a> |
| <span class="sourceLineNo">8450</span> throw new NotServingRegionException(<a name="line.8450"></a> |
| <span class="sourceLineNo">8451</span> getRegionInfo().getRegionNameAsString() + " is closing");<a name="line.8451"></a> |
| <span class="sourceLineNo">8452</span> }<a name="line.8452"></a> |
| <span class="sourceLineNo">8453</span> throw new InterruptedIOException();<a name="line.8453"></a> |
| <span class="sourceLineNo">8454</span> }<a name="line.8454"></a> |
| <span class="sourceLineNo">8455</span> }<a name="line.8455"></a> |
| <span class="sourceLineNo">8456</span><a name="line.8456"></a> |
| <span class="sourceLineNo">8457</span> /**<a name="line.8457"></a> |
| <span class="sourceLineNo">8458</span> * Throw the correct exception upon interrupt<a name="line.8458"></a> |
| <span class="sourceLineNo">8459</span> * @param t cause<a name="line.8459"></a> |
| <span class="sourceLineNo">8460</span> */<a name="line.8460"></a> |
| <span class="sourceLineNo">8461</span> // Package scope for tests<a name="line.8461"></a> |
| <span class="sourceLineNo">8462</span> IOException throwOnInterrupt(Throwable t) {<a name="line.8462"></a> |
| <span class="sourceLineNo">8463</span> if (this.closing.get()) {<a name="line.8463"></a> |
| <span class="sourceLineNo">8464</span> return (NotServingRegionException) new NotServingRegionException(<a name="line.8464"></a> |
| <span class="sourceLineNo">8465</span> getRegionInfo().getRegionNameAsString() + " is closing")<a name="line.8465"></a> |
| <span class="sourceLineNo">8466</span> .initCause(t);<a name="line.8466"></a> |
| <span class="sourceLineNo">8467</span> }<a name="line.8467"></a> |
| <span class="sourceLineNo">8468</span> return (InterruptedIOException) new InterruptedIOException().initCause(t);<a name="line.8468"></a> |
| <span class="sourceLineNo">8469</span> }<a name="line.8469"></a> |
| <span class="sourceLineNo">8470</span><a name="line.8470"></a> |
| <span class="sourceLineNo">8471</span> /**<a name="line.8471"></a> |
| <span class="sourceLineNo">8472</span> * {@inheritDoc}<a name="line.8472"></a> |
| <span class="sourceLineNo">8473</span> */<a name="line.8473"></a> |
| <span class="sourceLineNo">8474</span> @Override<a name="line.8474"></a> |
| <span class="sourceLineNo">8475</span> public void onConfigurationChange(Configuration conf) {<a name="line.8475"></a> |
| <span class="sourceLineNo">8476</span> this.storeHotnessProtector.update(conf);<a name="line.8476"></a> |
| <span class="sourceLineNo">8477</span> }<a name="line.8477"></a> |
| <span class="sourceLineNo">8478</span><a name="line.8478"></a> |
| <span class="sourceLineNo">8479</span> /**<a name="line.8479"></a> |
| <span class="sourceLineNo">8480</span> * {@inheritDoc}<a name="line.8480"></a> |
| <span class="sourceLineNo">8481</span> */<a name="line.8481"></a> |
| <span class="sourceLineNo">8482</span> @Override<a name="line.8482"></a> |
| <span class="sourceLineNo">8483</span> public void registerChildren(ConfigurationManager manager) {<a name="line.8483"></a> |
| <span class="sourceLineNo">8484</span> configurationManager = manager;<a name="line.8484"></a> |
| <span class="sourceLineNo">8485</span> stores.values().forEach(manager::registerObserver);<a name="line.8485"></a> |
| <span class="sourceLineNo">8486</span> }<a name="line.8486"></a> |
| <span class="sourceLineNo">8487</span><a name="line.8487"></a> |
| <span class="sourceLineNo">8488</span> /**<a name="line.8488"></a> |
| <span class="sourceLineNo">8489</span> * {@inheritDoc}<a name="line.8489"></a> |
| <span class="sourceLineNo">8490</span> */<a name="line.8490"></a> |
| <span class="sourceLineNo">8491</span> @Override<a name="line.8491"></a> |
| <span class="sourceLineNo">8492</span> public void deregisterChildren(ConfigurationManager manager) {<a name="line.8492"></a> |
| <span class="sourceLineNo">8493</span> stores.values().forEach(configurationManager::deregisterObserver);<a name="line.8493"></a> |
| <span class="sourceLineNo">8494</span> }<a name="line.8494"></a> |
| <span class="sourceLineNo">8495</span><a name="line.8495"></a> |
| <span class="sourceLineNo">8496</span> @Override<a name="line.8496"></a> |
| <span class="sourceLineNo">8497</span> public CellComparator getCellComparator() {<a name="line.8497"></a> |
| <span class="sourceLineNo">8498</span> return cellComparator;<a name="line.8498"></a> |
| <span class="sourceLineNo">8499</span> }<a name="line.8499"></a> |
| <span class="sourceLineNo">8500</span><a name="line.8500"></a> |
| <span class="sourceLineNo">8501</span> public long getMemStoreFlushSize() {<a name="line.8501"></a> |
| <span class="sourceLineNo">8502</span> return this.memstoreFlushSize;<a name="line.8502"></a> |
| <span class="sourceLineNo">8503</span> }<a name="line.8503"></a> |
| <span class="sourceLineNo">8504</span><a name="line.8504"></a> |
| <span class="sourceLineNo">8505</span><a name="line.8505"></a> |
| <span class="sourceLineNo">8506</span> //// method for debugging tests<a name="line.8506"></a> |
| <span class="sourceLineNo">8507</span> void throwException(String title, String regionName) {<a name="line.8507"></a> |
| <span class="sourceLineNo">8508</span> StringBuilder buf = new StringBuilder();<a name="line.8508"></a> |
| <span class="sourceLineNo">8509</span> buf.append(title + ", ");<a name="line.8509"></a> |
| <span class="sourceLineNo">8510</span> buf.append(getRegionInfo().toString());<a name="line.8510"></a> |
| <span class="sourceLineNo">8511</span> buf.append(getRegionInfo().isMetaRegion() ? " meta region " : " ");<a name="line.8511"></a> |
| <span class="sourceLineNo">8512</span> buf.append("stores: ");<a name="line.8512"></a> |
| <span class="sourceLineNo">8513</span> for (HStore s : stores.values()) {<a name="line.8513"></a> |
| <span class="sourceLineNo">8514</span> buf.append(s.getColumnFamilyDescriptor().getNameAsString());<a name="line.8514"></a> |
| <span class="sourceLineNo">8515</span> buf.append(" size: ");<a name="line.8515"></a> |
| <span class="sourceLineNo">8516</span> buf.append(s.getMemStoreSize().getDataSize());<a name="line.8516"></a> |
| <span class="sourceLineNo">8517</span> buf.append(" ");<a name="line.8517"></a> |
| <span class="sourceLineNo">8518</span> }<a name="line.8518"></a> |
| <span class="sourceLineNo">8519</span> buf.append("end-of-stores");<a name="line.8519"></a> |
| <span class="sourceLineNo">8520</span> buf.append(", memstore size ");<a name="line.8520"></a> |
| <span class="sourceLineNo">8521</span> buf.append(getMemStoreDataSize());<a name="line.8521"></a> |
| <span class="sourceLineNo">8522</span> if (getRegionInfo().getRegionNameAsString().startsWith(regionName)) {<a name="line.8522"></a> |
| <span class="sourceLineNo">8523</span> throw new RuntimeException(buf.toString());<a name="line.8523"></a> |
| <span class="sourceLineNo">8524</span> }<a name="line.8524"></a> |
| <span class="sourceLineNo">8525</span> }<a name="line.8525"></a> |
| <span class="sourceLineNo">8526</span><a name="line.8526"></a> |
| <span class="sourceLineNo">8527</span> @Override<a name="line.8527"></a> |
| <span class="sourceLineNo">8528</span> public void requestCompaction(String why, int priority, boolean major,<a name="line.8528"></a> |
| <span class="sourceLineNo">8529</span> CompactionLifeCycleTracker tracker) throws IOException {<a name="line.8529"></a> |
| <span class="sourceLineNo">8530</span> if (major) {<a name="line.8530"></a> |
| <span class="sourceLineNo">8531</span> stores.values().forEach(HStore::triggerMajorCompaction);<a name="line.8531"></a> |
| <span class="sourceLineNo">8532</span> }<a name="line.8532"></a> |
| <span class="sourceLineNo">8533</span> rsServices.getCompactionRequestor().requestCompaction(this, why, priority, tracker,<a name="line.8533"></a> |
| <span class="sourceLineNo">8534</span> RpcServer.getRequestUser().orElse(null));<a name="line.8534"></a> |
| <span class="sourceLineNo">8535</span> }<a name="line.8535"></a> |
| <span class="sourceLineNo">8536</span><a name="line.8536"></a> |
| <span class="sourceLineNo">8537</span> @Override<a name="line.8537"></a> |
| <span class="sourceLineNo">8538</span> public void requestCompaction(byte[] family, String why, int priority, boolean major,<a name="line.8538"></a> |
| <span class="sourceLineNo">8539</span> CompactionLifeCycleTracker tracker) throws IOException {<a name="line.8539"></a> |
| <span class="sourceLineNo">8540</span> HStore store = stores.get(family);<a name="line.8540"></a> |
| <span class="sourceLineNo">8541</span> if (store == null) {<a name="line.8541"></a> |
| <span class="sourceLineNo">8542</span> throw new NoSuchColumnFamilyException("column family " + Bytes.toString(family) +<a name="line.8542"></a> |
| <span class="sourceLineNo">8543</span> " does not exist in region " + getRegionInfo().getRegionNameAsString());<a name="line.8543"></a> |
| <span class="sourceLineNo">8544</span> }<a name="line.8544"></a> |
| <span class="sourceLineNo">8545</span> if (major) {<a name="line.8545"></a> |
| <span class="sourceLineNo">8546</span> store.triggerMajorCompaction();<a name="line.8546"></a> |
| <span class="sourceLineNo">8547</span> }<a name="line.8547"></a> |
| <span class="sourceLineNo">8548</span> rsServices.getCompactionRequestor().requestCompaction(this, store, why, priority, tracker,<a name="line.8548"></a> |
| <span class="sourceLineNo">8549</span> RpcServer.getRequestUser().orElse(null));<a name="line.8549"></a> |
| <span class="sourceLineNo">8550</span> }<a name="line.8550"></a> |
| <span class="sourceLineNo">8551</span><a name="line.8551"></a> |
| <span class="sourceLineNo">8552</span> private void requestFlushIfNeeded() throws RegionTooBusyException {<a name="line.8552"></a> |
| <span class="sourceLineNo">8553</span> if(isFlushSize(this.memStoreSizing.getMemStoreSize())) {<a name="line.8553"></a> |
| <span class="sourceLineNo">8554</span> requestFlush();<a name="line.8554"></a> |
| <span class="sourceLineNo">8555</span> }<a name="line.8555"></a> |
| <span class="sourceLineNo">8556</span> }<a name="line.8556"></a> |
| <span class="sourceLineNo">8557</span><a name="line.8557"></a> |
| <span class="sourceLineNo">8558</span> private void requestFlush() {<a name="line.8558"></a> |
| <span class="sourceLineNo">8559</span> if (this.rsServices == null) {<a name="line.8559"></a> |
| <span class="sourceLineNo">8560</span> return;<a name="line.8560"></a> |
| <span class="sourceLineNo">8561</span> }<a name="line.8561"></a> |
| <span class="sourceLineNo">8562</span> requestFlush0(FlushLifeCycleTracker.DUMMY);<a name="line.8562"></a> |
| <span class="sourceLineNo">8563</span> }<a name="line.8563"></a> |
| <span class="sourceLineNo">8564</span><a name="line.8564"></a> |
| <span class="sourceLineNo">8565</span> private void requestFlush0(FlushLifeCycleTracker tracker) {<a name="line.8565"></a> |
| <span class="sourceLineNo">8566</span> boolean shouldFlush = false;<a name="line.8566"></a> |
| <span class="sourceLineNo">8567</span> synchronized (writestate) {<a name="line.8567"></a> |
| <span class="sourceLineNo">8568</span> if (!this.writestate.isFlushRequested()) {<a name="line.8568"></a> |
| <span class="sourceLineNo">8569</span> shouldFlush = true;<a name="line.8569"></a> |
| <span class="sourceLineNo">8570</span> writestate.flushRequested = true;<a name="line.8570"></a> |
| <span class="sourceLineNo">8571</span> }<a name="line.8571"></a> |
| <span class="sourceLineNo">8572</span> }<a name="line.8572"></a> |
| <span class="sourceLineNo">8573</span> if (shouldFlush) {<a name="line.8573"></a> |
| <span class="sourceLineNo">8574</span> // Make request outside of synchronize block; HBASE-818.<a name="line.8574"></a> |
| <span class="sourceLineNo">8575</span> this.rsServices.getFlushRequester().requestFlush(this, tracker);<a name="line.8575"></a> |
| <span class="sourceLineNo">8576</span> if (LOG.isDebugEnabled()) {<a name="line.8576"></a> |
| <span class="sourceLineNo">8577</span> LOG.debug("Flush requested on " + this.getRegionInfo().getEncodedName());<a name="line.8577"></a> |
| <span class="sourceLineNo">8578</span> }<a name="line.8578"></a> |
| <span class="sourceLineNo">8579</span> } else {<a name="line.8579"></a> |
| <span class="sourceLineNo">8580</span> tracker.notExecuted("Flush already requested on " + this);<a name="line.8580"></a> |
| <span class="sourceLineNo">8581</span> }<a name="line.8581"></a> |
| <span class="sourceLineNo">8582</span> }<a name="line.8582"></a> |
| <span class="sourceLineNo">8583</span><a name="line.8583"></a> |
| <span class="sourceLineNo">8584</span> @Override<a name="line.8584"></a> |
| <span class="sourceLineNo">8585</span> public void requestFlush(FlushLifeCycleTracker tracker) throws IOException {<a name="line.8585"></a> |
| <span class="sourceLineNo">8586</span> requestFlush0(tracker);<a name="line.8586"></a> |
| <span class="sourceLineNo">8587</span> }<a name="line.8587"></a> |
| <span class="sourceLineNo">8588</span><a name="line.8588"></a> |
| <span class="sourceLineNo">8589</span> /**<a name="line.8589"></a> |
| <span class="sourceLineNo">8590</span> * This method modifies the region's configuration in order to inject replication-related<a name="line.8590"></a> |
| <span class="sourceLineNo">8591</span> * features<a name="line.8591"></a> |
| <span class="sourceLineNo">8592</span> * @param conf region configurations<a name="line.8592"></a> |
| <span class="sourceLineNo">8593</span> */<a name="line.8593"></a> |
| <span class="sourceLineNo">8594</span> private static void decorateRegionConfiguration(Configuration conf) {<a name="line.8594"></a> |
| <span class="sourceLineNo">8595</span> if (ReplicationUtils.isReplicationForBulkLoadDataEnabled(conf)) {<a name="line.8595"></a> |
| <span class="sourceLineNo">8596</span> String plugins = conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,"");<a name="line.8596"></a> |
| <span class="sourceLineNo">8597</span> String replicationCoprocessorClass = ReplicationObserver.class.getCanonicalName();<a name="line.8597"></a> |
| <span class="sourceLineNo">8598</span> if (!plugins.contains(replicationCoprocessorClass)) {<a name="line.8598"></a> |
| <span class="sourceLineNo">8599</span> conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,<a name="line.8599"></a> |
| <span class="sourceLineNo">8600</span> (plugins.equals("") ? "" : (plugins + ",")) + replicationCoprocessorClass);<a name="line.8600"></a> |
| <span class="sourceLineNo">8601</span> }<a name="line.8601"></a> |
| <span class="sourceLineNo">8602</span> }<a name="line.8602"></a> |
| <span class="sourceLineNo">8603</span> }<a name="line.8603"></a> |
| <span class="sourceLineNo">8604</span><a name="line.8604"></a> |
| <span class="sourceLineNo">8605</span> public void addReadRequestsCount(long readRequestsCount) {<a name="line.8605"></a> |
| <span class="sourceLineNo">8606</span> this.readRequestsCount.add(readRequestsCount);<a name="line.8606"></a> |
| <span class="sourceLineNo">8607</span> }<a name="line.8607"></a> |
| <span class="sourceLineNo">8608</span><a name="line.8608"></a> |
| <span class="sourceLineNo">8609</span> public void addWriteRequestsCount(long writeRequestsCount) {<a name="line.8609"></a> |
| <span class="sourceLineNo">8610</span> this.writeRequestsCount.add(writeRequestsCount);<a name="line.8610"></a> |
| <span class="sourceLineNo">8611</span> }<a name="line.8611"></a> |
| <span class="sourceLineNo">8612</span>}<a name="line.8612"></a> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </pre> |
| </div> |
| </body> |
| </html> |
