| <!DOCTYPE HTML> |
| <html lang="en"> |
| <head> |
| <!-- Generated by javadoc (17) --> |
| <title>Source code</title> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <meta name="description" content="source: package: org.apache.hadoop.hbase.regionserver, class: HRegion, class: RowLockContext"> |
| <meta name="generator" content="javadoc/SourceToHTMLConverter"> |
| <link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style"> |
| </head> |
| <body class="source-page"> |
| <main role="main"> |
| <div class="source-container"> |
| <pre><span class="source-line-no">001</span><span id="line-1">/*</span> |
| <span class="source-line-no">002</span><span id="line-2"> * Licensed to the Apache Software Foundation (ASF) under one</span> |
| <span class="source-line-no">003</span><span id="line-3"> * or more contributor license agreements. See the NOTICE file</span> |
| <span class="source-line-no">004</span><span id="line-4"> * distributed with this work for additional information</span> |
| <span class="source-line-no">005</span><span id="line-5"> * regarding copyright ownership. The ASF licenses this file</span> |
| <span class="source-line-no">006</span><span id="line-6"> * to you under the Apache License, Version 2.0 (the</span> |
| <span class="source-line-no">007</span><span id="line-7"> * "License"); you may not use this file except in compliance</span> |
| <span class="source-line-no">008</span><span id="line-8"> * with the License. You may obtain a copy of the License at</span> |
| <span class="source-line-no">009</span><span id="line-9"> *</span> |
| <span class="source-line-no">010</span><span id="line-10"> * http://www.apache.org/licenses/LICENSE-2.0</span> |
| <span class="source-line-no">011</span><span id="line-11"> *</span> |
| <span class="source-line-no">012</span><span id="line-12"> * Unless required by applicable law or agreed to in writing, software</span> |
| <span class="source-line-no">013</span><span id="line-13"> * distributed under the License is distributed on an "AS IS" BASIS,</span> |
| <span class="source-line-no">014</span><span id="line-14"> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> |
| <span class="source-line-no">015</span><span id="line-15"> * See the License for the specific language governing permissions and</span> |
| <span class="source-line-no">016</span><span id="line-16"> * limitations under the License.</span> |
| <span class="source-line-no">017</span><span id="line-17"> */</span> |
| <span class="source-line-no">018</span><span id="line-18">package org.apache.hadoop.hbase.regionserver;</span> |
| <span class="source-line-no">019</span><span id="line-19"></span> |
| <span class="source-line-no">020</span><span id="line-20">import static org.apache.hadoop.hbase.HConstants.REPLICATION_SCOPE_LOCAL;</span> |
| <span class="source-line-no">021</span><span id="line-21">import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY;</span> |
| <span class="source-line-no">022</span><span id="line-22">import static org.apache.hadoop.hbase.trace.HBaseSemanticAttributes.REGION_NAMES_KEY;</span> |
| <span class="source-line-no">023</span><span id="line-23">import static org.apache.hadoop.hbase.trace.HBaseSemanticAttributes.ROW_LOCK_READ_LOCK_KEY;</span> |
| <span class="source-line-no">024</span><span id="line-24">import static org.apache.hadoop.hbase.util.ConcurrentMapUtils.computeIfAbsent;</span> |
| <span class="source-line-no">025</span><span id="line-25"></span> |
| <span class="source-line-no">026</span><span id="line-26">import com.google.errorprone.annotations.RestrictedApi;</span> |
| <span class="source-line-no">027</span><span id="line-27">import edu.umd.cs.findbugs.annotations.Nullable;</span> |
| <span class="source-line-no">028</span><span id="line-28">import io.opentelemetry.api.trace.Span;</span> |
| <span class="source-line-no">029</span><span id="line-29">import java.io.EOFException;</span> |
| <span class="source-line-no">030</span><span id="line-30">import java.io.FileNotFoundException;</span> |
| <span class="source-line-no">031</span><span id="line-31">import java.io.IOException;</span> |
| <span class="source-line-no">032</span><span id="line-32">import java.io.InterruptedIOException;</span> |
| <span class="source-line-no">033</span><span id="line-33">import java.lang.reflect.Constructor;</span> |
| <span class="source-line-no">034</span><span id="line-34">import java.nio.ByteBuffer;</span> |
| <span class="source-line-no">035</span><span id="line-35">import java.nio.charset.StandardCharsets;</span> |
| <span class="source-line-no">036</span><span id="line-36">import java.text.ParseException;</span> |
| <span class="source-line-no">037</span><span id="line-37">import java.util.ArrayList;</span> |
| <span class="source-line-no">038</span><span id="line-38">import java.util.Arrays;</span> |
| <span class="source-line-no">039</span><span id="line-39">import java.util.Collection;</span> |
| <span class="source-line-no">040</span><span id="line-40">import java.util.Collections;</span> |
| <span class="source-line-no">041</span><span id="line-41">import java.util.HashMap;</span> |
| <span class="source-line-no">042</span><span id="line-42">import java.util.HashSet;</span> |
| <span class="source-line-no">043</span><span id="line-43">import java.util.Iterator;</span> |
| <span class="source-line-no">044</span><span id="line-44">import java.util.List;</span> |
| <span class="source-line-no">045</span><span id="line-45">import java.util.Map;</span> |
| <span class="source-line-no">046</span><span id="line-46">import java.util.Map.Entry;</span> |
| <span class="source-line-no">047</span><span id="line-47">import java.util.NavigableMap;</span> |
| <span class="source-line-no">048</span><span id="line-48">import java.util.NavigableSet;</span> |
| <span class="source-line-no">049</span><span id="line-49">import java.util.Objects;</span> |
| <span class="source-line-no">050</span><span id="line-50">import java.util.Optional;</span> |
| <span class="source-line-no">051</span><span id="line-51">import java.util.RandomAccess;</span> |
| <span class="source-line-no">052</span><span id="line-52">import java.util.Set;</span> |
| <span class="source-line-no">053</span><span id="line-53">import java.util.TreeMap;</span> |
| <span class="source-line-no">054</span><span id="line-54">import java.util.UUID;</span> |
| <span class="source-line-no">055</span><span id="line-55">import java.util.concurrent.Callable;</span> |
| <span class="source-line-no">056</span><span id="line-56">import java.util.concurrent.CompletionService;</span> |
| <span class="source-line-no">057</span><span id="line-57">import java.util.concurrent.ConcurrentHashMap;</span> |
| <span class="source-line-no">058</span><span id="line-58">import java.util.concurrent.ConcurrentMap;</span> |
| <span class="source-line-no">059</span><span id="line-59">import java.util.concurrent.ConcurrentSkipListMap;</span> |
| <span class="source-line-no">060</span><span id="line-60">import java.util.concurrent.ExecutionException;</span> |
| <span class="source-line-no">061</span><span id="line-61">import java.util.concurrent.ExecutorCompletionService;</span> |
| <span class="source-line-no">062</span><span id="line-62">import java.util.concurrent.Future;</span> |
| <span class="source-line-no">063</span><span id="line-63">import java.util.concurrent.ThreadFactory;</span> |
| <span class="source-line-no">064</span><span id="line-64">import java.util.concurrent.ThreadPoolExecutor;</span> |
| <span class="source-line-no">065</span><span id="line-65">import java.util.concurrent.TimeUnit;</span> |
| <span class="source-line-no">066</span><span id="line-66">import java.util.concurrent.atomic.AtomicBoolean;</span> |
| <span class="source-line-no">067</span><span id="line-67">import java.util.concurrent.atomic.AtomicInteger;</span> |
| <span class="source-line-no">068</span><span id="line-68">import java.util.concurrent.atomic.LongAdder;</span> |
| <span class="source-line-no">069</span><span id="line-69">import java.util.concurrent.locks.Lock;</span> |
| <span class="source-line-no">070</span><span id="line-70">import java.util.concurrent.locks.ReadWriteLock;</span> |
| <span class="source-line-no">071</span><span id="line-71">import java.util.concurrent.locks.ReentrantLock;</span> |
| <span class="source-line-no">072</span><span id="line-72">import java.util.concurrent.locks.ReentrantReadWriteLock;</span> |
| <span class="source-line-no">073</span><span id="line-73">import java.util.function.Function;</span> |
| <span class="source-line-no">074</span><span id="line-74">import java.util.stream.Collectors;</span> |
| <span class="source-line-no">075</span><span id="line-75">import java.util.stream.Stream;</span> |
| <span class="source-line-no">076</span><span id="line-76">import org.apache.hadoop.conf.Configuration;</span> |
| <span class="source-line-no">077</span><span id="line-77">import org.apache.hadoop.fs.FileStatus;</span> |
| <span class="source-line-no">078</span><span id="line-78">import org.apache.hadoop.fs.FileSystem;</span> |
| <span class="source-line-no">079</span><span id="line-79">import org.apache.hadoop.fs.LocatedFileStatus;</span> |
| <span class="source-line-no">080</span><span id="line-80">import org.apache.hadoop.fs.Path;</span> |
| <span class="source-line-no">081</span><span id="line-81">import org.apache.hadoop.hbase.Cell;</span> |
| <span class="source-line-no">082</span><span id="line-82">import org.apache.hadoop.hbase.CellBuilderType;</span> |
| <span class="source-line-no">083</span><span id="line-83">import org.apache.hadoop.hbase.CellComparator;</span> |
| <span class="source-line-no">084</span><span id="line-84">import org.apache.hadoop.hbase.CellComparatorImpl;</span> |
| <span class="source-line-no">085</span><span id="line-85">import org.apache.hadoop.hbase.CellScanner;</span> |
| <span class="source-line-no">086</span><span id="line-86">import org.apache.hadoop.hbase.CellUtil;</span> |
| <span class="source-line-no">087</span><span id="line-87">import org.apache.hadoop.hbase.CompareOperator;</span> |
| <span class="source-line-no">088</span><span id="line-88">import org.apache.hadoop.hbase.CompoundConfiguration;</span> |
| <span class="source-line-no">089</span><span id="line-89">import org.apache.hadoop.hbase.DoNotRetryIOException;</span> |
| <span class="source-line-no">090</span><span id="line-90">import org.apache.hadoop.hbase.DroppedSnapshotException;</span> |
| <span class="source-line-no">091</span><span id="line-91">import org.apache.hadoop.hbase.ExtendedCell;</span> |
| <span class="source-line-no">092</span><span id="line-92">import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;</span> |
| <span class="source-line-no">093</span><span id="line-93">import org.apache.hadoop.hbase.HConstants;</span> |
| <span class="source-line-no">094</span><span id="line-94">import org.apache.hadoop.hbase.HConstants.OperationStatusCode;</span> |
| <span class="source-line-no">095</span><span id="line-95">import org.apache.hadoop.hbase.HDFSBlocksDistribution;</span> |
| <span class="source-line-no">096</span><span id="line-96">import org.apache.hadoop.hbase.KeyValue;</span> |
| <span class="source-line-no">097</span><span id="line-97">import org.apache.hadoop.hbase.MetaCellComparator;</span> |
| <span class="source-line-no">098</span><span id="line-98">import org.apache.hadoop.hbase.NamespaceDescriptor;</span> |
| <span class="source-line-no">099</span><span id="line-99">import org.apache.hadoop.hbase.NotServingRegionException;</span> |
| <span class="source-line-no">100</span><span id="line-100">import org.apache.hadoop.hbase.PrivateCellUtil;</span> |
| <span class="source-line-no">101</span><span id="line-101">import org.apache.hadoop.hbase.RegionTooBusyException;</span> |
| <span class="source-line-no">102</span><span id="line-102">import org.apache.hadoop.hbase.Tag;</span> |
| <span class="source-line-no">103</span><span id="line-103">import org.apache.hadoop.hbase.TagUtil;</span> |
| <span class="source-line-no">104</span><span id="line-104">import org.apache.hadoop.hbase.client.Append;</span> |
| <span class="source-line-no">105</span><span id="line-105">import org.apache.hadoop.hbase.client.CheckAndMutate;</span> |
| <span class="source-line-no">106</span><span id="line-106">import org.apache.hadoop.hbase.client.CheckAndMutateResult;</span> |
| <span class="source-line-no">107</span><span id="line-107">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;</span> |
| <span class="source-line-no">108</span><span id="line-108">import org.apache.hadoop.hbase.client.CompactionState;</span> |
| <span class="source-line-no">109</span><span id="line-109">import org.apache.hadoop.hbase.client.Delete;</span> |
| <span class="source-line-no">110</span><span id="line-110">import org.apache.hadoop.hbase.client.Durability;</span> |
| <span class="source-line-no">111</span><span id="line-111">import org.apache.hadoop.hbase.client.Get;</span> |
| <span class="source-line-no">112</span><span id="line-112">import org.apache.hadoop.hbase.client.Increment;</span> |
| <span class="source-line-no">113</span><span id="line-113">import org.apache.hadoop.hbase.client.IsolationLevel;</span> |
| <span class="source-line-no">114</span><span id="line-114">import org.apache.hadoop.hbase.client.Mutation;</span> |
| <span class="source-line-no">115</span><span id="line-115">import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor;</span> |
| <span class="source-line-no">116</span><span id="line-116">import org.apache.hadoop.hbase.client.Put;</span> |
| <span class="source-line-no">117</span><span id="line-117">import org.apache.hadoop.hbase.client.RegionInfo;</span> |
| <span class="source-line-no">118</span><span id="line-118">import org.apache.hadoop.hbase.client.RegionReplicaUtil;</span> |
| <span class="source-line-no">119</span><span id="line-119">import org.apache.hadoop.hbase.client.Result;</span> |
| <span class="source-line-no">120</span><span id="line-120">import org.apache.hadoop.hbase.client.Row;</span> |
| <span class="source-line-no">121</span><span id="line-121">import org.apache.hadoop.hbase.client.RowMutations;</span> |
| <span class="source-line-no">122</span><span id="line-122">import org.apache.hadoop.hbase.client.Scan;</span> |
| <span class="source-line-no">123</span><span id="line-123">import org.apache.hadoop.hbase.client.TableDescriptor;</span> |
| <span class="source-line-no">124</span><span id="line-124">import org.apache.hadoop.hbase.client.TableDescriptorBuilder;</span> |
| <span class="source-line-no">125</span><span id="line-125">import org.apache.hadoop.hbase.conf.ConfigurationManager;</span> |
| <span class="source-line-no">126</span><span id="line-126">import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver;</span> |
| <span class="source-line-no">127</span><span id="line-127">import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;</span> |
| <span class="source-line-no">128</span><span id="line-128">import org.apache.hadoop.hbase.coprocessor.ReadOnlyConfiguration;</span> |
| <span class="source-line-no">129</span><span id="line-129">import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;</span> |
| <span class="source-line-no">130</span><span id="line-130">import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;</span> |
| <span class="source-line-no">131</span><span id="line-131">import org.apache.hadoop.hbase.exceptions.TimeoutIOException;</span> |
| <span class="source-line-no">132</span><span id="line-132">import org.apache.hadoop.hbase.exceptions.UnknownProtocolException;</span> |
| <span class="source-line-no">133</span><span id="line-133">import org.apache.hadoop.hbase.filter.BinaryComparator;</span> |
| <span class="source-line-no">134</span><span id="line-134">import org.apache.hadoop.hbase.filter.ByteArrayComparable;</span> |
| <span class="source-line-no">135</span><span id="line-135">import org.apache.hadoop.hbase.filter.Filter;</span> |
| <span class="source-line-no">136</span><span id="line-136">import org.apache.hadoop.hbase.io.HFileLink;</span> |
| <span class="source-line-no">137</span><span id="line-137">import org.apache.hadoop.hbase.io.HeapSize;</span> |
| <span class="source-line-no">138</span><span id="line-138">import org.apache.hadoop.hbase.io.TimeRange;</span> |
| <span class="source-line-no">139</span><span id="line-139">import org.apache.hadoop.hbase.io.hfile.BlockCache;</span> |
| <span class="source-line-no">140</span><span id="line-140">import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache;</span> |
| <span class="source-line-no">141</span><span id="line-141">import org.apache.hadoop.hbase.io.hfile.HFile;</span> |
| <span class="source-line-no">142</span><span id="line-142">import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;</span> |
| <span class="source-line-no">143</span><span id="line-143">import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;</span> |
| <span class="source-line-no">144</span><span id="line-144">import org.apache.hadoop.hbase.ipc.RpcCall;</span> |
| <span class="source-line-no">145</span><span id="line-145">import org.apache.hadoop.hbase.ipc.RpcServer;</span> |
| <span class="source-line-no">146</span><span id="line-146">import org.apache.hadoop.hbase.ipc.ServerCall;</span> |
| <span class="source-line-no">147</span><span id="line-147">import org.apache.hadoop.hbase.mob.MobFileCache;</span> |
| <span class="source-line-no">148</span><span id="line-148">import org.apache.hadoop.hbase.monitoring.MonitoredTask;</span> |
| <span class="source-line-no">149</span><span id="line-149">import org.apache.hadoop.hbase.monitoring.TaskMonitor;</span> |
| <span class="source-line-no">150</span><span id="line-150">import org.apache.hadoop.hbase.quotas.RegionServerSpaceQuotaManager;</span> |
| <span class="source-line-no">151</span><span id="line-151">import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl.WriteEntry;</span> |
| <span class="source-line-no">152</span><span id="line-152">import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;</span> |
| <span class="source-line-no">153</span><span id="line-153">import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;</span> |
| <span class="source-line-no">154</span><span id="line-154">import org.apache.hadoop.hbase.regionserver.compactions.ForbidMajorCompactionChecker;</span> |
| <span class="source-line-no">155</span><span id="line-155">import org.apache.hadoop.hbase.regionserver.metrics.MetricsTableRequests;</span> |
| <span class="source-line-no">156</span><span id="line-156">import org.apache.hadoop.hbase.regionserver.regionreplication.RegionReplicationSink;</span> |
| <span class="source-line-no">157</span><span id="line-157">import org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory;</span> |
| <span class="source-line-no">158</span><span id="line-158">import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;</span> |
| <span class="source-line-no">159</span><span id="line-159">import org.apache.hadoop.hbase.regionserver.throttle.StoreHotnessProtector;</span> |
| <span class="source-line-no">160</span><span id="line-160">import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;</span> |
| <span class="source-line-no">161</span><span id="line-161">import org.apache.hadoop.hbase.regionserver.wal.WALSyncTimeoutIOException;</span> |
| <span class="source-line-no">162</span><span id="line-162">import org.apache.hadoop.hbase.regionserver.wal.WALUtil;</span> |
| <span class="source-line-no">163</span><span id="line-163">import org.apache.hadoop.hbase.replication.ReplicationUtils;</span> |
| <span class="source-line-no">164</span><span id="line-164">import org.apache.hadoop.hbase.replication.regionserver.ReplicationObserver;</span> |
| <span class="source-line-no">165</span><span id="line-165">import org.apache.hadoop.hbase.security.User;</span> |
| <span class="source-line-no">166</span><span id="line-166">import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;</span> |
| <span class="source-line-no">167</span><span id="line-167">import org.apache.hadoop.hbase.snapshot.SnapshotManifest;</span> |
| <span class="source-line-no">168</span><span id="line-168">import org.apache.hadoop.hbase.trace.TraceUtil;</span> |
| <span class="source-line-no">169</span><span id="line-169">import org.apache.hadoop.hbase.util.Bytes;</span> |
| <span class="source-line-no">170</span><span id="line-170">import org.apache.hadoop.hbase.util.CancelableProgressable;</span> |
| <span class="source-line-no">171</span><span id="line-171">import org.apache.hadoop.hbase.util.ClassSize;</span> |
| <span class="source-line-no">172</span><span id="line-172">import org.apache.hadoop.hbase.util.CommonFSUtils;</span> |
| <span class="source-line-no">173</span><span id="line-173">import org.apache.hadoop.hbase.util.CoprocessorConfigurationUtil;</span> |
| <span class="source-line-no">174</span><span id="line-174">import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;</span> |
| <span class="source-line-no">175</span><span id="line-175">import org.apache.hadoop.hbase.util.FSUtils;</span> |
| <span class="source-line-no">176</span><span id="line-176">import org.apache.hadoop.hbase.util.HashedBytes;</span> |
| <span class="source-line-no">177</span><span id="line-177">import org.apache.hadoop.hbase.util.NonceKey;</span> |
| <span class="source-line-no">178</span><span id="line-178">import org.apache.hadoop.hbase.util.Pair;</span> |
| <span class="source-line-no">179</span><span id="line-179">import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;</span> |
| <span class="source-line-no">180</span><span id="line-180">import org.apache.hadoop.hbase.util.TableDescriptorChecker;</span> |
| <span class="source-line-no">181</span><span id="line-181">import org.apache.hadoop.hbase.util.Threads;</span> |
| <span class="source-line-no">182</span><span id="line-182">import org.apache.hadoop.hbase.wal.WAL;</span> |
| <span class="source-line-no">183</span><span id="line-183">import org.apache.hadoop.hbase.wal.WALEdit;</span> |
| <span class="source-line-no">184</span><span id="line-184">import org.apache.hadoop.hbase.wal.WALFactory;</span> |
| <span class="source-line-no">185</span><span id="line-185">import org.apache.hadoop.hbase.wal.WALKey;</span> |
| <span class="source-line-no">186</span><span id="line-186">import org.apache.hadoop.hbase.wal.WALKeyImpl;</span> |
| <span class="source-line-no">187</span><span id="line-187">import org.apache.hadoop.hbase.wal.WALSplitUtil;</span> |
| <span class="source-line-no">188</span><span id="line-188">import org.apache.hadoop.hbase.wal.WALSplitUtil.MutationReplay;</span> |
| <span class="source-line-no">189</span><span id="line-189">import org.apache.hadoop.hbase.wal.WALStreamReader;</span> |
| <span class="source-line-no">190</span><span id="line-190">import org.apache.hadoop.util.StringUtils;</span> |
| <span class="source-line-no">191</span><span id="line-191">import org.apache.yetus.audience.InterfaceAudience;</span> |
| <span class="source-line-no">192</span><span id="line-192">import org.slf4j.Logger;</span> |
| <span class="source-line-no">193</span><span id="line-193">import org.slf4j.LoggerFactory;</span> |
| <span class="source-line-no">194</span><span id="line-194"></span> |
| <span class="source-line-no">195</span><span id="line-195">import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;</span> |
| <span class="source-line-no">196</span><span id="line-196">import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;</span> |
| <span class="source-line-no">197</span><span id="line-197">import org.apache.hbase.thirdparty.com.google.common.collect.Lists;</span> |
| <span class="source-line-no">198</span><span id="line-198">import org.apache.hbase.thirdparty.com.google.common.collect.Maps;</span> |
| <span class="source-line-no">199</span><span id="line-199">import org.apache.hbase.thirdparty.com.google.common.io.Closeables;</span> |
| <span class="source-line-no">200</span><span id="line-200">import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.MethodDescriptor;</span> |
| <span class="source-line-no">201</span><span id="line-201">import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.ServiceDescriptor;</span> |
| <span class="source-line-no">202</span><span id="line-202">import org.apache.hbase.thirdparty.com.google.protobuf.Message;</span> |
| <span class="source-line-no">203</span><span id="line-203">import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;</span> |
| <span class="source-line-no">204</span><span id="line-204">import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;</span> |
| <span class="source-line-no">205</span><span id="line-205">import org.apache.hbase.thirdparty.com.google.protobuf.Service;</span> |
| <span class="source-line-no">206</span><span id="line-206">import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;</span> |
| <span class="source-line-no">207</span><span id="line-207">import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;</span> |
| <span class="source-line-no">208</span><span id="line-208">import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;</span> |
| <span class="source-line-no">209</span><span id="line-209"></span> |
| <span class="source-line-no">210</span><span id="line-210">import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;</span> |
| <span class="source-line-no">211</span><span id="line-211">import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry;</span> |
| <span class="source-line-no">212</span><span id="line-212">import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;</span> |
| <span class="source-line-no">213</span><span id="line-213">import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceCall;</span> |
| <span class="source-line-no">214</span><span id="line-214">import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionLoad;</span> |
| <span class="source-line-no">215</span><span id="line-215">import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.StoreSequenceId;</span> |
| <span class="source-line-no">216</span><span id="line-216">import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;</span> |
| <span class="source-line-no">217</span><span id="line-217">import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;</span> |
| <span class="source-line-no">218</span><span id="line-218">import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor;</span> |
| <span class="source-line-no">219</span><span id="line-219">import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor;</span> |
| <span class="source-line-no">220</span><span id="line-220">import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;</span> |
| <span class="source-line-no">221</span><span id="line-221">import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor;</span> |
| <span class="source-line-no">222</span><span id="line-222">import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor;</span> |
| <span class="source-line-no">223</span><span id="line-223">import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;</span> |
| <span class="source-line-no">224</span><span id="line-224">import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor;</span> |
| <span class="source-line-no">225</span><span id="line-225"></span> |
| <span class="source-line-no">226</span><span id="line-226">/**</span> |
| <span class="source-line-no">227</span><span id="line-227"> * Regions store data for a certain region of a table. It stores all columns for each row. A given</span> |
| <span class="source-line-no">228</span><span id="line-228"> * table consists of one or more Regions.</span> |
| <span class="source-line-no">229</span><span id="line-229"> * <p></span> |
| <span class="source-line-no">230</span><span id="line-230"> * An Region is defined by its table and its key extent.</span> |
| <span class="source-line-no">231</span><span id="line-231"> * <p></span> |
| <span class="source-line-no">232</span><span id="line-232"> * Locking at the Region level serves only one purpose: preventing the region from being closed (and</span> |
| <span class="source-line-no">233</span><span id="line-233"> * consequently split) while other operations are ongoing. Each row level operation obtains both a</span> |
| <span class="source-line-no">234</span><span id="line-234"> * row lock and a region read lock for the duration of the operation. While a scanner is being</span> |
| <span class="source-line-no">235</span><span id="line-235"> * constructed, getScanner holds a read lock. If the scanner is successfully constructed, it holds a</span> |
| <span class="source-line-no">236</span><span id="line-236"> * read lock until it is closed. A close takes out a write lock and consequently will block for</span> |
| <span class="source-line-no">237</span><span id="line-237"> * ongoing operations and will block new operations from starting while the close is in progress.</span> |
| <span class="source-line-no">238</span><span id="line-238"> */</span> |
| <span class="source-line-no">239</span><span id="line-239">@SuppressWarnings("deprecation")</span> |
| <span class="source-line-no">240</span><span id="line-240">@InterfaceAudience.Private</span> |
| <span class="source-line-no">241</span><span id="line-241">public class HRegion implements HeapSize, PropagatingConfigurationObserver, Region {</span> |
| <span class="source-line-no">242</span><span id="line-242"> private static final Logger LOG = LoggerFactory.getLogger(HRegion.class);</span> |
| <span class="source-line-no">243</span><span id="line-243"></span> |
| <span class="source-line-no">244</span><span id="line-244"> public static final String LOAD_CFS_ON_DEMAND_CONFIG_KEY =</span> |
| <span class="source-line-no">245</span><span id="line-245"> "hbase.hregion.scan.loadColumnFamiliesOnDemand";</span> |
| <span class="source-line-no">246</span><span id="line-246"></span> |
| <span class="source-line-no">247</span><span id="line-247"> public static final String HBASE_MAX_CELL_SIZE_KEY = "hbase.server.keyvalue.maxsize";</span> |
| <span class="source-line-no">248</span><span id="line-248"> public static final int DEFAULT_MAX_CELL_SIZE = 10485760;</span> |
| <span class="source-line-no">249</span><span id="line-249"></span> |
| <span class="source-line-no">250</span><span id="line-250"> public static final String HBASE_REGIONSERVER_MINIBATCH_SIZE =</span> |
| <span class="source-line-no">251</span><span id="line-251"> "hbase.regionserver.minibatch.size";</span> |
| <span class="source-line-no">252</span><span id="line-252"> public static final int DEFAULT_HBASE_REGIONSERVER_MINIBATCH_SIZE = 20000;</span> |
| <span class="source-line-no">253</span><span id="line-253"></span> |
| <span class="source-line-no">254</span><span id="line-254"> public static final String WAL_HSYNC_CONF_KEY = "hbase.wal.hsync";</span> |
| <span class="source-line-no">255</span><span id="line-255"> public static final boolean DEFAULT_WAL_HSYNC = false;</span> |
| <span class="source-line-no">256</span><span id="line-256"></span> |
| <span class="source-line-no">257</span><span id="line-257"> /** Parameter name for compaction after bulkload */</span> |
| <span class="source-line-no">258</span><span id="line-258"> public static final String COMPACTION_AFTER_BULKLOAD_ENABLE =</span> |
| <span class="source-line-no">259</span><span id="line-259"> "hbase.compaction.after.bulkload.enable";</span> |
| <span class="source-line-no">260</span><span id="line-260"></span> |
| <span class="source-line-no">261</span><span id="line-261"> /** Config for allow split when file count greater than the configured blocking file count */</span> |
| <span class="source-line-no">262</span><span id="line-262"> public static final String SPLIT_IGNORE_BLOCKING_ENABLED_KEY =</span> |
| <span class="source-line-no">263</span><span id="line-263"> "hbase.hregion.split.ignore.blocking.enabled";</span> |
| <span class="source-line-no">264</span><span id="line-264"></span> |
| <span class="source-line-no">265</span><span id="line-265"> public static final String REGION_STORAGE_POLICY_KEY = "hbase.hregion.block.storage.policy";</span> |
| <span class="source-line-no">266</span><span id="line-266"> public static final String DEFAULT_REGION_STORAGE_POLICY = "NONE";</span> |
| <span class="source-line-no">267</span><span id="line-267"></span> |
| <span class="source-line-no">268</span><span id="line-268"> /**</span> |
| <span class="source-line-no">269</span><span id="line-269"> * This is for for using HRegion as a local storage, where we may put the recovered edits in a</span> |
| <span class="source-line-no">270</span><span id="line-270"> * special place. Once this is set, we will only replay the recovered edits under this directory</span> |
| <span class="source-line-no">271</span><span id="line-271"> * and ignore the original replay directory configs.</span> |
| <span class="source-line-no">272</span><span id="line-272"> */</span> |
| <span class="source-line-no">273</span><span id="line-273"> public static final String SPECIAL_RECOVERED_EDITS_DIR =</span> |
| <span class="source-line-no">274</span><span id="line-274"> "hbase.hregion.special.recovered.edits.dir";</span> |
| <span class="source-line-no">275</span><span id="line-275"></span> |
| <span class="source-line-no">276</span><span id="line-276"> /**</span> |
| <span class="source-line-no">277</span><span id="line-277"> * Mainly used for master local region, where we will replay the WAL file directly without</span> |
| <span class="source-line-no">278</span><span id="line-278"> * splitting, so it is possible to have WAL files which are not closed cleanly, in this way,</span> |
| <span class="source-line-no">279</span><span id="line-279"> * hitting EOF is expected so should not consider it as a critical problem.</span> |
| <span class="source-line-no">280</span><span id="line-280"> */</span> |
| <span class="source-line-no">281</span><span id="line-281"> public static final String RECOVERED_EDITS_IGNORE_EOF =</span> |
| <span class="source-line-no">282</span><span id="line-282"> "hbase.hregion.recovered.edits.ignore.eof";</span> |
| <span class="source-line-no">283</span><span id="line-283"></span> |
| <span class="source-line-no">284</span><span id="line-284"> /**</span> |
| <span class="source-line-no">285</span><span id="line-285"> * Whether to use {@link MetaCellComparator} even if we are not meta region. Used when creating</span> |
| <span class="source-line-no">286</span><span id="line-286"> * master local region.</span> |
| <span class="source-line-no">287</span><span id="line-287"> */</span> |
| <span class="source-line-no">288</span><span id="line-288"> public static final String USE_META_CELL_COMPARATOR = "hbase.region.use.meta.cell.comparator";</span> |
| <span class="source-line-no">289</span><span id="line-289"></span> |
| <span class="source-line-no">290</span><span id="line-290"> public static final boolean DEFAULT_USE_META_CELL_COMPARATOR = false;</span> |
| <span class="source-line-no">291</span><span id="line-291"></span> |
| <span class="source-line-no">292</span><span id="line-292"> final AtomicBoolean closed = new AtomicBoolean(false);</span> |
| <span class="source-line-no">293</span><span id="line-293"></span> |
| <span class="source-line-no">294</span><span id="line-294"> /*</span> |
| <span class="source-line-no">295</span><span id="line-295"> * Closing can take some time; use the closing flag if there is stuff we don't want to do while in</span> |
| <span class="source-line-no">296</span><span id="line-296"> * closing state; e.g. like offer this region up to the master as a region to close if the</span> |
| <span class="source-line-no">297</span><span id="line-297"> * carrying regionserver is overloaded. Once set, it is never cleared.</span> |
| <span class="source-line-no">298</span><span id="line-298"> */</span> |
| <span class="source-line-no">299</span><span id="line-299"> final AtomicBoolean closing = new AtomicBoolean(false);</span> |
| <span class="source-line-no">300</span><span id="line-300"></span> |
| <span class="source-line-no">301</span><span id="line-301"> /**</span> |
| <span class="source-line-no">302</span><span id="line-302"> * The max sequence id of flushed data on this region. There is no edit in memory that is less</span> |
| <span class="source-line-no">303</span><span id="line-303"> * that this sequence id.</span> |
| <span class="source-line-no">304</span><span id="line-304"> */</span> |
| <span class="source-line-no">305</span><span id="line-305"> private volatile long maxFlushedSeqId = HConstants.NO_SEQNUM;</span> |
| <span class="source-line-no">306</span><span id="line-306"></span> |
| <span class="source-line-no">307</span><span id="line-307"> /**</span> |
| <span class="source-line-no">308</span><span id="line-308"> * Record the sequence id of last flush operation. Can be in advance of {@link #maxFlushedSeqId}</span> |
| <span class="source-line-no">309</span><span id="line-309"> * when flushing a single column family. In this case, {@link #maxFlushedSeqId} will be older than</span> |
| <span class="source-line-no">310</span><span id="line-310"> * the oldest edit in memory.</span> |
| <span class="source-line-no">311</span><span id="line-311"> */</span> |
| <span class="source-line-no">312</span><span id="line-312"> private volatile long lastFlushOpSeqId = HConstants.NO_SEQNUM;</span> |
| <span class="source-line-no">313</span><span id="line-313"></span> |
| <span class="source-line-no">314</span><span id="line-314"> /**</span> |
| <span class="source-line-no">315</span><span id="line-315"> * The sequence id of the last replayed open region event from the primary region. This is used to</span> |
| <span class="source-line-no">316</span><span id="line-316"> * skip entries before this due to the possibility of replay edits coming out of order from</span> |
| <span class="source-line-no">317</span><span id="line-317"> * replication.</span> |
| <span class="source-line-no">318</span><span id="line-318"> */</span> |
| <span class="source-line-no">319</span><span id="line-319"> protected volatile long lastReplayedOpenRegionSeqId = -1L;</span> |
| <span class="source-line-no">320</span><span id="line-320"> protected volatile long lastReplayedCompactionSeqId = -1L;</span> |
| <span class="source-line-no">321</span><span id="line-321"></span> |
| <span class="source-line-no">322</span><span id="line-322"> //////////////////////////////////////////////////////////////////////////////</span> |
| <span class="source-line-no">323</span><span id="line-323"> // Members</span> |
| <span class="source-line-no">324</span><span id="line-324"> //////////////////////////////////////////////////////////////////////////////</span> |
| <span class="source-line-no">325</span><span id="line-325"></span> |
| <span class="source-line-no">326</span><span id="line-326"> // map from a locked row to the context for that lock including:</span> |
| <span class="source-line-no">327</span><span id="line-327"> // - CountDownLatch for threads waiting on that row</span> |
| <span class="source-line-no">328</span><span id="line-328"> // - the thread that owns the lock (allow reentrancy)</span> |
| <span class="source-line-no">329</span><span id="line-329"> // - reference count of (reentrant) locks held by the thread</span> |
| <span class="source-line-no">330</span><span id="line-330"> // - the row itself</span> |
| <span class="source-line-no">331</span><span id="line-331"> private final ConcurrentHashMap<HashedBytes, RowLockContext> lockedRows =</span> |
| <span class="source-line-no">332</span><span id="line-332"> new ConcurrentHashMap<>();</span> |
| <span class="source-line-no">333</span><span id="line-333"></span> |
| <span class="source-line-no">334</span><span id="line-334"> protected final Map<byte[], HStore> stores =</span> |
| <span class="source-line-no">335</span><span id="line-335"> new ConcurrentSkipListMap<>(Bytes.BYTES_RAWCOMPARATOR);</span> |
| <span class="source-line-no">336</span><span id="line-336"></span> |
| <span class="source-line-no">337</span><span id="line-337"> // TODO: account for each registered handler in HeapSize computation</span> |
| <span class="source-line-no">338</span><span id="line-338"> private Map<String, Service> coprocessorServiceHandlers = Maps.newHashMap();</span> |
| <span class="source-line-no">339</span><span id="line-339"></span> |
| <span class="source-line-no">340</span><span id="line-340"> // Track data size in all memstores</span> |
| <span class="source-line-no">341</span><span id="line-341"> private final MemStoreSizing memStoreSizing = new ThreadSafeMemStoreSizing();</span> |
| <span class="source-line-no">342</span><span id="line-342"> RegionServicesForStores regionServicesForStores;</span> |
| <span class="source-line-no">343</span><span id="line-343"></span> |
| <span class="source-line-no">344</span><span id="line-344"> // Debug possible data loss due to WAL off</span> |
| <span class="source-line-no">345</span><span id="line-345"> final LongAdder numMutationsWithoutWAL = new LongAdder();</span> |
| <span class="source-line-no">346</span><span id="line-346"> final LongAdder dataInMemoryWithoutWAL = new LongAdder();</span> |
| <span class="source-line-no">347</span><span id="line-347"></span> |
| <span class="source-line-no">348</span><span id="line-348"> // Debug why CAS operations are taking a while.</span> |
| <span class="source-line-no">349</span><span id="line-349"> final LongAdder checkAndMutateChecksPassed = new LongAdder();</span> |
| <span class="source-line-no">350</span><span id="line-350"> final LongAdder checkAndMutateChecksFailed = new LongAdder();</span> |
| <span class="source-line-no">351</span><span id="line-351"></span> |
| <span class="source-line-no">352</span><span id="line-352"> // Number of requests</span> |
| <span class="source-line-no">353</span><span id="line-353"> // Count rows for scan</span> |
| <span class="source-line-no">354</span><span id="line-354"> final LongAdder readRequestsCount = new LongAdder();</span> |
| <span class="source-line-no">355</span><span id="line-355"> final LongAdder cpRequestsCount = new LongAdder();</span> |
| <span class="source-line-no">356</span><span id="line-356"> final LongAdder filteredReadRequestsCount = new LongAdder();</span> |
| <span class="source-line-no">357</span><span id="line-357"> // Count rows for multi row mutations</span> |
| <span class="source-line-no">358</span><span id="line-358"> final LongAdder writeRequestsCount = new LongAdder();</span> |
| <span class="source-line-no">359</span><span id="line-359"></span> |
| <span class="source-line-no">360</span><span id="line-360"> // Number of requests blocked by memstore size.</span> |
| <span class="source-line-no">361</span><span id="line-361"> private final LongAdder blockedRequestsCount = new LongAdder();</span> |
| <span class="source-line-no">362</span><span id="line-362"></span> |
| <span class="source-line-no">363</span><span id="line-363"> // Compaction LongAdders</span> |
| <span class="source-line-no">364</span><span id="line-364"> final LongAdder compactionsFinished = new LongAdder();</span> |
| <span class="source-line-no">365</span><span id="line-365"> final LongAdder compactionsFailed = new LongAdder();</span> |
| <span class="source-line-no">366</span><span id="line-366"> final LongAdder compactionNumFilesCompacted = new LongAdder();</span> |
| <span class="source-line-no">367</span><span id="line-367"> final LongAdder compactionNumBytesCompacted = new LongAdder();</span> |
| <span class="source-line-no">368</span><span id="line-368"> final LongAdder compactionsQueued = new LongAdder();</span> |
| <span class="source-line-no">369</span><span id="line-369"> final LongAdder flushesQueued = new LongAdder();</span> |
| <span class="source-line-no">370</span><span id="line-370"></span> |
| <span class="source-line-no">371</span><span id="line-371"> private BlockCache blockCache;</span> |
| <span class="source-line-no">372</span><span id="line-372"> private MobFileCache mobFileCache;</span> |
| <span class="source-line-no">373</span><span id="line-373"> private final WAL wal;</span> |
| <span class="source-line-no">374</span><span id="line-374"> private final HRegionFileSystem fs;</span> |
| <span class="source-line-no">375</span><span id="line-375"> protected final Configuration conf;</span> |
| <span class="source-line-no">376</span><span id="line-376"> private final Configuration baseConf;</span> |
| <span class="source-line-no">377</span><span id="line-377"> private final int rowLockWaitDuration;</span> |
| <span class="source-line-no">378</span><span id="line-378"> static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000;</span> |
| <span class="source-line-no">379</span><span id="line-379"></span> |
| <span class="source-line-no">380</span><span id="line-380"> private Path regionWalDir;</span> |
| <span class="source-line-no">381</span><span id="line-381"> private FileSystem walFS;</span> |
| <span class="source-line-no">382</span><span id="line-382"></span> |
| <span class="source-line-no">383</span><span id="line-383"> // set to true if the region is restored from snapshot for reading by ClientSideRegionScanner</span> |
| <span class="source-line-no">384</span><span id="line-384"> private boolean isRestoredRegion = false;</span> |
| <span class="source-line-no">385</span><span id="line-385"></span> |
| <span class="source-line-no">386</span><span id="line-386"> public void setRestoredRegion(boolean restoredRegion) {</span> |
| <span class="source-line-no">387</span><span id="line-387"> isRestoredRegion = restoredRegion;</span> |
| <span class="source-line-no">388</span><span id="line-388"> }</span> |
| <span class="source-line-no">389</span><span id="line-389"></span> |
| <span class="source-line-no">390</span><span id="line-390"> public MetricsTableRequests getMetricsTableRequests() {</span> |
| <span class="source-line-no">391</span><span id="line-391"> return metricsTableRequests;</span> |
| <span class="source-line-no">392</span><span id="line-392"> }</span> |
| <span class="source-line-no">393</span><span id="line-393"></span> |
| <span class="source-line-no">394</span><span id="line-394"> // Handle table latency metrics</span> |
| <span class="source-line-no">395</span><span id="line-395"> private MetricsTableRequests metricsTableRequests;</span> |
| <span class="source-line-no">396</span><span id="line-396"></span> |
| <span class="source-line-no">397</span><span id="line-397"> // The internal wait duration to acquire a lock before read/update</span> |
| <span class="source-line-no">398</span><span id="line-398"> // from the region. It is not per row. The purpose of this wait time</span> |
| <span class="source-line-no">399</span><span id="line-399"> // is to avoid waiting a long time while the region is busy, so that</span> |
| <span class="source-line-no">400</span><span id="line-400"> // we can release the IPC handler soon enough to improve the</span> |
| <span class="source-line-no">401</span><span id="line-401"> // availability of the region server. It can be adjusted by</span> |
| <span class="source-line-no">402</span><span id="line-402"> // tuning configuration "hbase.busy.wait.duration".</span> |
| <span class="source-line-no">403</span><span id="line-403"> final long busyWaitDuration;</span> |
| <span class="source-line-no">404</span><span id="line-404"> static final long DEFAULT_BUSY_WAIT_DURATION = HConstants.DEFAULT_HBASE_RPC_TIMEOUT;</span> |
| <span class="source-line-no">405</span><span id="line-405"></span> |
| <span class="source-line-no">406</span><span id="line-406"> // If updating multiple rows in one call, wait longer,</span> |
| <span class="source-line-no">407</span><span id="line-407"> // i.e. waiting for busyWaitDuration * # of rows. However,</span> |
| <span class="source-line-no">408</span><span id="line-408"> // we can limit the max multiplier.</span> |
| <span class="source-line-no">409</span><span id="line-409"> final int maxBusyWaitMultiplier;</span> |
| <span class="source-line-no">410</span><span id="line-410"></span> |
| <span class="source-line-no">411</span><span id="line-411"> // Max busy wait duration. There is no point to wait longer than the RPC</span> |
| <span class="source-line-no">412</span><span id="line-412"> // purge timeout, when a RPC call will be terminated by the RPC engine.</span> |
| <span class="source-line-no">413</span><span id="line-413"> final long maxBusyWaitDuration;</span> |
| <span class="source-line-no">414</span><span id="line-414"></span> |
| <span class="source-line-no">415</span><span id="line-415"> // Max cell size. If nonzero, the maximum allowed size for any given cell</span> |
| <span class="source-line-no">416</span><span id="line-416"> // in bytes</span> |
| <span class="source-line-no">417</span><span id="line-417"> final long maxCellSize;</span> |
| <span class="source-line-no">418</span><span id="line-418"></span> |
| <span class="source-line-no">419</span><span id="line-419"> // Number of mutations for minibatch processing.</span> |
| <span class="source-line-no">420</span><span id="line-420"> private final int miniBatchSize;</span> |
| <span class="source-line-no">421</span><span id="line-421"></span> |
| <span class="source-line-no">422</span><span id="line-422"> final ConcurrentHashMap<RegionScanner, Long> scannerReadPoints;</span> |
| <span class="source-line-no">423</span><span id="line-423"> final ReadPointCalculationLock smallestReadPointCalcLock;</span> |
| <span class="source-line-no">424</span><span id="line-424"></span> |
| <span class="source-line-no">425</span><span id="line-425"> /**</span> |
| <span class="source-line-no">426</span><span id="line-426"> * The sequence ID that was enLongAddered when this region was opened.</span> |
| <span class="source-line-no">427</span><span id="line-427"> */</span> |
| <span class="source-line-no">428</span><span id="line-428"> private long openSeqNum = HConstants.NO_SEQNUM;</span> |
| <span class="source-line-no">429</span><span id="line-429"></span> |
| <span class="source-line-no">430</span><span id="line-430"> /**</span> |
| <span class="source-line-no">431</span><span id="line-431"> * The default setting for whether to enable on-demand CF loading for scan requests to this</span> |
| <span class="source-line-no">432</span><span id="line-432"> * region. Requests can override it.</span> |
| <span class="source-line-no">433</span><span id="line-433"> */</span> |
| <span class="source-line-no">434</span><span id="line-434"> private boolean isLoadingCfsOnDemandDefault = false;</span> |
| <span class="source-line-no">435</span><span id="line-435"></span> |
| <span class="source-line-no">436</span><span id="line-436"> private final AtomicInteger majorInProgress = new AtomicInteger(0);</span> |
| <span class="source-line-no">437</span><span id="line-437"> private final AtomicInteger minorInProgress = new AtomicInteger(0);</span> |
| <span class="source-line-no">438</span><span id="line-438"></span> |
| <span class="source-line-no">439</span><span id="line-439"> //</span> |
| <span class="source-line-no">440</span><span id="line-440"> // Context: During replay we want to ensure that we do not lose any data. So, we</span> |
| <span class="source-line-no">441</span><span id="line-441"> // have to be conservative in how we replay wals. For each store, we calculate</span> |
| <span class="source-line-no">442</span><span id="line-442"> // the maxSeqId up to which the store was flushed. And, skip the edits which</span> |
| <span class="source-line-no">443</span><span id="line-443"> // are equal to or lower than maxSeqId for each store.</span> |
| <span class="source-line-no">444</span><span id="line-444"> // The following map is populated when opening the region</span> |
| <span class="source-line-no">445</span><span id="line-445"> Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">446</span><span id="line-446"></span> |
| <span class="source-line-no">447</span><span id="line-447"> // lock used to protect the replay operation for secondary replicas, so the below two fields does</span> |
| <span class="source-line-no">448</span><span id="line-448"> // not need to be volatile.</span> |
| <span class="source-line-no">449</span><span id="line-449"> private Lock replayLock;</span> |
| <span class="source-line-no">450</span><span id="line-450"></span> |
| <span class="source-line-no">451</span><span id="line-451"> /** Saved state from replaying prepare flush cache */</span> |
| <span class="source-line-no">452</span><span id="line-452"> private PrepareFlushResult prepareFlushResult = null;</span> |
| <span class="source-line-no">453</span><span id="line-453"></span> |
| <span class="source-line-no">454</span><span id="line-454"> private long lastReplayedSequenceId = HConstants.NO_SEQNUM;</span> |
| <span class="source-line-no">455</span><span id="line-455"></span> |
| <span class="source-line-no">456</span><span id="line-456"> private volatile ConfigurationManager configurationManager;</span> |
| <span class="source-line-no">457</span><span id="line-457"></span> |
| <span class="source-line-no">458</span><span id="line-458"> // Used for testing.</span> |
| <span class="source-line-no">459</span><span id="line-459"> private volatile Long timeoutForWriteLock = null;</span> |
| <span class="source-line-no">460</span><span id="line-460"></span> |
| <span class="source-line-no">461</span><span id="line-461"> private final CellComparator cellComparator;</span> |
| <span class="source-line-no">462</span><span id="line-462"></span> |
| <span class="source-line-no">463</span><span id="line-463"> private final int minBlockSizeBytes;</span> |
| <span class="source-line-no">464</span><span id="line-464"></span> |
| <span class="source-line-no">465</span><span id="line-465"> /**</span> |
| <span class="source-line-no">466</span><span id="line-466"> * @return The smallest mvcc readPoint across all the scanners in this region. Writes older than</span> |
| <span class="source-line-no">467</span><span id="line-467"> * this readPoint, are included in every read operation.</span> |
| <span class="source-line-no">468</span><span id="line-468"> */</span> |
| <span class="source-line-no">469</span><span id="line-469"> public long getSmallestReadPoint() {</span> |
| <span class="source-line-no">470</span><span id="line-470"> // We need to ensure that while we are calculating the smallestReadPoint</span> |
| <span class="source-line-no">471</span><span id="line-471"> // no new RegionScanners can grab a readPoint that we are unaware of.</span> |
| <span class="source-line-no">472</span><span id="line-472"> smallestReadPointCalcLock.lock(ReadPointCalculationLock.LockType.CALCULATION_LOCK);</span> |
| <span class="source-line-no">473</span><span id="line-473"> try {</span> |
| <span class="source-line-no">474</span><span id="line-474"> long minimumReadPoint = mvcc.getReadPoint();</span> |
| <span class="source-line-no">475</span><span id="line-475"> for (Long readPoint : this.scannerReadPoints.values()) {</span> |
| <span class="source-line-no">476</span><span id="line-476"> minimumReadPoint = Math.min(minimumReadPoint, readPoint);</span> |
| <span class="source-line-no">477</span><span id="line-477"> }</span> |
| <span class="source-line-no">478</span><span id="line-478"> return minimumReadPoint;</span> |
| <span class="source-line-no">479</span><span id="line-479"> } finally {</span> |
| <span class="source-line-no">480</span><span id="line-480"> smallestReadPointCalcLock.unlock(ReadPointCalculationLock.LockType.CALCULATION_LOCK);</span> |
| <span class="source-line-no">481</span><span id="line-481"> }</span> |
| <span class="source-line-no">482</span><span id="line-482"> }</span> |
| <span class="source-line-no">483</span><span id="line-483"></span> |
| <span class="source-line-no">484</span><span id="line-484"> /*</span> |
| <span class="source-line-no">485</span><span id="line-485"> * Data structure of write state flags used coordinating flushes, compactions and closes.</span> |
| <span class="source-line-no">486</span><span id="line-486"> */</span> |
| <span class="source-line-no">487</span><span id="line-487"> static class WriteState {</span> |
| <span class="source-line-no">488</span><span id="line-488"> // Set while a memstore flush is happening.</span> |
| <span class="source-line-no">489</span><span id="line-489"> volatile boolean flushing = false;</span> |
| <span class="source-line-no">490</span><span id="line-490"> // Set when a flush has been requested.</span> |
| <span class="source-line-no">491</span><span id="line-491"> volatile boolean flushRequested = false;</span> |
| <span class="source-line-no">492</span><span id="line-492"> // Number of compactions running.</span> |
| <span class="source-line-no">493</span><span id="line-493"> AtomicInteger compacting = new AtomicInteger(0);</span> |
| <span class="source-line-no">494</span><span id="line-494"> // Gets set in close. If set, cannot compact or flush again.</span> |
| <span class="source-line-no">495</span><span id="line-495"> volatile boolean writesEnabled = true;</span> |
| <span class="source-line-no">496</span><span id="line-496"> // Set if region is read-only</span> |
| <span class="source-line-no">497</span><span id="line-497"> volatile boolean readOnly = false;</span> |
| <span class="source-line-no">498</span><span id="line-498"> // whether the reads are enabled. This is different than readOnly, because readOnly is</span> |
| <span class="source-line-no">499</span><span id="line-499"> // static in the lifetime of the region, while readsEnabled is dynamic</span> |
| <span class="source-line-no">500</span><span id="line-500"> volatile boolean readsEnabled = true;</span> |
| <span class="source-line-no">501</span><span id="line-501"></span> |
| <span class="source-line-no">502</span><span id="line-502"> /**</span> |
| <span class="source-line-no">503</span><span id="line-503"> * Set flags that make this region read-only.</span> |
| <span class="source-line-no">504</span><span id="line-504"> * @param onOff flip value for region r/o setting</span> |
| <span class="source-line-no">505</span><span id="line-505"> */</span> |
| <span class="source-line-no">506</span><span id="line-506"> synchronized void setReadOnly(final boolean onOff) {</span> |
| <span class="source-line-no">507</span><span id="line-507"> this.writesEnabled = !onOff;</span> |
| <span class="source-line-no">508</span><span id="line-508"> this.readOnly = onOff;</span> |
| <span class="source-line-no">509</span><span id="line-509"> }</span> |
| <span class="source-line-no">510</span><span id="line-510"></span> |
| <span class="source-line-no">511</span><span id="line-511"> boolean isReadOnly() {</span> |
| <span class="source-line-no">512</span><span id="line-512"> return this.readOnly;</span> |
| <span class="source-line-no">513</span><span id="line-513"> }</span> |
| <span class="source-line-no">514</span><span id="line-514"></span> |
| <span class="source-line-no">515</span><span id="line-515"> boolean isFlushRequested() {</span> |
| <span class="source-line-no">516</span><span id="line-516"> return this.flushRequested;</span> |
| <span class="source-line-no">517</span><span id="line-517"> }</span> |
| <span class="source-line-no">518</span><span id="line-518"></span> |
| <span class="source-line-no">519</span><span id="line-519"> void setReadsEnabled(boolean readsEnabled) {</span> |
| <span class="source-line-no">520</span><span id="line-520"> this.readsEnabled = readsEnabled;</span> |
| <span class="source-line-no">521</span><span id="line-521"> }</span> |
| <span class="source-line-no">522</span><span id="line-522"></span> |
| <span class="source-line-no">523</span><span id="line-523"> static final long HEAP_SIZE = ClassSize.align(ClassSize.OBJECT + 5 * Bytes.SIZEOF_BOOLEAN);</span> |
| <span class="source-line-no">524</span><span id="line-524"> }</span> |
| <span class="source-line-no">525</span><span id="line-525"></span> |
| <span class="source-line-no">526</span><span id="line-526"> /**</span> |
| <span class="source-line-no">527</span><span id="line-527"> * Objects from this class are created when flushing to describe all the different states that</span> |
| <span class="source-line-no">528</span><span id="line-528"> * that method ends up in. The Result enum describes those states. The sequence id should only be</span> |
| <span class="source-line-no">529</span><span id="line-529"> * specified if the flush was successful, and the failure message should only be specified if it</span> |
| <span class="source-line-no">530</span><span id="line-530"> * didn't flush.</span> |
| <span class="source-line-no">531</span><span id="line-531"> */</span> |
| <span class="source-line-no">532</span><span id="line-532"> public static class FlushResultImpl implements FlushResult {</span> |
| <span class="source-line-no">533</span><span id="line-533"> final Result result;</span> |
| <span class="source-line-no">534</span><span id="line-534"> final String failureReason;</span> |
| <span class="source-line-no">535</span><span id="line-535"> final long flushSequenceId;</span> |
| <span class="source-line-no">536</span><span id="line-536"> final boolean wroteFlushWalMarker;</span> |
| <span class="source-line-no">537</span><span id="line-537"></span> |
| <span class="source-line-no">538</span><span id="line-538"> /**</span> |
| <span class="source-line-no">539</span><span id="line-539"> * Convenience constructor to use when the flush is successful, the failure message is set to</span> |
| <span class="source-line-no">540</span><span id="line-540"> * null.</span> |
| <span class="source-line-no">541</span><span id="line-541"> * @param result Expecting FLUSHED_NO_COMPACTION_NEEDED or FLUSHED_COMPACTION_NEEDED.</span> |
| <span class="source-line-no">542</span><span id="line-542"> * @param flushSequenceId Generated sequence id that comes right after the edits in the</span> |
| <span class="source-line-no">543</span><span id="line-543"> * memstores.</span> |
| <span class="source-line-no">544</span><span id="line-544"> */</span> |
| <span class="source-line-no">545</span><span id="line-545"> FlushResultImpl(Result result, long flushSequenceId) {</span> |
| <span class="source-line-no">546</span><span id="line-546"> this(result, flushSequenceId, null, false);</span> |
| <span class="source-line-no">547</span><span id="line-547"> assert result == Result.FLUSHED_NO_COMPACTION_NEEDED</span> |
| <span class="source-line-no">548</span><span id="line-548"> || result == Result.FLUSHED_COMPACTION_NEEDED;</span> |
| <span class="source-line-no">549</span><span id="line-549"> }</span> |
| <span class="source-line-no">550</span><span id="line-550"></span> |
| <span class="source-line-no">551</span><span id="line-551"> /**</span> |
| <span class="source-line-no">552</span><span id="line-552"> * Convenience constructor to use when we cannot flush.</span> |
| <span class="source-line-no">553</span><span id="line-553"> * @param result Expecting CANNOT_FLUSH_MEMSTORE_EMPTY or CANNOT_FLUSH.</span> |
| <span class="source-line-no">554</span><span id="line-554"> * @param failureReason Reason why we couldn't flush.</span> |
| <span class="source-line-no">555</span><span id="line-555"> */</span> |
| <span class="source-line-no">556</span><span id="line-556"> FlushResultImpl(Result result, String failureReason, boolean wroteFlushMarker) {</span> |
| <span class="source-line-no">557</span><span id="line-557"> this(result, -1, failureReason, wroteFlushMarker);</span> |
| <span class="source-line-no">558</span><span id="line-558"> assert result == Result.CANNOT_FLUSH_MEMSTORE_EMPTY || result == Result.CANNOT_FLUSH;</span> |
| <span class="source-line-no">559</span><span id="line-559"> }</span> |
| <span class="source-line-no">560</span><span id="line-560"></span> |
| <span class="source-line-no">561</span><span id="line-561"> /**</span> |
| <span class="source-line-no">562</span><span id="line-562"> * Constructor with all the parameters.</span> |
| <span class="source-line-no">563</span><span id="line-563"> * @param result Any of the Result.</span> |
| <span class="source-line-no">564</span><span id="line-564"> * @param flushSequenceId Generated sequence id if the memstores were flushed else -1.</span> |
| <span class="source-line-no">565</span><span id="line-565"> * @param failureReason Reason why we couldn't flush, or null.</span> |
| <span class="source-line-no">566</span><span id="line-566"> */</span> |
| <span class="source-line-no">567</span><span id="line-567"> FlushResultImpl(Result result, long flushSequenceId, String failureReason,</span> |
| <span class="source-line-no">568</span><span id="line-568"> boolean wroteFlushMarker) {</span> |
| <span class="source-line-no">569</span><span id="line-569"> this.result = result;</span> |
| <span class="source-line-no">570</span><span id="line-570"> this.flushSequenceId = flushSequenceId;</span> |
| <span class="source-line-no">571</span><span id="line-571"> this.failureReason = failureReason;</span> |
| <span class="source-line-no">572</span><span id="line-572"> this.wroteFlushWalMarker = wroteFlushMarker;</span> |
| <span class="source-line-no">573</span><span id="line-573"> }</span> |
| <span class="source-line-no">574</span><span id="line-574"></span> |
| <span class="source-line-no">575</span><span id="line-575"> /**</span> |
| <span class="source-line-no">576</span><span id="line-576"> * Convenience method, the equivalent of checking if result is FLUSHED_NO_COMPACTION_NEEDED or</span> |
| <span class="source-line-no">577</span><span id="line-577"> * FLUSHED_NO_COMPACTION_NEEDED.</span> |
| <span class="source-line-no">578</span><span id="line-578"> * @return true if the memstores were flushed, else false.</span> |
| <span class="source-line-no">579</span><span id="line-579"> */</span> |
| <span class="source-line-no">580</span><span id="line-580"> @Override</span> |
| <span class="source-line-no">581</span><span id="line-581"> public boolean isFlushSucceeded() {</span> |
| <span class="source-line-no">582</span><span id="line-582"> return result == Result.FLUSHED_NO_COMPACTION_NEEDED</span> |
| <span class="source-line-no">583</span><span id="line-583"> || result == Result.FLUSHED_COMPACTION_NEEDED;</span> |
| <span class="source-line-no">584</span><span id="line-584"> }</span> |
| <span class="source-line-no">585</span><span id="line-585"></span> |
| <span class="source-line-no">586</span><span id="line-586"> /**</span> |
| <span class="source-line-no">587</span><span id="line-587"> * Convenience method, the equivalent of checking if result is FLUSHED_COMPACTION_NEEDED.</span> |
| <span class="source-line-no">588</span><span id="line-588"> * @return True if the flush requested a compaction, else false (doesn't even mean it flushed).</span> |
| <span class="source-line-no">589</span><span id="line-589"> */</span> |
| <span class="source-line-no">590</span><span id="line-590"> @Override</span> |
| <span class="source-line-no">591</span><span id="line-591"> public boolean isCompactionNeeded() {</span> |
| <span class="source-line-no">592</span><span id="line-592"> return result == Result.FLUSHED_COMPACTION_NEEDED;</span> |
| <span class="source-line-no">593</span><span id="line-593"> }</span> |
| <span class="source-line-no">594</span><span id="line-594"></span> |
| <span class="source-line-no">595</span><span id="line-595"> @Override</span> |
| <span class="source-line-no">596</span><span id="line-596"> public String toString() {</span> |
| <span class="source-line-no">597</span><span id="line-597"> return new StringBuilder().append("flush result:").append(result).append(", ")</span> |
| <span class="source-line-no">598</span><span id="line-598"> .append("failureReason:").append(failureReason).append(",").append("flush seq id")</span> |
| <span class="source-line-no">599</span><span id="line-599"> .append(flushSequenceId).toString();</span> |
| <span class="source-line-no">600</span><span id="line-600"> }</span> |
| <span class="source-line-no">601</span><span id="line-601"></span> |
| <span class="source-line-no">602</span><span id="line-602"> @Override</span> |
| <span class="source-line-no">603</span><span id="line-603"> public Result getResult() {</span> |
| <span class="source-line-no">604</span><span id="line-604"> return result;</span> |
| <span class="source-line-no">605</span><span id="line-605"> }</span> |
| <span class="source-line-no">606</span><span id="line-606"> }</span> |
| <span class="source-line-no">607</span><span id="line-607"></span> |
| <span class="source-line-no">608</span><span id="line-608"> /** A result object from prepare flush cache stage */</span> |
| <span class="source-line-no">609</span><span id="line-609"> protected static class PrepareFlushResult {</span> |
| <span class="source-line-no">610</span><span id="line-610"> final FlushResultImpl result; // indicating a failure result from prepare</span> |
| <span class="source-line-no">611</span><span id="line-611"> final TreeMap<byte[], StoreFlushContext> storeFlushCtxs;</span> |
| <span class="source-line-no">612</span><span id="line-612"> final TreeMap<byte[], List<Path>> committedFiles;</span> |
| <span class="source-line-no">613</span><span id="line-613"> final TreeMap<byte[], MemStoreSize> storeFlushableSize;</span> |
| <span class="source-line-no">614</span><span id="line-614"> final long startTime;</span> |
| <span class="source-line-no">615</span><span id="line-615"> final long flushOpSeqId;</span> |
| <span class="source-line-no">616</span><span id="line-616"> final long flushedSeqId;</span> |
| <span class="source-line-no">617</span><span id="line-617"> final MemStoreSizing totalFlushableSize;</span> |
| <span class="source-line-no">618</span><span id="line-618"></span> |
| <span class="source-line-no">619</span><span id="line-619"> /** Constructs an early exit case */</span> |
| <span class="source-line-no">620</span><span id="line-620"> PrepareFlushResult(FlushResultImpl result, long flushSeqId) {</span> |
| <span class="source-line-no">621</span><span id="line-621"> this(result, null, null, null, Math.max(0, flushSeqId), 0, 0, MemStoreSizing.DUD);</span> |
| <span class="source-line-no">622</span><span id="line-622"> }</span> |
| <span class="source-line-no">623</span><span id="line-623"></span> |
| <span class="source-line-no">624</span><span id="line-624"> /** Constructs a successful prepare flush result */</span> |
| <span class="source-line-no">625</span><span id="line-625"> PrepareFlushResult(TreeMap<byte[], StoreFlushContext> storeFlushCtxs,</span> |
| <span class="source-line-no">626</span><span id="line-626"> TreeMap<byte[], List<Path>> committedFiles, TreeMap<byte[], MemStoreSize> storeFlushableSize,</span> |
| <span class="source-line-no">627</span><span id="line-627"> long startTime, long flushSeqId, long flushedSeqId, MemStoreSizing totalFlushableSize) {</span> |
| <span class="source-line-no">628</span><span id="line-628"> this(null, storeFlushCtxs, committedFiles, storeFlushableSize, startTime, flushSeqId,</span> |
| <span class="source-line-no">629</span><span id="line-629"> flushedSeqId, totalFlushableSize);</span> |
| <span class="source-line-no">630</span><span id="line-630"> }</span> |
| <span class="source-line-no">631</span><span id="line-631"></span> |
| <span class="source-line-no">632</span><span id="line-632"> private PrepareFlushResult(FlushResultImpl result,</span> |
| <span class="source-line-no">633</span><span id="line-633"> TreeMap<byte[], StoreFlushContext> storeFlushCtxs, TreeMap<byte[], List<Path>> committedFiles,</span> |
| <span class="source-line-no">634</span><span id="line-634"> TreeMap<byte[], MemStoreSize> storeFlushableSize, long startTime, long flushSeqId,</span> |
| <span class="source-line-no">635</span><span id="line-635"> long flushedSeqId, MemStoreSizing totalFlushableSize) {</span> |
| <span class="source-line-no">636</span><span id="line-636"> this.result = result;</span> |
| <span class="source-line-no">637</span><span id="line-637"> this.storeFlushCtxs = storeFlushCtxs;</span> |
| <span class="source-line-no">638</span><span id="line-638"> this.committedFiles = committedFiles;</span> |
| <span class="source-line-no">639</span><span id="line-639"> this.storeFlushableSize = storeFlushableSize;</span> |
| <span class="source-line-no">640</span><span id="line-640"> this.startTime = startTime;</span> |
| <span class="source-line-no">641</span><span id="line-641"> this.flushOpSeqId = flushSeqId;</span> |
| <span class="source-line-no">642</span><span id="line-642"> this.flushedSeqId = flushedSeqId;</span> |
| <span class="source-line-no">643</span><span id="line-643"> this.totalFlushableSize = totalFlushableSize;</span> |
| <span class="source-line-no">644</span><span id="line-644"> }</span> |
| <span class="source-line-no">645</span><span id="line-645"></span> |
| <span class="source-line-no">646</span><span id="line-646"> public FlushResult getResult() {</span> |
| <span class="source-line-no">647</span><span id="line-647"> return this.result;</span> |
| <span class="source-line-no">648</span><span id="line-648"> }</span> |
| <span class="source-line-no">649</span><span id="line-649"> }</span> |
| <span class="source-line-no">650</span><span id="line-650"></span> |
| <span class="source-line-no">651</span><span id="line-651"> /**</span> |
| <span class="source-line-no">652</span><span id="line-652"> * A class that tracks exceptions that have been observed in one batch. Not thread safe.</span> |
| <span class="source-line-no">653</span><span id="line-653"> */</span> |
| <span class="source-line-no">654</span><span id="line-654"> static class ObservedExceptionsInBatch {</span> |
| <span class="source-line-no">655</span><span id="line-655"> private boolean wrongRegion = false;</span> |
| <span class="source-line-no">656</span><span id="line-656"> private boolean failedSanityCheck = false;</span> |
| <span class="source-line-no">657</span><span id="line-657"> private boolean wrongFamily = false;</span> |
| <span class="source-line-no">658</span><span id="line-658"></span> |
| <span class="source-line-no">659</span><span id="line-659"> /** Returns If a {@link WrongRegionException} has been observed. */</span> |
| <span class="source-line-no">660</span><span id="line-660"> boolean hasSeenWrongRegion() {</span> |
| <span class="source-line-no">661</span><span id="line-661"> return wrongRegion;</span> |
| <span class="source-line-no">662</span><span id="line-662"> }</span> |
| <span class="source-line-no">663</span><span id="line-663"></span> |
| <span class="source-line-no">664</span><span id="line-664"> /**</span> |
| <span class="source-line-no">665</span><span id="line-665"> * Records that a {@link WrongRegionException} has been observed.</span> |
| <span class="source-line-no">666</span><span id="line-666"> */</span> |
| <span class="source-line-no">667</span><span id="line-667"> void sawWrongRegion() {</span> |
| <span class="source-line-no">668</span><span id="line-668"> wrongRegion = true;</span> |
| <span class="source-line-no">669</span><span id="line-669"> }</span> |
| <span class="source-line-no">670</span><span id="line-670"></span> |
| <span class="source-line-no">671</span><span id="line-671"> /** Returns If a {@link FailedSanityCheckException} has been observed. */</span> |
| <span class="source-line-no">672</span><span id="line-672"> boolean hasSeenFailedSanityCheck() {</span> |
| <span class="source-line-no">673</span><span id="line-673"> return failedSanityCheck;</span> |
| <span class="source-line-no">674</span><span id="line-674"> }</span> |
| <span class="source-line-no">675</span><span id="line-675"></span> |
| <span class="source-line-no">676</span><span id="line-676"> /**</span> |
| <span class="source-line-no">677</span><span id="line-677"> * Records that a {@link FailedSanityCheckException} has been observed.</span> |
| <span class="source-line-no">678</span><span id="line-678"> */</span> |
| <span class="source-line-no">679</span><span id="line-679"> void sawFailedSanityCheck() {</span> |
| <span class="source-line-no">680</span><span id="line-680"> failedSanityCheck = true;</span> |
| <span class="source-line-no">681</span><span id="line-681"> }</span> |
| <span class="source-line-no">682</span><span id="line-682"></span> |
| <span class="source-line-no">683</span><span id="line-683"> /** Returns If a {@link NoSuchColumnFamilyException} has been observed. */</span> |
| <span class="source-line-no">684</span><span id="line-684"> boolean hasSeenNoSuchFamily() {</span> |
| <span class="source-line-no">685</span><span id="line-685"> return wrongFamily;</span> |
| <span class="source-line-no">686</span><span id="line-686"> }</span> |
| <span class="source-line-no">687</span><span id="line-687"></span> |
| <span class="source-line-no">688</span><span id="line-688"> /**</span> |
| <span class="source-line-no">689</span><span id="line-689"> * Records that a {@link NoSuchColumnFamilyException} has been observed.</span> |
| <span class="source-line-no">690</span><span id="line-690"> */</span> |
| <span class="source-line-no">691</span><span id="line-691"> void sawNoSuchFamily() {</span> |
| <span class="source-line-no">692</span><span id="line-692"> wrongFamily = true;</span> |
| <span class="source-line-no">693</span><span id="line-693"> }</span> |
| <span class="source-line-no">694</span><span id="line-694"> }</span> |
| <span class="source-line-no">695</span><span id="line-695"></span> |
| <span class="source-line-no">696</span><span id="line-696"> final WriteState writestate = new WriteState();</span> |
| <span class="source-line-no">697</span><span id="line-697"></span> |
| <span class="source-line-no">698</span><span id="line-698"> long memstoreFlushSize;</span> |
| <span class="source-line-no">699</span><span id="line-699"> final long timestampSlop;</span> |
| <span class="source-line-no">700</span><span id="line-700"></span> |
| <span class="source-line-no">701</span><span id="line-701"> // Last flush time for each Store. Useful when we are flushing for each column</span> |
| <span class="source-line-no">702</span><span id="line-702"> private final ConcurrentMap<HStore, Long> lastStoreFlushTimeMap = new ConcurrentHashMap<>();</span> |
| <span class="source-line-no">703</span><span id="line-703"></span> |
| <span class="source-line-no">704</span><span id="line-704"> protected RegionServerServices rsServices;</span> |
| <span class="source-line-no">705</span><span id="line-705"> private RegionServerAccounting rsAccounting;</span> |
| <span class="source-line-no">706</span><span id="line-706"> private long flushCheckInterval;</span> |
| <span class="source-line-no">707</span><span id="line-707"> // flushPerChanges is to prevent too many changes in memstore</span> |
| <span class="source-line-no">708</span><span id="line-708"> private long flushPerChanges;</span> |
| <span class="source-line-no">709</span><span id="line-709"> private long blockingMemStoreSize;</span> |
| <span class="source-line-no">710</span><span id="line-710"> // Used to guard closes</span> |
| <span class="source-line-no">711</span><span id="line-711"> final ReentrantReadWriteLock lock;</span> |
| <span class="source-line-no">712</span><span id="line-712"> // Used to track interruptible holders of the region lock. Currently that is only RPC handler</span> |
| <span class="source-line-no">713</span><span id="line-713"> // threads. Boolean value in map determines if lock holder can be interrupted, normally true,</span> |
| <span class="source-line-no">714</span><span id="line-714"> // but may be false when thread is transiting a critical section.</span> |
| <span class="source-line-no">715</span><span id="line-715"> final ConcurrentHashMap<Thread, Boolean> regionLockHolders;</span> |
| <span class="source-line-no">716</span><span id="line-716"></span> |
| <span class="source-line-no">717</span><span id="line-717"> // Stop updates lock</span> |
| <span class="source-line-no">718</span><span id="line-718"> private final ReentrantReadWriteLock updatesLock = new ReentrantReadWriteLock();</span> |
| <span class="source-line-no">719</span><span id="line-719"></span> |
| <span class="source-line-no">720</span><span id="line-720"> private final MultiVersionConcurrencyControl mvcc;</span> |
| <span class="source-line-no">721</span><span id="line-721"></span> |
| <span class="source-line-no">722</span><span id="line-722"> // Coprocessor host</span> |
| <span class="source-line-no">723</span><span id="line-723"> private volatile RegionCoprocessorHost coprocessorHost;</span> |
| <span class="source-line-no">724</span><span id="line-724"></span> |
| <span class="source-line-no">725</span><span id="line-725"> private TableDescriptor htableDescriptor = null;</span> |
| <span class="source-line-no">726</span><span id="line-726"> private RegionSplitPolicy splitPolicy;</span> |
| <span class="source-line-no">727</span><span id="line-727"> private RegionSplitRestriction splitRestriction;</span> |
| <span class="source-line-no">728</span><span id="line-728"> private FlushPolicy flushPolicy;</span> |
| <span class="source-line-no">729</span><span id="line-729"></span> |
| <span class="source-line-no">730</span><span id="line-730"> private final MetricsRegion metricsRegion;</span> |
| <span class="source-line-no">731</span><span id="line-731"> private final MetricsRegionWrapperImpl metricsRegionWrapper;</span> |
| <span class="source-line-no">732</span><span id="line-732"> private final Durability regionDurability;</span> |
| <span class="source-line-no">733</span><span id="line-733"> private final boolean regionStatsEnabled;</span> |
| <span class="source-line-no">734</span><span id="line-734"> // Stores the replication scope of the various column families of the table</span> |
| <span class="source-line-no">735</span><span id="line-735"> // that has non-default scope</span> |
| <span class="source-line-no">736</span><span id="line-736"> private final NavigableMap<byte[], Integer> replicationScope =</span> |
| <span class="source-line-no">737</span><span id="line-737"> new TreeMap<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">738</span><span id="line-738"></span> |
| <span class="source-line-no">739</span><span id="line-739"> private final StoreHotnessProtector storeHotnessProtector;</span> |
| <span class="source-line-no">740</span><span id="line-740"></span> |
| <span class="source-line-no">741</span><span id="line-741"> protected Optional<RegionReplicationSink> regionReplicationSink = Optional.empty();</span> |
| <span class="source-line-no">742</span><span id="line-742"></span> |
| <span class="source-line-no">743</span><span id="line-743"> /**</span> |
| <span class="source-line-no">744</span><span id="line-744"> * HRegion constructor. This constructor should only be used for testing and extensions. Instances</span> |
| <span class="source-line-no">745</span><span id="line-745"> * of HRegion should be instantiated with the {@link HRegion#createHRegion} or</span> |
| <span class="source-line-no">746</span><span id="line-746"> * {@link HRegion#openHRegion} method.</span> |
| <span class="source-line-no">747</span><span id="line-747"> * @param tableDir qualified path of directory where region should be located, usually the table</span> |
| <span class="source-line-no">748</span><span id="line-748"> * directory.</span> |
| <span class="source-line-no">749</span><span id="line-749"> * @param wal The WAL is the outbound log for any updates to the HRegion The wal file is a</span> |
| <span class="source-line-no">750</span><span id="line-750"> * logfile from the previous execution that's custom-computed for this HRegion.</span> |
| <span class="source-line-no">751</span><span id="line-751"> * The HRegionServer computes and sorts the appropriate wal info for this</span> |
| <span class="source-line-no">752</span><span id="line-752"> * HRegion. If there is a previous wal file (implying that the HRegion has been</span> |
| <span class="source-line-no">753</span><span id="line-753"> * written-to before), then read it from the supplied path.</span> |
| <span class="source-line-no">754</span><span id="line-754"> * @param fs is the filesystem.</span> |
| <span class="source-line-no">755</span><span id="line-755"> * @param confParam is global configuration settings.</span> |
| <span class="source-line-no">756</span><span id="line-756"> * @param regionInfo - RegionInfo that describes the region is new), then read them from the</span> |
| <span class="source-line-no">757</span><span id="line-757"> * supplied path.</span> |
| <span class="source-line-no">758</span><span id="line-758"> * @param htd the table descriptor</span> |
| <span class="source-line-no">759</span><span id="line-759"> * @param rsServices reference to {@link RegionServerServices} or null</span> |
| <span class="source-line-no">760</span><span id="line-760"> * @deprecated Use other constructors.</span> |
| <span class="source-line-no">761</span><span id="line-761"> */</span> |
| <span class="source-line-no">762</span><span id="line-762"> @Deprecated</span> |
| <span class="source-line-no">763</span><span id="line-763"> public HRegion(final Path tableDir, final WAL wal, final FileSystem fs,</span> |
| <span class="source-line-no">764</span><span id="line-764"> final Configuration confParam, final RegionInfo regionInfo, final TableDescriptor htd,</span> |
| <span class="source-line-no">765</span><span id="line-765"> final RegionServerServices rsServices) {</span> |
| <span class="source-line-no">766</span><span id="line-766"> this(new HRegionFileSystem(confParam, fs, tableDir, regionInfo), wal, confParam, htd,</span> |
| <span class="source-line-no">767</span><span id="line-767"> rsServices);</span> |
| <span class="source-line-no">768</span><span id="line-768"> }</span> |
| <span class="source-line-no">769</span><span id="line-769"></span> |
| <span class="source-line-no">770</span><span id="line-770"> /**</span> |
| <span class="source-line-no">771</span><span id="line-771"> * HRegion constructor. This constructor should only be used for testing and extensions. Instances</span> |
| <span class="source-line-no">772</span><span id="line-772"> * of HRegion should be instantiated with the {@link HRegion#createHRegion} or</span> |
| <span class="source-line-no">773</span><span id="line-773"> * {@link HRegion#openHRegion} method.</span> |
| <span class="source-line-no">774</span><span id="line-774"> * @param fs is the filesystem.</span> |
| <span class="source-line-no">775</span><span id="line-775"> * @param wal The WAL is the outbound log for any updates to the HRegion The wal file is a</span> |
| <span class="source-line-no">776</span><span id="line-776"> * logfile from the previous execution that's custom-computed for this HRegion.</span> |
| <span class="source-line-no">777</span><span id="line-777"> * The HRegionServer computes and sorts the appropriate wal info for this</span> |
| <span class="source-line-no">778</span><span id="line-778"> * HRegion. If there is a previous wal file (implying that the HRegion has been</span> |
| <span class="source-line-no">779</span><span id="line-779"> * written-to before), then read it from the supplied path.</span> |
| <span class="source-line-no">780</span><span id="line-780"> * @param confParam is global configuration settings.</span> |
| <span class="source-line-no">781</span><span id="line-781"> * @param htd the table descriptor</span> |
| <span class="source-line-no">782</span><span id="line-782"> * @param rsServices reference to {@link RegionServerServices} or null</span> |
| <span class="source-line-no">783</span><span id="line-783"> */</span> |
| <span class="source-line-no">784</span><span id="line-784"> public HRegion(final HRegionFileSystem fs, final WAL wal, final Configuration confParam,</span> |
| <span class="source-line-no">785</span><span id="line-785"> final TableDescriptor htd, final RegionServerServices rsServices) {</span> |
| <span class="source-line-no">786</span><span id="line-786"> if (htd == null) {</span> |
| <span class="source-line-no">787</span><span id="line-787"> throw new IllegalArgumentException("Need table descriptor");</span> |
| <span class="source-line-no">788</span><span id="line-788"> }</span> |
| <span class="source-line-no">789</span><span id="line-789"></span> |
| <span class="source-line-no">790</span><span id="line-790"> if (confParam instanceof CompoundConfiguration) {</span> |
| <span class="source-line-no">791</span><span id="line-791"> throw new IllegalArgumentException("Need original base configuration");</span> |
| <span class="source-line-no">792</span><span id="line-792"> }</span> |
| <span class="source-line-no">793</span><span id="line-793"></span> |
| <span class="source-line-no">794</span><span id="line-794"> this.wal = wal;</span> |
| <span class="source-line-no">795</span><span id="line-795"> this.fs = fs;</span> |
| <span class="source-line-no">796</span><span id="line-796"> this.mvcc = new MultiVersionConcurrencyControl(getRegionInfo().getShortNameToLog());</span> |
| <span class="source-line-no">797</span><span id="line-797"></span> |
| <span class="source-line-no">798</span><span id="line-798"> // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor</span> |
| <span class="source-line-no">799</span><span id="line-799"> this.baseConf = confParam;</span> |
| <span class="source-line-no">800</span><span id="line-800"> this.conf = new CompoundConfiguration().add(confParam).addBytesMap(htd.getValues());</span> |
| <span class="source-line-no">801</span><span id="line-801"> this.cellComparator = htd.isMetaTable()</span> |
| <span class="source-line-no">802</span><span id="line-802"> || conf.getBoolean(USE_META_CELL_COMPARATOR, DEFAULT_USE_META_CELL_COMPARATOR)</span> |
| <span class="source-line-no">803</span><span id="line-803"> ? MetaCellComparator.META_COMPARATOR</span> |
| <span class="source-line-no">804</span><span id="line-804"> : CellComparatorImpl.COMPARATOR;</span> |
| <span class="source-line-no">805</span><span id="line-805"> this.lock = new ReentrantReadWriteLock(</span> |
| <span class="source-line-no">806</span><span id="line-806"> conf.getBoolean(FAIR_REENTRANT_CLOSE_LOCK, DEFAULT_FAIR_REENTRANT_CLOSE_LOCK));</span> |
| <span class="source-line-no">807</span><span id="line-807"> this.regionLockHolders = new ConcurrentHashMap<>();</span> |
| <span class="source-line-no">808</span><span id="line-808"> this.flushCheckInterval =</span> |
| <span class="source-line-no">809</span><span id="line-809"> conf.getInt(MEMSTORE_PERIODIC_FLUSH_INTERVAL, DEFAULT_CACHE_FLUSH_INTERVAL);</span> |
| <span class="source-line-no">810</span><span id="line-810"> this.flushPerChanges = conf.getLong(MEMSTORE_FLUSH_PER_CHANGES, DEFAULT_FLUSH_PER_CHANGES);</span> |
| <span class="source-line-no">811</span><span id="line-811"> if (this.flushPerChanges > MAX_FLUSH_PER_CHANGES) {</span> |
| <span class="source-line-no">812</span><span id="line-812"> throw new IllegalArgumentException(</span> |
| <span class="source-line-no">813</span><span id="line-813"> MEMSTORE_FLUSH_PER_CHANGES + " can not exceed " + MAX_FLUSH_PER_CHANGES);</span> |
| <span class="source-line-no">814</span><span id="line-814"> }</span> |
| <span class="source-line-no">815</span><span id="line-815"> int tmpRowLockDuration =</span> |
| <span class="source-line-no">816</span><span id="line-816"> conf.getInt("hbase.rowlock.wait.duration", DEFAULT_ROWLOCK_WAIT_DURATION);</span> |
| <span class="source-line-no">817</span><span id="line-817"> if (tmpRowLockDuration <= 0) {</span> |
| <span class="source-line-no">818</span><span id="line-818"> LOG.info("Found hbase.rowlock.wait.duration set to {}. values <= 0 will cause all row "</span> |
| <span class="source-line-no">819</span><span id="line-819"> + "locking to fail. Treating it as 1ms to avoid region failure.", tmpRowLockDuration);</span> |
| <span class="source-line-no">820</span><span id="line-820"> tmpRowLockDuration = 1;</span> |
| <span class="source-line-no">821</span><span id="line-821"> }</span> |
| <span class="source-line-no">822</span><span id="line-822"> this.rowLockWaitDuration = tmpRowLockDuration;</span> |
| <span class="source-line-no">823</span><span id="line-823"></span> |
| <span class="source-line-no">824</span><span id="line-824"> this.smallestReadPointCalcLock = new ReadPointCalculationLock(conf);</span> |
| <span class="source-line-no">825</span><span id="line-825"></span> |
| <span class="source-line-no">826</span><span id="line-826"> this.isLoadingCfsOnDemandDefault = conf.getBoolean(LOAD_CFS_ON_DEMAND_CONFIG_KEY, true);</span> |
| <span class="source-line-no">827</span><span id="line-827"> this.htableDescriptor = htd;</span> |
| <span class="source-line-no">828</span><span id="line-828"> Set<byte[]> families = this.htableDescriptor.getColumnFamilyNames();</span> |
| <span class="source-line-no">829</span><span id="line-829"> for (byte[] family : families) {</span> |
| <span class="source-line-no">830</span><span id="line-830"> if (!replicationScope.containsKey(family)) {</span> |
| <span class="source-line-no">831</span><span id="line-831"> int scope = htd.getColumnFamily(family).getScope();</span> |
| <span class="source-line-no">832</span><span id="line-832"> // Only store those families that has NON-DEFAULT scope</span> |
| <span class="source-line-no">833</span><span id="line-833"> if (scope != REPLICATION_SCOPE_LOCAL) {</span> |
| <span class="source-line-no">834</span><span id="line-834"> // Do a copy before storing it here.</span> |
| <span class="source-line-no">835</span><span id="line-835"> replicationScope.put(Bytes.copy(family), scope);</span> |
| <span class="source-line-no">836</span><span id="line-836"> }</span> |
| <span class="source-line-no">837</span><span id="line-837"> }</span> |
| <span class="source-line-no">838</span><span id="line-838"> }</span> |
| <span class="source-line-no">839</span><span id="line-839"></span> |
| <span class="source-line-no">840</span><span id="line-840"> this.rsServices = rsServices;</span> |
| <span class="source-line-no">841</span><span id="line-841"> if (this.rsServices != null) {</span> |
| <span class="source-line-no">842</span><span id="line-842"> this.blockCache = rsServices.getBlockCache().orElse(null);</span> |
| <span class="source-line-no">843</span><span id="line-843"> this.mobFileCache = rsServices.getMobFileCache().orElse(null);</span> |
| <span class="source-line-no">844</span><span id="line-844"> }</span> |
| <span class="source-line-no">845</span><span id="line-845"> this.regionServicesForStores = new RegionServicesForStores(this, rsServices);</span> |
| <span class="source-line-no">846</span><span id="line-846"></span> |
| <span class="source-line-no">847</span><span id="line-847"> setHTableSpecificConf();</span> |
| <span class="source-line-no">848</span><span id="line-848"> this.scannerReadPoints = new ConcurrentHashMap<>();</span> |
| <span class="source-line-no">849</span><span id="line-849"></span> |
| <span class="source-line-no">850</span><span id="line-850"> this.busyWaitDuration = conf.getLong("hbase.busy.wait.duration", DEFAULT_BUSY_WAIT_DURATION);</span> |
| <span class="source-line-no">851</span><span id="line-851"> this.maxBusyWaitMultiplier = conf.getInt("hbase.busy.wait.multiplier.max", 2);</span> |
| <span class="source-line-no">852</span><span id="line-852"> if (busyWaitDuration * maxBusyWaitMultiplier <= 0L) {</span> |
| <span class="source-line-no">853</span><span id="line-853"> throw new IllegalArgumentException("Invalid hbase.busy.wait.duration (" + busyWaitDuration</span> |
| <span class="source-line-no">854</span><span id="line-854"> + ") or hbase.busy.wait.multiplier.max (" + maxBusyWaitMultiplier</span> |
| <span class="source-line-no">855</span><span id="line-855"> + "). Their product should be positive");</span> |
| <span class="source-line-no">856</span><span id="line-856"> }</span> |
| <span class="source-line-no">857</span><span id="line-857"> this.maxBusyWaitDuration =</span> |
| <span class="source-line-no">858</span><span id="line-858"> conf.getLong("hbase.ipc.client.call.purge.timeout", 2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);</span> |
| <span class="source-line-no">859</span><span id="line-859"></span> |
| <span class="source-line-no">860</span><span id="line-860"> /*</span> |
| <span class="source-line-no">861</span><span id="line-861"> * timestamp.slop provides a server-side constraint on the timestamp. This assumes that you base</span> |
| <span class="source-line-no">862</span><span id="line-862"> * your TS around EnvironmentEdgeManager.currentTime(). In this case, throw an error to the user</span> |
| <span class="source-line-no">863</span><span id="line-863"> * if the user-specified TS is newer than now + slop. LATEST_TIMESTAMP == don't use this</span> |
| <span class="source-line-no">864</span><span id="line-864"> * functionality</span> |
| <span class="source-line-no">865</span><span id="line-865"> */</span> |
| <span class="source-line-no">866</span><span id="line-866"> this.timestampSlop =</span> |
| <span class="source-line-no">867</span><span id="line-867"> conf.getLong("hbase.hregion.keyvalue.timestamp.slop.millisecs", HConstants.LATEST_TIMESTAMP);</span> |
| <span class="source-line-no">868</span><span id="line-868"></span> |
| <span class="source-line-no">869</span><span id="line-869"> this.storeHotnessProtector = new StoreHotnessProtector(this, conf);</span> |
| <span class="source-line-no">870</span><span id="line-870"></span> |
| <span class="source-line-no">871</span><span id="line-871"> boolean forceSync = conf.getBoolean(WAL_HSYNC_CONF_KEY, DEFAULT_WAL_HSYNC);</span> |
| <span class="source-line-no">872</span><span id="line-872"> /**</span> |
| <span class="source-line-no">873</span><span id="line-873"> * This is the global default value for durability. All tables/mutations not defining a</span> |
| <span class="source-line-no">874</span><span id="line-874"> * durability or using USE_DEFAULT will default to this value.</span> |
| <span class="source-line-no">875</span><span id="line-875"> */</span> |
| <span class="source-line-no">876</span><span id="line-876"> Durability defaultDurability = forceSync ? Durability.FSYNC_WAL : Durability.SYNC_WAL;</span> |
| <span class="source-line-no">877</span><span id="line-877"> this.regionDurability = this.htableDescriptor.getDurability() == Durability.USE_DEFAULT</span> |
| <span class="source-line-no">878</span><span id="line-878"> ? defaultDurability</span> |
| <span class="source-line-no">879</span><span id="line-879"> : this.htableDescriptor.getDurability();</span> |
| <span class="source-line-no">880</span><span id="line-880"></span> |
| <span class="source-line-no">881</span><span id="line-881"> decorateRegionConfiguration(conf);</span> |
| <span class="source-line-no">882</span><span id="line-882"> if (rsServices != null) {</span> |
| <span class="source-line-no">883</span><span id="line-883"> this.rsAccounting = this.rsServices.getRegionServerAccounting();</span> |
| <span class="source-line-no">884</span><span id="line-884"> // don't initialize coprocessors if not running within a regionserver</span> |
| <span class="source-line-no">885</span><span id="line-885"> // TODO: revisit if coprocessors should load in other cases</span> |
| <span class="source-line-no">886</span><span id="line-886"> this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);</span> |
| <span class="source-line-no">887</span><span id="line-887"> this.metricsRegionWrapper = new MetricsRegionWrapperImpl(this);</span> |
| <span class="source-line-no">888</span><span id="line-888"> this.metricsRegion = new MetricsRegion(this.metricsRegionWrapper, conf);</span> |
| <span class="source-line-no">889</span><span id="line-889"> } else {</span> |
| <span class="source-line-no">890</span><span id="line-890"> this.metricsRegionWrapper = null;</span> |
| <span class="source-line-no">891</span><span id="line-891"> this.metricsRegion = null;</span> |
| <span class="source-line-no">892</span><span id="line-892"> }</span> |
| <span class="source-line-no">893</span><span id="line-893"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">894</span><span id="line-894"> // Write out region name, its encoded name and storeHotnessProtector as string.</span> |
| <span class="source-line-no">895</span><span id="line-895"> LOG.debug("Instantiated " + this + "; " + storeHotnessProtector.toString());</span> |
| <span class="source-line-no">896</span><span id="line-896"> }</span> |
| <span class="source-line-no">897</span><span id="line-897"></span> |
| <span class="source-line-no">898</span><span id="line-898"> configurationManager = null;</span> |
| <span class="source-line-no">899</span><span id="line-899"></span> |
| <span class="source-line-no">900</span><span id="line-900"> // disable stats tracking system tables, but check the config for everything else</span> |
| <span class="source-line-no">901</span><span id="line-901"> this.regionStatsEnabled = htd.getTableName().getNamespaceAsString()</span> |
| <span class="source-line-no">902</span><span id="line-902"> .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)</span> |
| <span class="source-line-no">903</span><span id="line-903"> ? false</span> |
| <span class="source-line-no">904</span><span id="line-904"> : conf.getBoolean(HConstants.ENABLE_CLIENT_BACKPRESSURE,</span> |
| <span class="source-line-no">905</span><span id="line-905"> HConstants.DEFAULT_ENABLE_CLIENT_BACKPRESSURE);</span> |
| <span class="source-line-no">906</span><span id="line-906"></span> |
| <span class="source-line-no">907</span><span id="line-907"> this.maxCellSize = conf.getLong(HBASE_MAX_CELL_SIZE_KEY, DEFAULT_MAX_CELL_SIZE);</span> |
| <span class="source-line-no">908</span><span id="line-908"> this.miniBatchSize =</span> |
| <span class="source-line-no">909</span><span id="line-909"> conf.getInt(HBASE_REGIONSERVER_MINIBATCH_SIZE, DEFAULT_HBASE_REGIONSERVER_MINIBATCH_SIZE);</span> |
| <span class="source-line-no">910</span><span id="line-910"></span> |
| <span class="source-line-no">911</span><span id="line-911"> // recover the metrics of read and write requests count if they were retained</span> |
| <span class="source-line-no">912</span><span id="line-912"> if (rsServices != null && rsServices.getRegionServerAccounting() != null) {</span> |
| <span class="source-line-no">913</span><span id="line-913"> Pair<Long, Long> retainedRWRequestsCnt = rsServices.getRegionServerAccounting()</span> |
| <span class="source-line-no">914</span><span id="line-914"> .getRetainedRegionRWRequestsCnt().get(getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">915</span><span id="line-915"> if (retainedRWRequestsCnt != null) {</span> |
| <span class="source-line-no">916</span><span id="line-916"> this.addReadRequestsCount(retainedRWRequestsCnt.getFirst());</span> |
| <span class="source-line-no">917</span><span id="line-917"> this.addWriteRequestsCount(retainedRWRequestsCnt.getSecond());</span> |
| <span class="source-line-no">918</span><span id="line-918"> // remove them since won't use again</span> |
| <span class="source-line-no">919</span><span id="line-919"> rsServices.getRegionServerAccounting().getRetainedRegionRWRequestsCnt()</span> |
| <span class="source-line-no">920</span><span id="line-920"> .remove(getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">921</span><span id="line-921"> }</span> |
| <span class="source-line-no">922</span><span id="line-922"> }</span> |
| <span class="source-line-no">923</span><span id="line-923"></span> |
| <span class="source-line-no">924</span><span id="line-924"> minBlockSizeBytes = Arrays.stream(this.htableDescriptor.getColumnFamilies())</span> |
| <span class="source-line-no">925</span><span id="line-925"> .mapToInt(ColumnFamilyDescriptor::getBlocksize).min().orElse(HConstants.DEFAULT_BLOCKSIZE);</span> |
| <span class="source-line-no">926</span><span id="line-926"> }</span> |
| <span class="source-line-no">927</span><span id="line-927"></span> |
| <span class="source-line-no">928</span><span id="line-928"> private void setHTableSpecificConf() {</span> |
| <span class="source-line-no">929</span><span id="line-929"> if (this.htableDescriptor == null) {</span> |
| <span class="source-line-no">930</span><span id="line-930"> return;</span> |
| <span class="source-line-no">931</span><span id="line-931"> }</span> |
| <span class="source-line-no">932</span><span id="line-932"> long flushSize = this.htableDescriptor.getMemStoreFlushSize();</span> |
| <span class="source-line-no">933</span><span id="line-933"></span> |
| <span class="source-line-no">934</span><span id="line-934"> if (flushSize <= 0) {</span> |
| <span class="source-line-no">935</span><span id="line-935"> flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,</span> |
| <span class="source-line-no">936</span><span id="line-936"> TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE);</span> |
| <span class="source-line-no">937</span><span id="line-937"> }</span> |
| <span class="source-line-no">938</span><span id="line-938"> this.memstoreFlushSize = flushSize;</span> |
| <span class="source-line-no">939</span><span id="line-939"> long mult = conf.getLong(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER,</span> |
| <span class="source-line-no">940</span><span id="line-940"> HConstants.DEFAULT_HREGION_MEMSTORE_BLOCK_MULTIPLIER);</span> |
| <span class="source-line-no">941</span><span id="line-941"> this.blockingMemStoreSize = this.memstoreFlushSize * mult;</span> |
| <span class="source-line-no">942</span><span id="line-942"> }</span> |
| <span class="source-line-no">943</span><span id="line-943"></span> |
| <span class="source-line-no">944</span><span id="line-944"> /**</span> |
| <span class="source-line-no">945</span><span id="line-945"> * Initialize this region. Used only by tests and SplitTransaction to reopen the region. You</span> |
| <span class="source-line-no">946</span><span id="line-946"> * should use createHRegion() or openHRegion()</span> |
| <span class="source-line-no">947</span><span id="line-947"> * @return What the next sequence (edit) id should be.</span> |
| <span class="source-line-no">948</span><span id="line-948"> * @throws IOException e</span> |
| <span class="source-line-no">949</span><span id="line-949"> * @deprecated use HRegion.createHRegion() or HRegion.openHRegion()</span> |
| <span class="source-line-no">950</span><span id="line-950"> */</span> |
| <span class="source-line-no">951</span><span id="line-951"> @Deprecated</span> |
| <span class="source-line-no">952</span><span id="line-952"> public long initialize() throws IOException {</span> |
| <span class="source-line-no">953</span><span id="line-953"> return initialize(null);</span> |
| <span class="source-line-no">954</span><span id="line-954"> }</span> |
| <span class="source-line-no">955</span><span id="line-955"></span> |
| <span class="source-line-no">956</span><span id="line-956"> /**</span> |
| <span class="source-line-no">957</span><span id="line-957"> * Initialize this region.</span> |
| <span class="source-line-no">958</span><span id="line-958"> * @param reporter Tickle every so often if initialize is taking a while.</span> |
| <span class="source-line-no">959</span><span id="line-959"> * @return What the next sequence (edit) id should be.</span> |
| <span class="source-line-no">960</span><span id="line-960"> */</span> |
| <span class="source-line-no">961</span><span id="line-961"> long initialize(final CancelableProgressable reporter) throws IOException {</span> |
| <span class="source-line-no">962</span><span id="line-962"></span> |
| <span class="source-line-no">963</span><span id="line-963"> // Refuse to open the region if there is no column family in the table</span> |
| <span class="source-line-no">964</span><span id="line-964"> if (htableDescriptor.getColumnFamilyCount() == 0) {</span> |
| <span class="source-line-no">965</span><span id="line-965"> throw new DoNotRetryIOException("Table " + htableDescriptor.getTableName().getNameAsString()</span> |
| <span class="source-line-no">966</span><span id="line-966"> + " should have at least one column family.");</span> |
| <span class="source-line-no">967</span><span id="line-967"> }</span> |
| <span class="source-line-no">968</span><span id="line-968"></span> |
| <span class="source-line-no">969</span><span id="line-969"> MonitoredTask status =</span> |
| <span class="source-line-no">970</span><span id="line-970"> TaskMonitor.get().createStatus("Initializing region " + this, false, true);</span> |
| <span class="source-line-no">971</span><span id="line-971"> long nextSeqId = -1;</span> |
| <span class="source-line-no">972</span><span id="line-972"> try {</span> |
| <span class="source-line-no">973</span><span id="line-973"> nextSeqId = initializeRegionInternals(reporter, status);</span> |
| <span class="source-line-no">974</span><span id="line-974"> return nextSeqId;</span> |
| <span class="source-line-no">975</span><span id="line-975"> } catch (IOException e) {</span> |
| <span class="source-line-no">976</span><span id="line-976"> LOG.warn("Failed initialize of region= {}, starting to roll back memstore",</span> |
| <span class="source-line-no">977</span><span id="line-977"> getRegionInfo().getRegionNameAsString(), e);</span> |
| <span class="source-line-no">978</span><span id="line-978"> // global memstore size will be decreased when dropping memstore</span> |
| <span class="source-line-no">979</span><span id="line-979"> try {</span> |
| <span class="source-line-no">980</span><span id="line-980"> // drop the memory used by memstore if open region fails</span> |
| <span class="source-line-no">981</span><span id="line-981"> dropMemStoreContents();</span> |
| <span class="source-line-no">982</span><span id="line-982"> } catch (IOException ioE) {</span> |
| <span class="source-line-no">983</span><span id="line-983"> if (conf.getBoolean(MemStoreLAB.USEMSLAB_KEY, MemStoreLAB.USEMSLAB_DEFAULT)) {</span> |
| <span class="source-line-no">984</span><span id="line-984"> LOG.warn(</span> |
| <span class="source-line-no">985</span><span id="line-985"> "Failed drop memstore of region= {}, "</span> |
| <span class="source-line-no">986</span><span id="line-986"> + "some chunks may not released forever since MSLAB is enabled",</span> |
| <span class="source-line-no">987</span><span id="line-987"> getRegionInfo().getRegionNameAsString());</span> |
| <span class="source-line-no">988</span><span id="line-988"> }</span> |
| <span class="source-line-no">989</span><span id="line-989"></span> |
| <span class="source-line-no">990</span><span id="line-990"> }</span> |
| <span class="source-line-no">991</span><span id="line-991"> if (metricsTableRequests != null) {</span> |
| <span class="source-line-no">992</span><span id="line-992"> metricsTableRequests.removeRegistry();</span> |
| <span class="source-line-no">993</span><span id="line-993"> }</span> |
| <span class="source-line-no">994</span><span id="line-994"> throw e;</span> |
| <span class="source-line-no">995</span><span id="line-995"> } finally {</span> |
| <span class="source-line-no">996</span><span id="line-996"> // nextSeqid will be -1 if the initialization fails.</span> |
| <span class="source-line-no">997</span><span id="line-997"> // At least it will be 0 otherwise.</span> |
| <span class="source-line-no">998</span><span id="line-998"> if (nextSeqId == -1) {</span> |
| <span class="source-line-no">999</span><span id="line-999"> status.abort("Exception during region " + getRegionInfo().getRegionNameAsString()</span> |
| <span class="source-line-no">1000</span><span id="line-1000"> + " initialization.");</span> |
| <span class="source-line-no">1001</span><span id="line-1001"> }</span> |
| <span class="source-line-no">1002</span><span id="line-1002"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">1003</span><span id="line-1003"> LOG.debug("Region open journal for {}:\n{}", this.getRegionInfo().getEncodedName(),</span> |
| <span class="source-line-no">1004</span><span id="line-1004"> status.prettyPrintJournal());</span> |
| <span class="source-line-no">1005</span><span id="line-1005"> }</span> |
| <span class="source-line-no">1006</span><span id="line-1006"> status.cleanup();</span> |
| <span class="source-line-no">1007</span><span id="line-1007"> }</span> |
| <span class="source-line-no">1008</span><span id="line-1008"> }</span> |
| <span class="source-line-no">1009</span><span id="line-1009"></span> |
| <span class="source-line-no">1010</span><span id="line-1010"> private long initializeRegionInternals(final CancelableProgressable reporter,</span> |
| <span class="source-line-no">1011</span><span id="line-1011"> final MonitoredTask status) throws IOException {</span> |
| <span class="source-line-no">1012</span><span id="line-1012"> if (coprocessorHost != null) {</span> |
| <span class="source-line-no">1013</span><span id="line-1013"> status.setStatus("Running coprocessor pre-open hook");</span> |
| <span class="source-line-no">1014</span><span id="line-1014"> coprocessorHost.preOpen();</span> |
| <span class="source-line-no">1015</span><span id="line-1015"> }</span> |
| <span class="source-line-no">1016</span><span id="line-1016"></span> |
| <span class="source-line-no">1017</span><span id="line-1017"> String policyName = this.conf.get(REGION_STORAGE_POLICY_KEY, DEFAULT_REGION_STORAGE_POLICY);</span> |
| <span class="source-line-no">1018</span><span id="line-1018"> this.fs.setStoragePolicy(policyName.trim());</span> |
| <span class="source-line-no">1019</span><span id="line-1019"></span> |
| <span class="source-line-no">1020</span><span id="line-1020"> // Write HRI to a file in case we need to recover hbase:meta</span> |
| <span class="source-line-no">1021</span><span id="line-1021"> // Only the primary replica should write .regioninfo</span> |
| <span class="source-line-no">1022</span><span id="line-1022"> if (this.getRegionInfo().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {</span> |
| <span class="source-line-no">1023</span><span id="line-1023"> status.setStatus("Writing region info on filesystem");</span> |
| <span class="source-line-no">1024</span><span id="line-1024"> fs.checkRegionInfoOnFilesystem();</span> |
| <span class="source-line-no">1025</span><span id="line-1025"> }</span> |
| <span class="source-line-no">1026</span><span id="line-1026"></span> |
| <span class="source-line-no">1027</span><span id="line-1027"> // Initialize all the HStores</span> |
| <span class="source-line-no">1028</span><span id="line-1028"> status.setStatus("Initializing all the Stores");</span> |
| <span class="source-line-no">1029</span><span id="line-1029"> long maxSeqId = initializeStores(reporter, status);</span> |
| <span class="source-line-no">1030</span><span id="line-1030"> this.mvcc.advanceTo(maxSeqId);</span> |
| <span class="source-line-no">1031</span><span id="line-1031"> if (!isRestoredRegion && ServerRegionReplicaUtil.shouldReplayRecoveredEdits(this)) {</span> |
| <span class="source-line-no">1032</span><span id="line-1032"> Collection<HStore> stores = this.stores.values();</span> |
| <span class="source-line-no">1033</span><span id="line-1033"> try {</span> |
| <span class="source-line-no">1034</span><span id="line-1034"> // update the stores that we are replaying</span> |
| <span class="source-line-no">1035</span><span id="line-1035"> LOG.debug("replaying wal for " + this.getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">1036</span><span id="line-1036"> stores.forEach(HStore::startReplayingFromWAL);</span> |
| <span class="source-line-no">1037</span><span id="line-1037"> // Recover any edits if available.</span> |
| <span class="source-line-no">1038</span><span id="line-1038"> maxSeqId =</span> |
| <span class="source-line-no">1039</span><span id="line-1039"> Math.max(maxSeqId, replayRecoveredEditsIfAny(maxSeqIdInStores, reporter, status));</span> |
| <span class="source-line-no">1040</span><span id="line-1040"> // Recover any hfiles if available</span> |
| <span class="source-line-no">1041</span><span id="line-1041"> maxSeqId = Math.max(maxSeqId, loadRecoveredHFilesIfAny(stores));</span> |
| <span class="source-line-no">1042</span><span id="line-1042"> // Make sure mvcc is up to max.</span> |
| <span class="source-line-no">1043</span><span id="line-1043"> this.mvcc.advanceTo(maxSeqId);</span> |
| <span class="source-line-no">1044</span><span id="line-1044"> } finally {</span> |
| <span class="source-line-no">1045</span><span id="line-1045"> LOG.debug("stopping wal replay for " + this.getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">1046</span><span id="line-1046"> // update the stores that we are done replaying</span> |
| <span class="source-line-no">1047</span><span id="line-1047"> stores.forEach(HStore::stopReplayingFromWAL);</span> |
| <span class="source-line-no">1048</span><span id="line-1048"> }</span> |
| <span class="source-line-no">1049</span><span id="line-1049"> }</span> |
| <span class="source-line-no">1050</span><span id="line-1050"> this.lastReplayedOpenRegionSeqId = maxSeqId;</span> |
| <span class="source-line-no">1051</span><span id="line-1051"></span> |
| <span class="source-line-no">1052</span><span id="line-1052"> this.writestate.setReadOnly(ServerRegionReplicaUtil.isReadOnly(this));</span> |
| <span class="source-line-no">1053</span><span id="line-1053"> this.writestate.flushRequested = false;</span> |
| <span class="source-line-no">1054</span><span id="line-1054"> this.writestate.compacting.set(0);</span> |
| <span class="source-line-no">1055</span><span id="line-1055"></span> |
| <span class="source-line-no">1056</span><span id="line-1056"> if (this.writestate.writesEnabled) {</span> |
| <span class="source-line-no">1057</span><span id="line-1057"> LOG.debug("Cleaning up temporary data for " + this.getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">1058</span><span id="line-1058"> // Remove temporary data left over from old regions</span> |
| <span class="source-line-no">1059</span><span id="line-1059"> status.setStatus("Cleaning up temporary data from old regions");</span> |
| <span class="source-line-no">1060</span><span id="line-1060"> fs.cleanupTempDir();</span> |
| <span class="source-line-no">1061</span><span id="line-1061"> }</span> |
| <span class="source-line-no">1062</span><span id="line-1062"></span> |
| <span class="source-line-no">1063</span><span id="line-1063"> // Initialize split policy</span> |
| <span class="source-line-no">1064</span><span id="line-1064"> this.splitPolicy = RegionSplitPolicy.create(this, conf);</span> |
| <span class="source-line-no">1065</span><span id="line-1065"></span> |
| <span class="source-line-no">1066</span><span id="line-1066"> // Initialize split restriction</span> |
| <span class="source-line-no">1067</span><span id="line-1067"> splitRestriction = RegionSplitRestriction.create(getTableDescriptor(), conf);</span> |
| <span class="source-line-no">1068</span><span id="line-1068"></span> |
| <span class="source-line-no">1069</span><span id="line-1069"> // Initialize flush policy</span> |
| <span class="source-line-no">1070</span><span id="line-1070"> this.flushPolicy = FlushPolicyFactory.create(this, conf);</span> |
| <span class="source-line-no">1071</span><span id="line-1071"></span> |
| <span class="source-line-no">1072</span><span id="line-1072"> long lastFlushTime = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">1073</span><span id="line-1073"> for (HStore store : stores.values()) {</span> |
| <span class="source-line-no">1074</span><span id="line-1074"> this.lastStoreFlushTimeMap.put(store, lastFlushTime);</span> |
| <span class="source-line-no">1075</span><span id="line-1075"> }</span> |
| <span class="source-line-no">1076</span><span id="line-1076"></span> |
| <span class="source-line-no">1077</span><span id="line-1077"> // Use maximum of log sequenceid or that which was found in stores</span> |
| <span class="source-line-no">1078</span><span id="line-1078"> // (particularly if no recovered edits, seqid will be -1).</span> |
| <span class="source-line-no">1079</span><span id="line-1079"> long nextSeqId = maxSeqId + 1;</span> |
| <span class="source-line-no">1080</span><span id="line-1080"> if (!isRestoredRegion) {</span> |
| <span class="source-line-no">1081</span><span id="line-1081"> // always get openSeqNum from the default replica, even if we are secondary replicas</span> |
| <span class="source-line-no">1082</span><span id="line-1082"> long maxSeqIdFromFile = WALSplitUtil.getMaxRegionSequenceId(conf,</span> |
| <span class="source-line-no">1083</span><span id="line-1083"> RegionReplicaUtil.getRegionInfoForDefaultReplica(getRegionInfo()), this::getFilesystem,</span> |
| <span class="source-line-no">1084</span><span id="line-1084"> this::getWalFileSystem);</span> |
| <span class="source-line-no">1085</span><span id="line-1085"> nextSeqId = Math.max(maxSeqId, maxSeqIdFromFile) + 1;</span> |
| <span class="source-line-no">1086</span><span id="line-1086"> // The openSeqNum will always be increase even for read only region, as we rely on it to</span> |
| <span class="source-line-no">1087</span><span id="line-1087"> // determine whether a region has been successfully reopened, so here we always need to update</span> |
| <span class="source-line-no">1088</span><span id="line-1088"> // the max sequence id file.</span> |
| <span class="source-line-no">1089</span><span id="line-1089"> if (RegionReplicaUtil.isDefaultReplica(getRegionInfo())) {</span> |
| <span class="source-line-no">1090</span><span id="line-1090"> LOG.debug("writing seq id for {}", this.getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">1091</span><span id="line-1091"> WALSplitUtil.writeRegionSequenceIdFile(getWalFileSystem(), getWALRegionDir(),</span> |
| <span class="source-line-no">1092</span><span id="line-1092"> nextSeqId - 1);</span> |
| <span class="source-line-no">1093</span><span id="line-1093"> // This means we have replayed all the recovered edits and also written out the max sequence</span> |
| <span class="source-line-no">1094</span><span id="line-1094"> // id file, let's delete the wrong directories introduced in HBASE-20734, see HBASE-22617</span> |
| <span class="source-line-no">1095</span><span id="line-1095"> // for more details.</span> |
| <span class="source-line-no">1096</span><span id="line-1096"> Path wrongRegionWALDir = CommonFSUtils.getWrongWALRegionDir(conf,</span> |
| <span class="source-line-no">1097</span><span id="line-1097"> getRegionInfo().getTable(), getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">1098</span><span id="line-1098"> FileSystem walFs = getWalFileSystem();</span> |
| <span class="source-line-no">1099</span><span id="line-1099"> if (walFs.exists(wrongRegionWALDir)) {</span> |
| <span class="source-line-no">1100</span><span id="line-1100"> if (!walFs.delete(wrongRegionWALDir, true)) {</span> |
| <span class="source-line-no">1101</span><span id="line-1101"> LOG.debug("Failed to clean up wrong region WAL directory {}", wrongRegionWALDir);</span> |
| <span class="source-line-no">1102</span><span id="line-1102"> }</span> |
| <span class="source-line-no">1103</span><span id="line-1103"> }</span> |
| <span class="source-line-no">1104</span><span id="line-1104"> } else {</span> |
| <span class="source-line-no">1105</span><span id="line-1105"> lastReplayedSequenceId = nextSeqId - 1;</span> |
| <span class="source-line-no">1106</span><span id="line-1106"> replayLock = new ReentrantLock();</span> |
| <span class="source-line-no">1107</span><span id="line-1107"> }</span> |
| <span class="source-line-no">1108</span><span id="line-1108"> initializeRegionReplicationSink(reporter, status);</span> |
| <span class="source-line-no">1109</span><span id="line-1109"> }</span> |
| <span class="source-line-no">1110</span><span id="line-1110"></span> |
| <span class="source-line-no">1111</span><span id="line-1111"> LOG.info("Opened {}; next sequenceid={}; {}, {}", this.getRegionInfo().getShortNameToLog(),</span> |
| <span class="source-line-no">1112</span><span id="line-1112"> nextSeqId, this.splitPolicy, this.flushPolicy);</span> |
| <span class="source-line-no">1113</span><span id="line-1113"></span> |
| <span class="source-line-no">1114</span><span id="line-1114"> // A region can be reopened if failed a split; reset flags</span> |
| <span class="source-line-no">1115</span><span id="line-1115"> this.closing.set(false);</span> |
| <span class="source-line-no">1116</span><span id="line-1116"> this.closed.set(false);</span> |
| <span class="source-line-no">1117</span><span id="line-1117"></span> |
| <span class="source-line-no">1118</span><span id="line-1118"> if (coprocessorHost != null) {</span> |
| <span class="source-line-no">1119</span><span id="line-1119"> LOG.debug("Running coprocessor post-open hooks for " + this.getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">1120</span><span id="line-1120"> status.setStatus("Running coprocessor post-open hooks");</span> |
| <span class="source-line-no">1121</span><span id="line-1121"> coprocessorHost.postOpen();</span> |
| <span class="source-line-no">1122</span><span id="line-1122"> }</span> |
| <span class="source-line-no">1123</span><span id="line-1123"></span> |
| <span class="source-line-no">1124</span><span id="line-1124"> metricsTableRequests = new MetricsTableRequests(htableDescriptor.getTableName(), conf);</span> |
| <span class="source-line-no">1125</span><span id="line-1125"></span> |
| <span class="source-line-no">1126</span><span id="line-1126"> status.markComplete("Region opened successfully");</span> |
| <span class="source-line-no">1127</span><span id="line-1127"> return nextSeqId;</span> |
| <span class="source-line-no">1128</span><span id="line-1128"> }</span> |
| <span class="source-line-no">1129</span><span id="line-1129"></span> |
| <span class="source-line-no">1130</span><span id="line-1130"> private void initializeRegionReplicationSink(CancelableProgressable reporter,</span> |
| <span class="source-line-no">1131</span><span id="line-1131"> MonitoredTask status) {</span> |
| <span class="source-line-no">1132</span><span id="line-1132"> RegionServerServices rss = getRegionServerServices();</span> |
| <span class="source-line-no">1133</span><span id="line-1133"> TableDescriptor td = getTableDescriptor();</span> |
| <span class="source-line-no">1134</span><span id="line-1134"> int regionReplication = td.getRegionReplication();</span> |
| <span class="source-line-no">1135</span><span id="line-1135"> RegionInfo regionInfo = getRegionInfo();</span> |
| <span class="source-line-no">1136</span><span id="line-1136"> if (</span> |
| <span class="source-line-no">1137</span><span id="line-1137"> regionReplication <= 1 || !RegionReplicaUtil.isDefaultReplica(regionInfo)</span> |
| <span class="source-line-no">1138</span><span id="line-1138"> || !ServerRegionReplicaUtil.isRegionReplicaReplicationEnabled(conf, regionInfo.getTable())</span> |
| <span class="source-line-no">1139</span><span id="line-1139"> || rss == null</span> |
| <span class="source-line-no">1140</span><span id="line-1140"> ) {</span> |
| <span class="source-line-no">1141</span><span id="line-1141"> regionReplicationSink = Optional.empty();</span> |
| <span class="source-line-no">1142</span><span id="line-1142"> return;</span> |
| <span class="source-line-no">1143</span><span id="line-1143"> }</span> |
| <span class="source-line-no">1144</span><span id="line-1144"> status.setStatus("Initializaing region replication sink");</span> |
| <span class="source-line-no">1145</span><span id="line-1145"> regionReplicationSink = Optional.of(new RegionReplicationSink(conf, regionInfo, td,</span> |
| <span class="source-line-no">1146</span><span id="line-1146"> rss.getRegionReplicationBufferManager(), () -> rss.getFlushRequester().requestFlush(this,</span> |
| <span class="source-line-no">1147</span><span id="line-1147"> new ArrayList<>(td.getColumnFamilyNames()), FlushLifeCycleTracker.DUMMY),</span> |
| <span class="source-line-no">1148</span><span id="line-1148"> rss.getAsyncClusterConnection()));</span> |
| <span class="source-line-no">1149</span><span id="line-1149"> }</span> |
| <span class="source-line-no">1150</span><span id="line-1150"></span> |
| <span class="source-line-no">1151</span><span id="line-1151"> /**</span> |
| <span class="source-line-no">1152</span><span id="line-1152"> * Open all Stores.</span> |
| <span class="source-line-no">1153</span><span id="line-1153"> * @return Highest sequenceId found out in a Store.</span> |
| <span class="source-line-no">1154</span><span id="line-1154"> */</span> |
| <span class="source-line-no">1155</span><span id="line-1155"> private long initializeStores(CancelableProgressable reporter, MonitoredTask status)</span> |
| <span class="source-line-no">1156</span><span id="line-1156"> throws IOException {</span> |
| <span class="source-line-no">1157</span><span id="line-1157"> return initializeStores(reporter, status, false);</span> |
| <span class="source-line-no">1158</span><span id="line-1158"> }</span> |
| <span class="source-line-no">1159</span><span id="line-1159"></span> |
| <span class="source-line-no">1160</span><span id="line-1160"> private long initializeStores(CancelableProgressable reporter, MonitoredTask status,</span> |
| <span class="source-line-no">1161</span><span id="line-1161"> boolean warmup) throws IOException {</span> |
| <span class="source-line-no">1162</span><span id="line-1162"> // Load in all the HStores.</span> |
| <span class="source-line-no">1163</span><span id="line-1163"> long maxSeqId = -1;</span> |
| <span class="source-line-no">1164</span><span id="line-1164"> // initialized to -1 so that we pick up MemstoreTS from column families</span> |
| <span class="source-line-no">1165</span><span id="line-1165"> long maxMemstoreTS = -1;</span> |
| <span class="source-line-no">1166</span><span id="line-1166"></span> |
| <span class="source-line-no">1167</span><span id="line-1167"> if (htableDescriptor.getColumnFamilyCount() != 0) {</span> |
| <span class="source-line-no">1168</span><span id="line-1168"> // initialize the thread pool for opening stores in parallel.</span> |
| <span class="source-line-no">1169</span><span id="line-1169"> ThreadPoolExecutor storeOpenerThreadPool =</span> |
| <span class="source-line-no">1170</span><span id="line-1170"> getStoreOpenAndCloseThreadPool("StoreOpener-" + this.getRegionInfo().getShortNameToLog());</span> |
| <span class="source-line-no">1171</span><span id="line-1171"> CompletionService<HStore> completionService =</span> |
| <span class="source-line-no">1172</span><span id="line-1172"> new ExecutorCompletionService<>(storeOpenerThreadPool);</span> |
| <span class="source-line-no">1173</span><span id="line-1173"></span> |
| <span class="source-line-no">1174</span><span id="line-1174"> // initialize each store in parallel</span> |
| <span class="source-line-no">1175</span><span id="line-1175"> for (final ColumnFamilyDescriptor family : htableDescriptor.getColumnFamilies()) {</span> |
| <span class="source-line-no">1176</span><span id="line-1176"> status.setStatus("Instantiating store for column family " + family);</span> |
| <span class="source-line-no">1177</span><span id="line-1177"> completionService.submit(new Callable<HStore>() {</span> |
| <span class="source-line-no">1178</span><span id="line-1178"> @Override</span> |
| <span class="source-line-no">1179</span><span id="line-1179"> public HStore call() throws IOException {</span> |
| <span class="source-line-no">1180</span><span id="line-1180"> return instantiateHStore(family, warmup);</span> |
| <span class="source-line-no">1181</span><span id="line-1181"> }</span> |
| <span class="source-line-no">1182</span><span id="line-1182"> });</span> |
| <span class="source-line-no">1183</span><span id="line-1183"> }</span> |
| <span class="source-line-no">1184</span><span id="line-1184"> boolean allStoresOpened = false;</span> |
| <span class="source-line-no">1185</span><span id="line-1185"> boolean hasSloppyStores = false;</span> |
| <span class="source-line-no">1186</span><span id="line-1186"> try {</span> |
| <span class="source-line-no">1187</span><span id="line-1187"> for (int i = 0; i < htableDescriptor.getColumnFamilyCount(); i++) {</span> |
| <span class="source-line-no">1188</span><span id="line-1188"> Future<HStore> future = completionService.take();</span> |
| <span class="source-line-no">1189</span><span id="line-1189"> HStore store = future.get();</span> |
| <span class="source-line-no">1190</span><span id="line-1190"> this.stores.put(store.getColumnFamilyDescriptor().getName(), store);</span> |
| <span class="source-line-no">1191</span><span id="line-1191"> if (store.isSloppyMemStore()) {</span> |
| <span class="source-line-no">1192</span><span id="line-1192"> hasSloppyStores = true;</span> |
| <span class="source-line-no">1193</span><span id="line-1193"> }</span> |
| <span class="source-line-no">1194</span><span id="line-1194"></span> |
| <span class="source-line-no">1195</span><span id="line-1195"> long storeMaxSequenceId = store.getMaxSequenceId().orElse(0L);</span> |
| <span class="source-line-no">1196</span><span id="line-1196"> maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), storeMaxSequenceId);</span> |
| <span class="source-line-no">1197</span><span id="line-1197"> if (maxSeqId == -1 || storeMaxSequenceId > maxSeqId) {</span> |
| <span class="source-line-no">1198</span><span id="line-1198"> maxSeqId = storeMaxSequenceId;</span> |
| <span class="source-line-no">1199</span><span id="line-1199"> }</span> |
| <span class="source-line-no">1200</span><span id="line-1200"> long maxStoreMemstoreTS = store.getMaxMemStoreTS().orElse(0L);</span> |
| <span class="source-line-no">1201</span><span id="line-1201"> if (maxStoreMemstoreTS > maxMemstoreTS) {</span> |
| <span class="source-line-no">1202</span><span id="line-1202"> maxMemstoreTS = maxStoreMemstoreTS;</span> |
| <span class="source-line-no">1203</span><span id="line-1203"> }</span> |
| <span class="source-line-no">1204</span><span id="line-1204"> }</span> |
| <span class="source-line-no">1205</span><span id="line-1205"> allStoresOpened = true;</span> |
| <span class="source-line-no">1206</span><span id="line-1206"> if (hasSloppyStores) {</span> |
| <span class="source-line-no">1207</span><span id="line-1207"> htableDescriptor = TableDescriptorBuilder.newBuilder(htableDescriptor)</span> |
| <span class="source-line-no">1208</span><span id="line-1208"> .setFlushPolicyClassName(FlushNonSloppyStoresFirstPolicy.class.getName()).build();</span> |
| <span class="source-line-no">1209</span><span id="line-1209"> LOG.info("Setting FlushNonSloppyStoresFirstPolicy for the region=" + this);</span> |
| <span class="source-line-no">1210</span><span id="line-1210"> }</span> |
| <span class="source-line-no">1211</span><span id="line-1211"> } catch (InterruptedException e) {</span> |
| <span class="source-line-no">1212</span><span id="line-1212"> throw throwOnInterrupt(e);</span> |
| <span class="source-line-no">1213</span><span id="line-1213"> } catch (ExecutionException e) {</span> |
| <span class="source-line-no">1214</span><span id="line-1214"> throw new IOException(e.getCause());</span> |
| <span class="source-line-no">1215</span><span id="line-1215"> } finally {</span> |
| <span class="source-line-no">1216</span><span id="line-1216"> storeOpenerThreadPool.shutdownNow();</span> |
| <span class="source-line-no">1217</span><span id="line-1217"> if (!allStoresOpened) {</span> |
| <span class="source-line-no">1218</span><span id="line-1218"> // something went wrong, close all opened stores</span> |
| <span class="source-line-no">1219</span><span id="line-1219"> LOG.error("Could not initialize all stores for the region=" + this);</span> |
| <span class="source-line-no">1220</span><span id="line-1220"> for (HStore store : this.stores.values()) {</span> |
| <span class="source-line-no">1221</span><span id="line-1221"> try {</span> |
| <span class="source-line-no">1222</span><span id="line-1222"> store.close();</span> |
| <span class="source-line-no">1223</span><span id="line-1223"> } catch (IOException e) {</span> |
| <span class="source-line-no">1224</span><span id="line-1224"> LOG.warn("close store {} failed in region {}", store.toString(), this, e);</span> |
| <span class="source-line-no">1225</span><span id="line-1225"> }</span> |
| <span class="source-line-no">1226</span><span id="line-1226"> }</span> |
| <span class="source-line-no">1227</span><span id="line-1227"> }</span> |
| <span class="source-line-no">1228</span><span id="line-1228"> }</span> |
| <span class="source-line-no">1229</span><span id="line-1229"> }</span> |
| <span class="source-line-no">1230</span><span id="line-1230"> return Math.max(maxSeqId, maxMemstoreTS + 1);</span> |
| <span class="source-line-no">1231</span><span id="line-1231"> }</span> |
| <span class="source-line-no">1232</span><span id="line-1232"></span> |
| <span class="source-line-no">1233</span><span id="line-1233"> private void initializeWarmup(final CancelableProgressable reporter) throws IOException {</span> |
| <span class="source-line-no">1234</span><span id="line-1234"> MonitoredTask status = TaskMonitor.get().createStatus("Initializing region " + this);</span> |
| <span class="source-line-no">1235</span><span id="line-1235"> // Initialize all the HStores</span> |
| <span class="source-line-no">1236</span><span id="line-1236"> status.setStatus("Warmup all stores of " + this.getRegionInfo().getRegionNameAsString());</span> |
| <span class="source-line-no">1237</span><span id="line-1237"> try {</span> |
| <span class="source-line-no">1238</span><span id="line-1238"> initializeStores(reporter, status, true);</span> |
| <span class="source-line-no">1239</span><span id="line-1239"> } finally {</span> |
| <span class="source-line-no">1240</span><span id="line-1240"> status.markComplete("Warmed up " + this.getRegionInfo().getRegionNameAsString());</span> |
| <span class="source-line-no">1241</span><span id="line-1241"> }</span> |
| <span class="source-line-no">1242</span><span id="line-1242"> }</span> |
| <span class="source-line-no">1243</span><span id="line-1243"></span> |
| <span class="source-line-no">1244</span><span id="line-1244"> /** Returns Map of StoreFiles by column family */</span> |
| <span class="source-line-no">1245</span><span id="line-1245"> private NavigableMap<byte[], List<Path>> getStoreFiles() {</span> |
| <span class="source-line-no">1246</span><span id="line-1246"> NavigableMap<byte[], List<Path>> allStoreFiles = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">1247</span><span id="line-1247"> for (HStore store : stores.values()) {</span> |
| <span class="source-line-no">1248</span><span id="line-1248"> Collection<HStoreFile> storeFiles = store.getStorefiles();</span> |
| <span class="source-line-no">1249</span><span id="line-1249"> if (storeFiles == null) {</span> |
| <span class="source-line-no">1250</span><span id="line-1250"> continue;</span> |
| <span class="source-line-no">1251</span><span id="line-1251"> }</span> |
| <span class="source-line-no">1252</span><span id="line-1252"> List<Path> storeFileNames = new ArrayList<>();</span> |
| <span class="source-line-no">1253</span><span id="line-1253"> for (HStoreFile storeFile : storeFiles) {</span> |
| <span class="source-line-no">1254</span><span id="line-1254"> storeFileNames.add(storeFile.getPath());</span> |
| <span class="source-line-no">1255</span><span id="line-1255"> }</span> |
| <span class="source-line-no">1256</span><span id="line-1256"> allStoreFiles.put(store.getColumnFamilyDescriptor().getName(), storeFileNames);</span> |
| <span class="source-line-no">1257</span><span id="line-1257"> }</span> |
| <span class="source-line-no">1258</span><span id="line-1258"> return allStoreFiles;</span> |
| <span class="source-line-no">1259</span><span id="line-1259"> }</span> |
| <span class="source-line-no">1260</span><span id="line-1260"></span> |
| <span class="source-line-no">1261</span><span id="line-1261"> protected void writeRegionOpenMarker(WAL wal, long openSeqId) throws IOException {</span> |
| <span class="source-line-no">1262</span><span id="line-1262"> Map<byte[], List<Path>> storeFiles = getStoreFiles();</span> |
| <span class="source-line-no">1263</span><span id="line-1263"> RegionEventDescriptor regionOpenDesc =</span> |
| <span class="source-line-no">1264</span><span id="line-1264"> ProtobufUtil.toRegionEventDescriptor(RegionEventDescriptor.EventType.REGION_OPEN,</span> |
| <span class="source-line-no">1265</span><span id="line-1265"> getRegionInfo(), openSeqId, getRegionServerServices().getServerName(), storeFiles);</span> |
| <span class="source-line-no">1266</span><span id="line-1266"> WALUtil.writeRegionEventMarker(wal, getReplicationScope(), getRegionInfo(), regionOpenDesc,</span> |
| <span class="source-line-no">1267</span><span id="line-1267"> mvcc, regionReplicationSink.orElse(null));</span> |
| <span class="source-line-no">1268</span><span id="line-1268"> }</span> |
| <span class="source-line-no">1269</span><span id="line-1269"></span> |
| <span class="source-line-no">1270</span><span id="line-1270"> private void writeRegionCloseMarker(WAL wal) throws IOException {</span> |
| <span class="source-line-no">1271</span><span id="line-1271"> Map<byte[], List<Path>> storeFiles = getStoreFiles();</span> |
| <span class="source-line-no">1272</span><span id="line-1272"> RegionEventDescriptor regionEventDesc = ProtobufUtil.toRegionEventDescriptor(</span> |
| <span class="source-line-no">1273</span><span id="line-1273"> RegionEventDescriptor.EventType.REGION_CLOSE, getRegionInfo(), mvcc.getReadPoint(),</span> |
| <span class="source-line-no">1274</span><span id="line-1274"> getRegionServerServices().getServerName(), storeFiles);</span> |
| <span class="source-line-no">1275</span><span id="line-1275"> // we do not care region close event at secondary replica side so just pass a null</span> |
| <span class="source-line-no">1276</span><span id="line-1276"> // RegionReplicationSink</span> |
| <span class="source-line-no">1277</span><span id="line-1277"> WALUtil.writeRegionEventMarker(wal, getReplicationScope(), getRegionInfo(), regionEventDesc,</span> |
| <span class="source-line-no">1278</span><span id="line-1278"> mvcc, null);</span> |
| <span class="source-line-no">1279</span><span id="line-1279"></span> |
| <span class="source-line-no">1280</span><span id="line-1280"> // Store SeqId in WAL FileSystem when a region closes</span> |
| <span class="source-line-no">1281</span><span id="line-1281"> // checking region folder exists is due to many tests which delete the table folder while a</span> |
| <span class="source-line-no">1282</span><span id="line-1282"> // table is still online</span> |
| <span class="source-line-no">1283</span><span id="line-1283"> if (getWalFileSystem().exists(getWALRegionDir())) {</span> |
| <span class="source-line-no">1284</span><span id="line-1284"> WALSplitUtil.writeRegionSequenceIdFile(getWalFileSystem(), getWALRegionDir(),</span> |
| <span class="source-line-no">1285</span><span id="line-1285"> mvcc.getReadPoint());</span> |
| <span class="source-line-no">1286</span><span id="line-1286"> }</span> |
| <span class="source-line-no">1287</span><span id="line-1287"> }</span> |
| <span class="source-line-no">1288</span><span id="line-1288"></span> |
| <span class="source-line-no">1289</span><span id="line-1289"> /** Returns True if this region has references. */</span> |
| <span class="source-line-no">1290</span><span id="line-1290"> public boolean hasReferences() {</span> |
| <span class="source-line-no">1291</span><span id="line-1291"> return stores.values().stream().anyMatch(HStore::hasReferences);</span> |
| <span class="source-line-no">1292</span><span id="line-1292"> }</span> |
| <span class="source-line-no">1293</span><span id="line-1293"></span> |
| <span class="source-line-no">1294</span><span id="line-1294"> public void blockUpdates() {</span> |
| <span class="source-line-no">1295</span><span id="line-1295"> this.updatesLock.writeLock().lock();</span> |
| <span class="source-line-no">1296</span><span id="line-1296"> }</span> |
| <span class="source-line-no">1297</span><span id="line-1297"></span> |
| <span class="source-line-no">1298</span><span id="line-1298"> public void unblockUpdates() {</span> |
| <span class="source-line-no">1299</span><span id="line-1299"> this.updatesLock.writeLock().unlock();</span> |
| <span class="source-line-no">1300</span><span id="line-1300"> }</span> |
| <span class="source-line-no">1301</span><span id="line-1301"></span> |
| <span class="source-line-no">1302</span><span id="line-1302"> public HDFSBlocksDistribution getHDFSBlocksDistribution() {</span> |
| <span class="source-line-no">1303</span><span id="line-1303"> HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();</span> |
| <span class="source-line-no">1304</span><span id="line-1304"> stores.values().stream().filter(s -> s.getStorefiles() != null)</span> |
| <span class="source-line-no">1305</span><span id="line-1305"> .flatMap(s -> s.getStorefiles().stream()).map(HStoreFile::getHDFSBlockDistribution)</span> |
| <span class="source-line-no">1306</span><span id="line-1306"> .forEachOrdered(hdfsBlocksDistribution::add);</span> |
| <span class="source-line-no">1307</span><span id="line-1307"> return hdfsBlocksDistribution;</span> |
| <span class="source-line-no">1308</span><span id="line-1308"> }</span> |
| <span class="source-line-no">1309</span><span id="line-1309"></span> |
| <span class="source-line-no">1310</span><span id="line-1310"> /**</span> |
| <span class="source-line-no">1311</span><span id="line-1311"> * This is a helper function to compute HDFS block distribution on demand</span> |
| <span class="source-line-no">1312</span><span id="line-1312"> * @param conf configuration</span> |
| <span class="source-line-no">1313</span><span id="line-1313"> * @param tableDescriptor TableDescriptor of the table</span> |
| <span class="source-line-no">1314</span><span id="line-1314"> * @param regionInfo encoded name of the region</span> |
| <span class="source-line-no">1315</span><span id="line-1315"> * @return The HDFS blocks distribution for the given region.</span> |
| <span class="source-line-no">1316</span><span id="line-1316"> */</span> |
| <span class="source-line-no">1317</span><span id="line-1317"> public static HDFSBlocksDistribution computeHDFSBlocksDistribution(Configuration conf,</span> |
| <span class="source-line-no">1318</span><span id="line-1318"> TableDescriptor tableDescriptor, RegionInfo regionInfo) throws IOException {</span> |
| <span class="source-line-no">1319</span><span id="line-1319"> Path tablePath =</span> |
| <span class="source-line-no">1320</span><span id="line-1320"> CommonFSUtils.getTableDir(CommonFSUtils.getRootDir(conf), tableDescriptor.getTableName());</span> |
| <span class="source-line-no">1321</span><span id="line-1321"> return computeHDFSBlocksDistribution(conf, tableDescriptor, regionInfo, tablePath);</span> |
| <span class="source-line-no">1322</span><span id="line-1322"> }</span> |
| <span class="source-line-no">1323</span><span id="line-1323"></span> |
| <span class="source-line-no">1324</span><span id="line-1324"> /**</span> |
| <span class="source-line-no">1325</span><span id="line-1325"> * This is a helper function to compute HDFS block distribution on demand</span> |
| <span class="source-line-no">1326</span><span id="line-1326"> * @param conf configuration</span> |
| <span class="source-line-no">1327</span><span id="line-1327"> * @param tableDescriptor TableDescriptor of the table</span> |
| <span class="source-line-no">1328</span><span id="line-1328"> * @param regionInfo encoded name of the region</span> |
| <span class="source-line-no">1329</span><span id="line-1329"> * @param tablePath the table directory</span> |
| <span class="source-line-no">1330</span><span id="line-1330"> * @return The HDFS blocks distribution for the given region.</span> |
| <span class="source-line-no">1331</span><span id="line-1331"> */</span> |
| <span class="source-line-no">1332</span><span id="line-1332"> public static HDFSBlocksDistribution computeHDFSBlocksDistribution(Configuration conf,</span> |
| <span class="source-line-no">1333</span><span id="line-1333"> TableDescriptor tableDescriptor, RegionInfo regionInfo, Path tablePath) throws IOException {</span> |
| <span class="source-line-no">1334</span><span id="line-1334"> HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();</span> |
| <span class="source-line-no">1335</span><span id="line-1335"> FileSystem fs = tablePath.getFileSystem(conf);</span> |
| <span class="source-line-no">1336</span><span id="line-1336"></span> |
| <span class="source-line-no">1337</span><span id="line-1337"> HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tablePath, regionInfo);</span> |
| <span class="source-line-no">1338</span><span id="line-1338"> for (ColumnFamilyDescriptor family : tableDescriptor.getColumnFamilies()) {</span> |
| <span class="source-line-no">1339</span><span id="line-1339"> List<LocatedFileStatus> locatedFileStatusList =</span> |
| <span class="source-line-no">1340</span><span id="line-1340"> HRegionFileSystem.getStoreFilesLocatedStatus(regionFs, family.getNameAsString(), true);</span> |
| <span class="source-line-no">1341</span><span id="line-1341"> if (locatedFileStatusList == null) {</span> |
| <span class="source-line-no">1342</span><span id="line-1342"> continue;</span> |
| <span class="source-line-no">1343</span><span id="line-1343"> }</span> |
| <span class="source-line-no">1344</span><span id="line-1344"></span> |
| <span class="source-line-no">1345</span><span id="line-1345"> for (LocatedFileStatus status : locatedFileStatusList) {</span> |
| <span class="source-line-no">1346</span><span id="line-1346"> Path p = status.getPath();</span> |
| <span class="source-line-no">1347</span><span id="line-1347"> if (StoreFileInfo.isReference(p) || HFileLink.isHFileLink(p)) {</span> |
| <span class="source-line-no">1348</span><span id="line-1348"> // Only construct StoreFileInfo object if its not a hfile, save obj</span> |
| <span class="source-line-no">1349</span><span id="line-1349"> // creation</span> |
| <span class="source-line-no">1350</span><span id="line-1350"> StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, status);</span> |
| <span class="source-line-no">1351</span><span id="line-1351"> hdfsBlocksDistribution.add(storeFileInfo.computeHDFSBlocksDistribution(fs));</span> |
| <span class="source-line-no">1352</span><span id="line-1352"> } else if (StoreFileInfo.isHFile(p)) {</span> |
| <span class="source-line-no">1353</span><span id="line-1353"> // If its a HFile, then lets just add to the block distribution</span> |
| <span class="source-line-no">1354</span><span id="line-1354"> // lets not create more objects here, not even another HDFSBlocksDistribution</span> |
| <span class="source-line-no">1355</span><span id="line-1355"> FSUtils.addToHDFSBlocksDistribution(hdfsBlocksDistribution, status.getBlockLocations());</span> |
| <span class="source-line-no">1356</span><span id="line-1356"> } else {</span> |
| <span class="source-line-no">1357</span><span id="line-1357"> throw new IOException("path=" + p + " doesn't look like a valid StoreFile");</span> |
| <span class="source-line-no">1358</span><span id="line-1358"> }</span> |
| <span class="source-line-no">1359</span><span id="line-1359"> }</span> |
| <span class="source-line-no">1360</span><span id="line-1360"> }</span> |
| <span class="source-line-no">1361</span><span id="line-1361"> return hdfsBlocksDistribution;</span> |
| <span class="source-line-no">1362</span><span id="line-1362"> }</span> |
| <span class="source-line-no">1363</span><span id="line-1363"></span> |
| <span class="source-line-no">1364</span><span id="line-1364"> /**</span> |
| <span class="source-line-no">1365</span><span id="line-1365"> * Increase the size of mem store in this region and the size of global mem store</span> |
| <span class="source-line-no">1366</span><span id="line-1366"> */</span> |
| <span class="source-line-no">1367</span><span id="line-1367"> private void incMemStoreSize(MemStoreSize mss) {</span> |
| <span class="source-line-no">1368</span><span id="line-1368"> incMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(),</span> |
| <span class="source-line-no">1369</span><span id="line-1369"> mss.getCellsCount());</span> |
| <span class="source-line-no">1370</span><span id="line-1370"> }</span> |
| <span class="source-line-no">1371</span><span id="line-1371"></span> |
| <span class="source-line-no">1372</span><span id="line-1372"> void incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta,</span> |
| <span class="source-line-no">1373</span><span id="line-1373"> int cellsCountDelta) {</span> |
| <span class="source-line-no">1374</span><span id="line-1374"> if (this.rsAccounting != null) {</span> |
| <span class="source-line-no">1375</span><span id="line-1375"> rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta);</span> |
| <span class="source-line-no">1376</span><span id="line-1376"> }</span> |
| <span class="source-line-no">1377</span><span id="line-1377"> long dataSize = this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta,</span> |
| <span class="source-line-no">1378</span><span id="line-1378"> offHeapSizeDelta, cellsCountDelta);</span> |
| <span class="source-line-no">1379</span><span id="line-1379"> checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);</span> |
| <span class="source-line-no">1380</span><span id="line-1380"> }</span> |
| <span class="source-line-no">1381</span><span id="line-1381"></span> |
| <span class="source-line-no">1382</span><span id="line-1382"> void decrMemStoreSize(MemStoreSize mss) {</span> |
| <span class="source-line-no">1383</span><span id="line-1383"> decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(),</span> |
| <span class="source-line-no">1384</span><span id="line-1384"> mss.getCellsCount());</span> |
| <span class="source-line-no">1385</span><span id="line-1385"> }</span> |
| <span class="source-line-no">1386</span><span id="line-1386"></span> |
| <span class="source-line-no">1387</span><span id="line-1387"> private void decrMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta,</span> |
| <span class="source-line-no">1388</span><span id="line-1388"> int cellsCountDelta) {</span> |
| <span class="source-line-no">1389</span><span id="line-1389"> if (this.rsAccounting != null) {</span> |
| <span class="source-line-no">1390</span><span id="line-1390"> rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta);</span> |
| <span class="source-line-no">1391</span><span id="line-1391"> }</span> |
| <span class="source-line-no">1392</span><span id="line-1392"> long dataSize = this.memStoreSizing.decMemStoreSize(dataSizeDelta, heapSizeDelta,</span> |
| <span class="source-line-no">1393</span><span id="line-1393"> offHeapSizeDelta, cellsCountDelta);</span> |
| <span class="source-line-no">1394</span><span id="line-1394"> checkNegativeMemStoreDataSize(dataSize, -dataSizeDelta);</span> |
| <span class="source-line-no">1395</span><span id="line-1395"> }</span> |
| <span class="source-line-no">1396</span><span id="line-1396"></span> |
| <span class="source-line-no">1397</span><span id="line-1397"> private void checkNegativeMemStoreDataSize(long memStoreDataSize, long delta) {</span> |
| <span class="source-line-no">1398</span><span id="line-1398"> // This is extremely bad if we make memStoreSizing negative. Log as much info on the offending</span> |
| <span class="source-line-no">1399</span><span id="line-1399"> // caller as possible. (memStoreSizing might be a negative value already -- freeing memory)</span> |
| <span class="source-line-no">1400</span><span id="line-1400"> if (memStoreDataSize < 0) {</span> |
| <span class="source-line-no">1401</span><span id="line-1401"> LOG.error("Asked to modify this region's (" + this.toString()</span> |
| <span class="source-line-no">1402</span><span id="line-1402"> + ") memStoreSizing to a negative value which is incorrect. Current memStoreSizing="</span> |
| <span class="source-line-no">1403</span><span id="line-1403"> + (memStoreDataSize - delta) + ", delta=" + delta, new Exception());</span> |
| <span class="source-line-no">1404</span><span id="line-1404"> }</span> |
| <span class="source-line-no">1405</span><span id="line-1405"> }</span> |
| <span class="source-line-no">1406</span><span id="line-1406"></span> |
| <span class="source-line-no">1407</span><span id="line-1407"> @Override</span> |
| <span class="source-line-no">1408</span><span id="line-1408"> public RegionInfo getRegionInfo() {</span> |
| <span class="source-line-no">1409</span><span id="line-1409"> return this.fs.getRegionInfo();</span> |
| <span class="source-line-no">1410</span><span id="line-1410"> }</span> |
| <span class="source-line-no">1411</span><span id="line-1411"></span> |
| <span class="source-line-no">1412</span><span id="line-1412"> /**</span> |
| <span class="source-line-no">1413</span><span id="line-1413"> * Returns Instance of {@link RegionServerServices} used by this HRegion. Can be null.</span> |
| <span class="source-line-no">1414</span><span id="line-1414"> */</span> |
| <span class="source-line-no">1415</span><span id="line-1415"> RegionServerServices getRegionServerServices() {</span> |
| <span class="source-line-no">1416</span><span id="line-1416"> return this.rsServices;</span> |
| <span class="source-line-no">1417</span><span id="line-1417"> }</span> |
| <span class="source-line-no">1418</span><span id="line-1418"></span> |
| <span class="source-line-no">1419</span><span id="line-1419"> @Override</span> |
| <span class="source-line-no">1420</span><span id="line-1420"> public long getReadRequestsCount() {</span> |
| <span class="source-line-no">1421</span><span id="line-1421"> return readRequestsCount.sum();</span> |
| <span class="source-line-no">1422</span><span id="line-1422"> }</span> |
| <span class="source-line-no">1423</span><span id="line-1423"></span> |
| <span class="source-line-no">1424</span><span id="line-1424"> @Override</span> |
| <span class="source-line-no">1425</span><span id="line-1425"> public long getCpRequestsCount() {</span> |
| <span class="source-line-no">1426</span><span id="line-1426"> return cpRequestsCount.sum();</span> |
| <span class="source-line-no">1427</span><span id="line-1427"> }</span> |
| <span class="source-line-no">1428</span><span id="line-1428"></span> |
| <span class="source-line-no">1429</span><span id="line-1429"> @Override</span> |
| <span class="source-line-no">1430</span><span id="line-1430"> public long getFilteredReadRequestsCount() {</span> |
| <span class="source-line-no">1431</span><span id="line-1431"> return filteredReadRequestsCount.sum();</span> |
| <span class="source-line-no">1432</span><span id="line-1432"> }</span> |
| <span class="source-line-no">1433</span><span id="line-1433"></span> |
| <span class="source-line-no">1434</span><span id="line-1434"> @Override</span> |
| <span class="source-line-no">1435</span><span id="line-1435"> public long getWriteRequestsCount() {</span> |
| <span class="source-line-no">1436</span><span id="line-1436"> return writeRequestsCount.sum();</span> |
| <span class="source-line-no">1437</span><span id="line-1437"> }</span> |
| <span class="source-line-no">1438</span><span id="line-1438"></span> |
| <span class="source-line-no">1439</span><span id="line-1439"> @Override</span> |
| <span class="source-line-no">1440</span><span id="line-1440"> public long getMemStoreDataSize() {</span> |
| <span class="source-line-no">1441</span><span id="line-1441"> return memStoreSizing.getDataSize();</span> |
| <span class="source-line-no">1442</span><span id="line-1442"> }</span> |
| <span class="source-line-no">1443</span><span id="line-1443"></span> |
| <span class="source-line-no">1444</span><span id="line-1444"> @Override</span> |
| <span class="source-line-no">1445</span><span id="line-1445"> public long getMemStoreHeapSize() {</span> |
| <span class="source-line-no">1446</span><span id="line-1446"> return memStoreSizing.getHeapSize();</span> |
| <span class="source-line-no">1447</span><span id="line-1447"> }</span> |
| <span class="source-line-no">1448</span><span id="line-1448"></span> |
| <span class="source-line-no">1449</span><span id="line-1449"> @Override</span> |
| <span class="source-line-no">1450</span><span id="line-1450"> public long getMemStoreOffHeapSize() {</span> |
| <span class="source-line-no">1451</span><span id="line-1451"> return memStoreSizing.getOffHeapSize();</span> |
| <span class="source-line-no">1452</span><span id="line-1452"> }</span> |
| <span class="source-line-no">1453</span><span id="line-1453"></span> |
| <span class="source-line-no">1454</span><span id="line-1454"> /** Returns store services for this region, to access services required by store level needs */</span> |
| <span class="source-line-no">1455</span><span id="line-1455"> public RegionServicesForStores getRegionServicesForStores() {</span> |
| <span class="source-line-no">1456</span><span id="line-1456"> return regionServicesForStores;</span> |
| <span class="source-line-no">1457</span><span id="line-1457"> }</span> |
| <span class="source-line-no">1458</span><span id="line-1458"></span> |
| <span class="source-line-no">1459</span><span id="line-1459"> @Override</span> |
| <span class="source-line-no">1460</span><span id="line-1460"> public long getNumMutationsWithoutWAL() {</span> |
| <span class="source-line-no">1461</span><span id="line-1461"> return numMutationsWithoutWAL.sum();</span> |
| <span class="source-line-no">1462</span><span id="line-1462"> }</span> |
| <span class="source-line-no">1463</span><span id="line-1463"></span> |
| <span class="source-line-no">1464</span><span id="line-1464"> @Override</span> |
| <span class="source-line-no">1465</span><span id="line-1465"> public long getDataInMemoryWithoutWAL() {</span> |
| <span class="source-line-no">1466</span><span id="line-1466"> return dataInMemoryWithoutWAL.sum();</span> |
| <span class="source-line-no">1467</span><span id="line-1467"> }</span> |
| <span class="source-line-no">1468</span><span id="line-1468"></span> |
| <span class="source-line-no">1469</span><span id="line-1469"> @Override</span> |
| <span class="source-line-no">1470</span><span id="line-1470"> public long getBlockedRequestsCount() {</span> |
| <span class="source-line-no">1471</span><span id="line-1471"> return blockedRequestsCount.sum();</span> |
| <span class="source-line-no">1472</span><span id="line-1472"> }</span> |
| <span class="source-line-no">1473</span><span id="line-1473"></span> |
| <span class="source-line-no">1474</span><span id="line-1474"> @Override</span> |
| <span class="source-line-no">1475</span><span id="line-1475"> public long getCheckAndMutateChecksPassed() {</span> |
| <span class="source-line-no">1476</span><span id="line-1476"> return checkAndMutateChecksPassed.sum();</span> |
| <span class="source-line-no">1477</span><span id="line-1477"> }</span> |
| <span class="source-line-no">1478</span><span id="line-1478"></span> |
| <span class="source-line-no">1479</span><span id="line-1479"> @Override</span> |
| <span class="source-line-no">1480</span><span id="line-1480"> public long getCheckAndMutateChecksFailed() {</span> |
| <span class="source-line-no">1481</span><span id="line-1481"> return checkAndMutateChecksFailed.sum();</span> |
| <span class="source-line-no">1482</span><span id="line-1482"> }</span> |
| <span class="source-line-no">1483</span><span id="line-1483"></span> |
| <span class="source-line-no">1484</span><span id="line-1484"> // TODO Needs to check whether we should expose our metrics system to CPs. If CPs themselves doing</span> |
| <span class="source-line-no">1485</span><span id="line-1485"> // the op and bypassing the core, this might be needed? Should be stop supporting the bypass</span> |
| <span class="source-line-no">1486</span><span id="line-1486"> // feature?</span> |
| <span class="source-line-no">1487</span><span id="line-1487"> public MetricsRegion getMetrics() {</span> |
| <span class="source-line-no">1488</span><span id="line-1488"> return metricsRegion;</span> |
| <span class="source-line-no">1489</span><span id="line-1489"> }</span> |
| <span class="source-line-no">1490</span><span id="line-1490"></span> |
| <span class="source-line-no">1491</span><span id="line-1491"> @Override</span> |
| <span class="source-line-no">1492</span><span id="line-1492"> public boolean isClosed() {</span> |
| <span class="source-line-no">1493</span><span id="line-1493"> return this.closed.get();</span> |
| <span class="source-line-no">1494</span><span id="line-1494"> }</span> |
| <span class="source-line-no">1495</span><span id="line-1495"></span> |
| <span class="source-line-no">1496</span><span id="line-1496"> @Override</span> |
| <span class="source-line-no">1497</span><span id="line-1497"> public boolean isClosing() {</span> |
| <span class="source-line-no">1498</span><span id="line-1498"> return this.closing.get();</span> |
| <span class="source-line-no">1499</span><span id="line-1499"> }</span> |
| <span class="source-line-no">1500</span><span id="line-1500"></span> |
| <span class="source-line-no">1501</span><span id="line-1501"> @Override</span> |
| <span class="source-line-no">1502</span><span id="line-1502"> public boolean isReadOnly() {</span> |
| <span class="source-line-no">1503</span><span id="line-1503"> return this.writestate.isReadOnly();</span> |
| <span class="source-line-no">1504</span><span id="line-1504"> }</span> |
| <span class="source-line-no">1505</span><span id="line-1505"></span> |
| <span class="source-line-no">1506</span><span id="line-1506"> @Override</span> |
| <span class="source-line-no">1507</span><span id="line-1507"> public boolean isAvailable() {</span> |
| <span class="source-line-no">1508</span><span id="line-1508"> return !isClosed() && !isClosing();</span> |
| <span class="source-line-no">1509</span><span id="line-1509"> }</span> |
| <span class="source-line-no">1510</span><span id="line-1510"></span> |
| <span class="source-line-no">1511</span><span id="line-1511"> @Override</span> |
| <span class="source-line-no">1512</span><span id="line-1512"> public boolean isSplittable() {</span> |
| <span class="source-line-no">1513</span><span id="line-1513"> return splitPolicy.canSplit();</span> |
| <span class="source-line-no">1514</span><span id="line-1514"> }</span> |
| <span class="source-line-no">1515</span><span id="line-1515"></span> |
| <span class="source-line-no">1516</span><span id="line-1516"> @Override</span> |
| <span class="source-line-no">1517</span><span id="line-1517"> public boolean isMergeable() {</span> |
| <span class="source-line-no">1518</span><span id="line-1518"> if (!isAvailable()) {</span> |
| <span class="source-line-no">1519</span><span id="line-1519"> LOG.debug("Region " + this + " is not mergeable because it is closing or closed");</span> |
| <span class="source-line-no">1520</span><span id="line-1520"> return false;</span> |
| <span class="source-line-no">1521</span><span id="line-1521"> }</span> |
| <span class="source-line-no">1522</span><span id="line-1522"> if (hasReferences()) {</span> |
| <span class="source-line-no">1523</span><span id="line-1523"> LOG.debug("Region " + this + " is not mergeable because it has references");</span> |
| <span class="source-line-no">1524</span><span id="line-1524"> return false;</span> |
| <span class="source-line-no">1525</span><span id="line-1525"> }</span> |
| <span class="source-line-no">1526</span><span id="line-1526"></span> |
| <span class="source-line-no">1527</span><span id="line-1527"> return true;</span> |
| <span class="source-line-no">1528</span><span id="line-1528"> }</span> |
| <span class="source-line-no">1529</span><span id="line-1529"></span> |
| <span class="source-line-no">1530</span><span id="line-1530"> public boolean areWritesEnabled() {</span> |
| <span class="source-line-no">1531</span><span id="line-1531"> synchronized (this.writestate) {</span> |
| <span class="source-line-no">1532</span><span id="line-1532"> return this.writestate.writesEnabled;</span> |
| <span class="source-line-no">1533</span><span id="line-1533"> }</span> |
| <span class="source-line-no">1534</span><span id="line-1534"> }</span> |
| <span class="source-line-no">1535</span><span id="line-1535"></span> |
| <span class="source-line-no">1536</span><span id="line-1536"> public MultiVersionConcurrencyControl getMVCC() {</span> |
| <span class="source-line-no">1537</span><span id="line-1537"> return mvcc;</span> |
| <span class="source-line-no">1538</span><span id="line-1538"> }</span> |
| <span class="source-line-no">1539</span><span id="line-1539"></span> |
| <span class="source-line-no">1540</span><span id="line-1540"> @Override</span> |
| <span class="source-line-no">1541</span><span id="line-1541"> public long getMaxFlushedSeqId() {</span> |
| <span class="source-line-no">1542</span><span id="line-1542"> return maxFlushedSeqId;</span> |
| <span class="source-line-no">1543</span><span id="line-1543"> }</span> |
| <span class="source-line-no">1544</span><span id="line-1544"></span> |
| <span class="source-line-no">1545</span><span id="line-1545"> /** Returns readpoint considering given IsolationLevel. Pass {@code null} for default */</span> |
| <span class="source-line-no">1546</span><span id="line-1546"> public long getReadPoint(IsolationLevel isolationLevel) {</span> |
| <span class="source-line-no">1547</span><span id="line-1547"> if (isolationLevel != null && isolationLevel == IsolationLevel.READ_UNCOMMITTED) {</span> |
| <span class="source-line-no">1548</span><span id="line-1548"> // This scan can read even uncommitted transactions</span> |
| <span class="source-line-no">1549</span><span id="line-1549"> return Long.MAX_VALUE;</span> |
| <span class="source-line-no">1550</span><span id="line-1550"> }</span> |
| <span class="source-line-no">1551</span><span id="line-1551"> return mvcc.getReadPoint();</span> |
| <span class="source-line-no">1552</span><span id="line-1552"> }</span> |
| <span class="source-line-no">1553</span><span id="line-1553"></span> |
| <span class="source-line-no">1554</span><span id="line-1554"> public boolean isLoadingCfsOnDemandDefault() {</span> |
| <span class="source-line-no">1555</span><span id="line-1555"> return this.isLoadingCfsOnDemandDefault;</span> |
| <span class="source-line-no">1556</span><span id="line-1556"> }</span> |
| <span class="source-line-no">1557</span><span id="line-1557"></span> |
| <span class="source-line-no">1558</span><span id="line-1558"> /**</span> |
| <span class="source-line-no">1559</span><span id="line-1559"> * Close down this HRegion. Flush the cache, shut down each HStore, don't service any more calls.</span> |
| <span class="source-line-no">1560</span><span id="line-1560"> * <p></span> |
| <span class="source-line-no">1561</span><span id="line-1561"> * This method could take some time to execute, so don't call it from a time-sensitive thread.</span> |
| <span class="source-line-no">1562</span><span id="line-1562"> * @return Vector of all the storage files that the HRegion's component HStores make use of. It's</span> |
| <span class="source-line-no">1563</span><span id="line-1563"> * a list of all StoreFile objects. Returns empty vector if already closed and null if</span> |
| <span class="source-line-no">1564</span><span id="line-1564"> * judged that it should not close.</span> |
| <span class="source-line-no">1565</span><span id="line-1565"> * @throws IOException e</span> |
| <span class="source-line-no">1566</span><span id="line-1566"> * @throws DroppedSnapshotException Thrown when replay of wal is required because a Snapshot was</span> |
| <span class="source-line-no">1567</span><span id="line-1567"> * not properly persisted. The region is put in closing mode, and</span> |
| <span class="source-line-no">1568</span><span id="line-1568"> * the caller MUST abort after this.</span> |
| <span class="source-line-no">1569</span><span id="line-1569"> */</span> |
| <span class="source-line-no">1570</span><span id="line-1570"> public Map<byte[], List<HStoreFile>> close() throws IOException {</span> |
| <span class="source-line-no">1571</span><span id="line-1571"> return close(false);</span> |
| <span class="source-line-no">1572</span><span id="line-1572"> }</span> |
| <span class="source-line-no">1573</span><span id="line-1573"></span> |
| <span class="source-line-no">1574</span><span id="line-1574"> private final Object closeLock = new Object();</span> |
| <span class="source-line-no">1575</span><span id="line-1575"></span> |
| <span class="source-line-no">1576</span><span id="line-1576"> /** Conf key for fair locking policy */</span> |
| <span class="source-line-no">1577</span><span id="line-1577"> public static final String FAIR_REENTRANT_CLOSE_LOCK =</span> |
| <span class="source-line-no">1578</span><span id="line-1578"> "hbase.regionserver.fair.region.close.lock";</span> |
| <span class="source-line-no">1579</span><span id="line-1579"> public static final boolean DEFAULT_FAIR_REENTRANT_CLOSE_LOCK = true;</span> |
| <span class="source-line-no">1580</span><span id="line-1580"> /** Conf key for the periodic flush interval */</span> |
| <span class="source-line-no">1581</span><span id="line-1581"> public static final String MEMSTORE_PERIODIC_FLUSH_INTERVAL =</span> |
| <span class="source-line-no">1582</span><span id="line-1582"> "hbase.regionserver.optionalcacheflushinterval";</span> |
| <span class="source-line-no">1583</span><span id="line-1583"> /** Default interval for the memstore flush */</span> |
| <span class="source-line-no">1584</span><span id="line-1584"> public static final int DEFAULT_CACHE_FLUSH_INTERVAL = 3600000;</span> |
| <span class="source-line-no">1585</span><span id="line-1585"> /** Default interval for System tables memstore flush */</span> |
| <span class="source-line-no">1586</span><span id="line-1586"> public static final int SYSTEM_CACHE_FLUSH_INTERVAL = 300000; // 5 minutes</span> |
| <span class="source-line-no">1587</span><span id="line-1587"></span> |
| <span class="source-line-no">1588</span><span id="line-1588"> /** Conf key to force a flush if there are already enough changes for one region in memstore */</span> |
| <span class="source-line-no">1589</span><span id="line-1589"> public static final String MEMSTORE_FLUSH_PER_CHANGES = "hbase.regionserver.flush.per.changes";</span> |
| <span class="source-line-no">1590</span><span id="line-1590"> public static final long DEFAULT_FLUSH_PER_CHANGES = 30000000; // 30 millions</span> |
| <span class="source-line-no">1591</span><span id="line-1591"> /**</span> |
| <span class="source-line-no">1592</span><span id="line-1592"> * The following MAX_FLUSH_PER_CHANGES is large enough because each KeyValue has 20+ bytes</span> |
| <span class="source-line-no">1593</span><span id="line-1593"> * overhead. Therefore, even 1G empty KVs occupy at least 20GB memstore size for a single region</span> |
| <span class="source-line-no">1594</span><span id="line-1594"> */</span> |
| <span class="source-line-no">1595</span><span id="line-1595"> public static final long MAX_FLUSH_PER_CHANGES = 1000000000; // 1G</span> |
| <span class="source-line-no">1596</span><span id="line-1596"></span> |
| <span class="source-line-no">1597</span><span id="line-1597"> public static final String CLOSE_WAIT_ABORT = "hbase.regionserver.close.wait.abort";</span> |
| <span class="source-line-no">1598</span><span id="line-1598"> public static final boolean DEFAULT_CLOSE_WAIT_ABORT = true;</span> |
| <span class="source-line-no">1599</span><span id="line-1599"> public static final String CLOSE_WAIT_TIME = "hbase.regionserver.close.wait.time.ms";</span> |
| <span class="source-line-no">1600</span><span id="line-1600"> public static final long DEFAULT_CLOSE_WAIT_TIME = 60000; // 1 minute</span> |
| <span class="source-line-no">1601</span><span id="line-1601"> public static final String CLOSE_WAIT_INTERVAL = "hbase.regionserver.close.wait.interval.ms";</span> |
| <span class="source-line-no">1602</span><span id="line-1602"> public static final long DEFAULT_CLOSE_WAIT_INTERVAL = 10000; // 10 seconds</span> |
| <span class="source-line-no">1603</span><span id="line-1603"></span> |
| <span class="source-line-no">1604</span><span id="line-1604"> public Map<byte[], List<HStoreFile>> close(boolean abort) throws IOException {</span> |
| <span class="source-line-no">1605</span><span id="line-1605"> return close(abort, false);</span> |
| <span class="source-line-no">1606</span><span id="line-1606"> }</span> |
| <span class="source-line-no">1607</span><span id="line-1607"></span> |
| <span class="source-line-no">1608</span><span id="line-1608"> /**</span> |
| <span class="source-line-no">1609</span><span id="line-1609"> * Close this HRegion.</span> |
| <span class="source-line-no">1610</span><span id="line-1610"> * @param abort true if server is aborting (only during testing)</span> |
| <span class="source-line-no">1611</span><span id="line-1611"> * @param ignoreStatus true if ignore the status (won't be showed on task list)</span> |
| <span class="source-line-no">1612</span><span id="line-1612"> * @return Vector of all the storage files that the HRegion's component HStores make use of. It's</span> |
| <span class="source-line-no">1613</span><span id="line-1613"> * a list of StoreFile objects. Can be null if we are not to close at this time, or we are</span> |
| <span class="source-line-no">1614</span><span id="line-1614"> * already closed.</span> |
| <span class="source-line-no">1615</span><span id="line-1615"> * @throws IOException e</span> |
| <span class="source-line-no">1616</span><span id="line-1616"> * @throws DroppedSnapshotException Thrown when replay of wal is required because a Snapshot was</span> |
| <span class="source-line-no">1617</span><span id="line-1617"> * not properly persisted. The region is put in closing mode, and</span> |
| <span class="source-line-no">1618</span><span id="line-1618"> * the caller MUST abort after this.</span> |
| <span class="source-line-no">1619</span><span id="line-1619"> */</span> |
| <span class="source-line-no">1620</span><span id="line-1620"> public Map<byte[], List<HStoreFile>> close(boolean abort, boolean ignoreStatus)</span> |
| <span class="source-line-no">1621</span><span id="line-1621"> throws IOException {</span> |
| <span class="source-line-no">1622</span><span id="line-1622"> return close(abort, ignoreStatus, false);</span> |
| <span class="source-line-no">1623</span><span id="line-1623"> }</span> |
| <span class="source-line-no">1624</span><span id="line-1624"></span> |
| <span class="source-line-no">1625</span><span id="line-1625"> /**</span> |
| <span class="source-line-no">1626</span><span id="line-1626"> * Close down this HRegion. Flush the cache unless abort parameter is true, Shut down each HStore,</span> |
| <span class="source-line-no">1627</span><span id="line-1627"> * don't service any more calls. This method could take some time to execute, so don't call it</span> |
| <span class="source-line-no">1628</span><span id="line-1628"> * from a time-sensitive thread.</span> |
| <span class="source-line-no">1629</span><span id="line-1629"> * @param abort true if server is aborting (only during testing)</span> |
| <span class="source-line-no">1630</span><span id="line-1630"> * @param ignoreStatus true if ignore the status (wont be showed on task list)</span> |
| <span class="source-line-no">1631</span><span id="line-1631"> * @param isGracefulStop true if region is being closed during graceful stop and the blocks in the</span> |
| <span class="source-line-no">1632</span><span id="line-1632"> * BucketCache should not be evicted.</span> |
| <span class="source-line-no">1633</span><span id="line-1633"> * @return Vector of all the storage files that the HRegion's component HStores make use of. It's</span> |
| <span class="source-line-no">1634</span><span id="line-1634"> * a list of StoreFile objects. Can be null if we are not to close at this time or we are</span> |
| <span class="source-line-no">1635</span><span id="line-1635"> * already closed.</span> |
| <span class="source-line-no">1636</span><span id="line-1636"> * @throws IOException e</span> |
| <span class="source-line-no">1637</span><span id="line-1637"> * @throws DroppedSnapshotException Thrown when replay of wal is required because a Snapshot was</span> |
| <span class="source-line-no">1638</span><span id="line-1638"> * not properly persisted. The region is put in closing mode, and</span> |
| <span class="source-line-no">1639</span><span id="line-1639"> * the caller MUST abort after this.</span> |
| <span class="source-line-no">1640</span><span id="line-1640"> */</span> |
| <span class="source-line-no">1641</span><span id="line-1641"> public Map<byte[], List<HStoreFile>> close(boolean abort, boolean ignoreStatus,</span> |
| <span class="source-line-no">1642</span><span id="line-1642"> boolean isGracefulStop) throws IOException {</span> |
| <span class="source-line-no">1643</span><span id="line-1643"> // Only allow one thread to close at a time. Serialize them so dual</span> |
| <span class="source-line-no">1644</span><span id="line-1644"> // threads attempting to close will run up against each other.</span> |
| <span class="source-line-no">1645</span><span id="line-1645"> MonitoredTask status = TaskMonitor.get().createStatus(</span> |
| <span class="source-line-no">1646</span><span id="line-1646"> "Closing region " + this.getRegionInfo().getEncodedName() + (abort ? " due to abort" : ""),</span> |
| <span class="source-line-no">1647</span><span id="line-1647"> ignoreStatus, true);</span> |
| <span class="source-line-no">1648</span><span id="line-1648"> status.setStatus("Waiting for close lock");</span> |
| <span class="source-line-no">1649</span><span id="line-1649"> try {</span> |
| <span class="source-line-no">1650</span><span id="line-1650"> synchronized (closeLock) {</span> |
| <span class="source-line-no">1651</span><span id="line-1651"> if (isGracefulStop && rsServices != null) {</span> |
| <span class="source-line-no">1652</span><span id="line-1652"> rsServices.getBlockCache().ifPresent(blockCache -> {</span> |
| <span class="source-line-no">1653</span><span id="line-1653"> if (blockCache instanceof CombinedBlockCache) {</span> |
| <span class="source-line-no">1654</span><span id="line-1654"> BlockCache l2 = ((CombinedBlockCache) blockCache).getSecondLevelCache();</span> |
| <span class="source-line-no">1655</span><span id="line-1655"> if (l2 instanceof BucketCache) {</span> |
| <span class="source-line-no">1656</span><span id="line-1656"> if (((BucketCache) l2).isCachePersistenceEnabled()) {</span> |
| <span class="source-line-no">1657</span><span id="line-1657"> LOG.info(</span> |
| <span class="source-line-no">1658</span><span id="line-1658"> "Closing region {} during a graceful stop, and cache persistence is on, "</span> |
| <span class="source-line-no">1659</span><span id="line-1659"> + "so setting evict on close to false. ",</span> |
| <span class="source-line-no">1660</span><span id="line-1660"> this.getRegionInfo().getRegionNameAsString());</span> |
| <span class="source-line-no">1661</span><span id="line-1661"> this.getStores().forEach(s -> s.getCacheConfig().setEvictOnClose(false));</span> |
| <span class="source-line-no">1662</span><span id="line-1662"> }</span> |
| <span class="source-line-no">1663</span><span id="line-1663"> }</span> |
| <span class="source-line-no">1664</span><span id="line-1664"> }</span> |
| <span class="source-line-no">1665</span><span id="line-1665"> });</span> |
| <span class="source-line-no">1666</span><span id="line-1666"> }</span> |
| <span class="source-line-no">1667</span><span id="line-1667"> return doClose(abort, status);</span> |
| <span class="source-line-no">1668</span><span id="line-1668"> }</span> |
| <span class="source-line-no">1669</span><span id="line-1669"> } finally {</span> |
| <span class="source-line-no">1670</span><span id="line-1670"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">1671</span><span id="line-1671"> LOG.debug("Region close journal for {}:\n{}", this.getRegionInfo().getEncodedName(),</span> |
| <span class="source-line-no">1672</span><span id="line-1672"> status.prettyPrintJournal());</span> |
| <span class="source-line-no">1673</span><span id="line-1673"> }</span> |
| <span class="source-line-no">1674</span><span id="line-1674"> status.cleanup();</span> |
| <span class="source-line-no">1675</span><span id="line-1675"> }</span> |
| <span class="source-line-no">1676</span><span id="line-1676"> }</span> |
| <span class="source-line-no">1677</span><span id="line-1677"></span> |
| <span class="source-line-no">1678</span><span id="line-1678"> /**</span> |
| <span class="source-line-no">1679</span><span id="line-1679"> * Exposed for some very specific unit tests.</span> |
| <span class="source-line-no">1680</span><span id="line-1680"> */</span> |
| <span class="source-line-no">1681</span><span id="line-1681"> public void setClosing(boolean closing) {</span> |
| <span class="source-line-no">1682</span><span id="line-1682"> this.closing.set(closing);</span> |
| <span class="source-line-no">1683</span><span id="line-1683"> }</span> |
| <span class="source-line-no">1684</span><span id="line-1684"></span> |
| <span class="source-line-no">1685</span><span id="line-1685"> /**</span> |
| <span class="source-line-no">1686</span><span id="line-1686"> * The {@link HRegion#doClose} will block forever if someone tries proving the dead lock via the</span> |
| <span class="source-line-no">1687</span><span id="line-1687"> * unit test. Instead of blocking, the {@link HRegion#doClose} will throw exception if you set the</span> |
| <span class="source-line-no">1688</span><span id="line-1688"> * timeout.</span> |
| <span class="source-line-no">1689</span><span id="line-1689"> * @param timeoutForWriteLock the second time to wait for the write lock in</span> |
| <span class="source-line-no">1690</span><span id="line-1690"> * {@link HRegion#doClose}</span> |
| <span class="source-line-no">1691</span><span id="line-1691"> */</span> |
| <span class="source-line-no">1692</span><span id="line-1692"> public void setTimeoutForWriteLock(long timeoutForWriteLock) {</span> |
| <span class="source-line-no">1693</span><span id="line-1693"> assert timeoutForWriteLock >= 0;</span> |
| <span class="source-line-no">1694</span><span id="line-1694"> this.timeoutForWriteLock = timeoutForWriteLock;</span> |
| <span class="source-line-no">1695</span><span id="line-1695"> }</span> |
| <span class="source-line-no">1696</span><span id="line-1696"></span> |
| <span class="source-line-no">1697</span><span id="line-1697"> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "UL_UNRELEASED_LOCK_EXCEPTION_PATH",</span> |
| <span class="source-line-no">1698</span><span id="line-1698"> justification = "I think FindBugs is confused")</span> |
| <span class="source-line-no">1699</span><span id="line-1699"> private Map<byte[], List<HStoreFile>> doClose(boolean abort, MonitoredTask status)</span> |
| <span class="source-line-no">1700</span><span id="line-1700"> throws IOException {</span> |
| <span class="source-line-no">1701</span><span id="line-1701"> if (isClosed()) {</span> |
| <span class="source-line-no">1702</span><span id="line-1702"> LOG.warn("Region " + this + " already closed");</span> |
| <span class="source-line-no">1703</span><span id="line-1703"> return null;</span> |
| <span class="source-line-no">1704</span><span id="line-1704"> }</span> |
| <span class="source-line-no">1705</span><span id="line-1705"></span> |
| <span class="source-line-no">1706</span><span id="line-1706"> if (coprocessorHost != null) {</span> |
| <span class="source-line-no">1707</span><span id="line-1707"> status.setStatus("Running coprocessor pre-close hooks");</span> |
| <span class="source-line-no">1708</span><span id="line-1708"> this.coprocessorHost.preClose(abort);</span> |
| <span class="source-line-no">1709</span><span id="line-1709"> }</span> |
| <span class="source-line-no">1710</span><span id="line-1710"> status.setStatus("Disabling compacts and flushes for region");</span> |
| <span class="source-line-no">1711</span><span id="line-1711"> boolean canFlush = true;</span> |
| <span class="source-line-no">1712</span><span id="line-1712"> synchronized (writestate) {</span> |
| <span class="source-line-no">1713</span><span id="line-1713"> // Disable compacting and flushing by background threads for this</span> |
| <span class="source-line-no">1714</span><span id="line-1714"> // region.</span> |
| <span class="source-line-no">1715</span><span id="line-1715"> canFlush = !writestate.readOnly;</span> |
| <span class="source-line-no">1716</span><span id="line-1716"> writestate.writesEnabled = false;</span> |
| <span class="source-line-no">1717</span><span id="line-1717"> LOG.debug("Closing {}, disabling compactions & flushes",</span> |
| <span class="source-line-no">1718</span><span id="line-1718"> this.getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">1719</span><span id="line-1719"> waitForFlushesAndCompactions();</span> |
| <span class="source-line-no">1720</span><span id="line-1720"> }</span> |
| <span class="source-line-no">1721</span><span id="line-1721"> // If we were not just flushing, is it worth doing a preflush...one</span> |
| <span class="source-line-no">1722</span><span id="line-1722"> // that will clear out of the bulk of the memstore before we put up</span> |
| <span class="source-line-no">1723</span><span id="line-1723"> // the close flag?</span> |
| <span class="source-line-no">1724</span><span id="line-1724"> if (!abort && worthPreFlushing() && canFlush) {</span> |
| <span class="source-line-no">1725</span><span id="line-1725"> status.setStatus("Pre-flushing region before close");</span> |
| <span class="source-line-no">1726</span><span id="line-1726"> LOG.info("Running close preflush of {}", this.getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">1727</span><span id="line-1727"> try {</span> |
| <span class="source-line-no">1728</span><span id="line-1728"> internalFlushcache(status);</span> |
| <span class="source-line-no">1729</span><span id="line-1729"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">1730</span><span id="line-1730"> // Failed to flush the region. Keep going.</span> |
| <span class="source-line-no">1731</span><span id="line-1731"> status.setStatus("Failed pre-flush " + this + "; " + ioe.getMessage());</span> |
| <span class="source-line-no">1732</span><span id="line-1732"> }</span> |
| <span class="source-line-no">1733</span><span id="line-1733"> }</span> |
| <span class="source-line-no">1734</span><span id="line-1734"> if (regionReplicationSink.isPresent()) {</span> |
| <span class="source-line-no">1735</span><span id="line-1735"> // stop replicating to secondary replicas</span> |
| <span class="source-line-no">1736</span><span id="line-1736"> // the open event marker can make secondary replicas refresh store files and catch up</span> |
| <span class="source-line-no">1737</span><span id="line-1737"> // everything, so here we just give up replicating later edits, to speed up the reopen process</span> |
| <span class="source-line-no">1738</span><span id="line-1738"> RegionReplicationSink sink = regionReplicationSink.get();</span> |
| <span class="source-line-no">1739</span><span id="line-1739"> sink.stop();</span> |
| <span class="source-line-no">1740</span><span id="line-1740"> try {</span> |
| <span class="source-line-no">1741</span><span id="line-1741"> regionReplicationSink.get().waitUntilStopped();</span> |
| <span class="source-line-no">1742</span><span id="line-1742"> } catch (InterruptedException e) {</span> |
| <span class="source-line-no">1743</span><span id="line-1743"> throw throwOnInterrupt(e);</span> |
| <span class="source-line-no">1744</span><span id="line-1744"> }</span> |
| <span class="source-line-no">1745</span><span id="line-1745"> }</span> |
| <span class="source-line-no">1746</span><span id="line-1746"> // Set the closing flag</span> |
| <span class="source-line-no">1747</span><span id="line-1747"> // From this point new arrivals at the region lock will get NSRE.</span> |
| <span class="source-line-no">1748</span><span id="line-1748"></span> |
| <span class="source-line-no">1749</span><span id="line-1749"> this.closing.set(true);</span> |
| <span class="source-line-no">1750</span><span id="line-1750"> LOG.info("Closing region {}", this);</span> |
| <span class="source-line-no">1751</span><span id="line-1751"></span> |
| <span class="source-line-no">1752</span><span id="line-1752"> // Acquire the close lock</span> |
| <span class="source-line-no">1753</span><span id="line-1753"></span> |
| <span class="source-line-no">1754</span><span id="line-1754"> // The configuration parameter CLOSE_WAIT_ABORT is overloaded to enable both</span> |
| <span class="source-line-no">1755</span><span id="line-1755"> // the new regionserver abort condition and interrupts for running requests.</span> |
| <span class="source-line-no">1756</span><span id="line-1756"> // If CLOSE_WAIT_ABORT is not enabled there is no change from earlier behavior,</span> |
| <span class="source-line-no">1757</span><span id="line-1757"> // we will not attempt to interrupt threads servicing requests nor crash out</span> |
| <span class="source-line-no">1758</span><span id="line-1758"> // the regionserver if something remains stubborn.</span> |
| <span class="source-line-no">1759</span><span id="line-1759"></span> |
| <span class="source-line-no">1760</span><span id="line-1760"> final boolean canAbort = conf.getBoolean(CLOSE_WAIT_ABORT, DEFAULT_CLOSE_WAIT_ABORT);</span> |
| <span class="source-line-no">1761</span><span id="line-1761"> boolean useTimedWait = false;</span> |
| <span class="source-line-no">1762</span><span id="line-1762"> if (timeoutForWriteLock != null && timeoutForWriteLock != Long.MAX_VALUE) {</span> |
| <span class="source-line-no">1763</span><span id="line-1763"> // convert legacy use of timeoutForWriteLock in seconds to new use in millis</span> |
| <span class="source-line-no">1764</span><span id="line-1764"> timeoutForWriteLock = TimeUnit.SECONDS.toMillis(timeoutForWriteLock);</span> |
| <span class="source-line-no">1765</span><span id="line-1765"> useTimedWait = true;</span> |
| <span class="source-line-no">1766</span><span id="line-1766"> } else if (canAbort) {</span> |
| <span class="source-line-no">1767</span><span id="line-1767"> timeoutForWriteLock = conf.getLong(CLOSE_WAIT_TIME, DEFAULT_CLOSE_WAIT_TIME);</span> |
| <span class="source-line-no">1768</span><span id="line-1768"> useTimedWait = true;</span> |
| <span class="source-line-no">1769</span><span id="line-1769"> }</span> |
| <span class="source-line-no">1770</span><span id="line-1770"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">1771</span><span id="line-1771"> LOG.debug((useTimedWait ? "Time limited wait" : "Waiting without time limit")</span> |
| <span class="source-line-no">1772</span><span id="line-1772"> + " for close lock on " + this);</span> |
| <span class="source-line-no">1773</span><span id="line-1773"> }</span> |
| <span class="source-line-no">1774</span><span id="line-1774"> final long closeWaitInterval = conf.getLong(CLOSE_WAIT_INTERVAL, DEFAULT_CLOSE_WAIT_INTERVAL);</span> |
| <span class="source-line-no">1775</span><span id="line-1775"> long elapsedWaitTime = 0;</span> |
| <span class="source-line-no">1776</span><span id="line-1776"> if (useTimedWait) {</span> |
| <span class="source-line-no">1777</span><span id="line-1777"> // Sanity check configuration</span> |
| <span class="source-line-no">1778</span><span id="line-1778"> long remainingWaitTime = timeoutForWriteLock;</span> |
| <span class="source-line-no">1779</span><span id="line-1779"> if (remainingWaitTime < closeWaitInterval) {</span> |
| <span class="source-line-no">1780</span><span id="line-1780"> LOG.warn("Time limit for close wait of " + timeoutForWriteLock</span> |
| <span class="source-line-no">1781</span><span id="line-1781"> + " ms is less than the configured lock acquisition wait interval " + closeWaitInterval</span> |
| <span class="source-line-no">1782</span><span id="line-1782"> + " ms, using wait interval as time limit");</span> |
| <span class="source-line-no">1783</span><span id="line-1783"> remainingWaitTime = closeWaitInterval;</span> |
| <span class="source-line-no">1784</span><span id="line-1784"> }</span> |
| <span class="source-line-no">1785</span><span id="line-1785"> boolean acquired = false;</span> |
| <span class="source-line-no">1786</span><span id="line-1786"> do {</span> |
| <span class="source-line-no">1787</span><span id="line-1787"> long start = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">1788</span><span id="line-1788"> try {</span> |
| <span class="source-line-no">1789</span><span id="line-1789"> acquired = lock.writeLock().tryLock(Math.min(remainingWaitTime, closeWaitInterval),</span> |
| <span class="source-line-no">1790</span><span id="line-1790"> TimeUnit.MILLISECONDS);</span> |
| <span class="source-line-no">1791</span><span id="line-1791"> } catch (InterruptedException e) {</span> |
| <span class="source-line-no">1792</span><span id="line-1792"> // Interrupted waiting for close lock. More likely the server is shutting down, not</span> |
| <span class="source-line-no">1793</span><span id="line-1793"> // normal operation, so aborting upon interrupt while waiting on this lock would not</span> |
| <span class="source-line-no">1794</span><span id="line-1794"> // provide much value. Throw an IOE (as IIOE) like we would in the case where we</span> |
| <span class="source-line-no">1795</span><span id="line-1795"> // fail to acquire the lock.</span> |
| <span class="source-line-no">1796</span><span id="line-1796"> String msg = "Interrupted while waiting for close lock on " + this;</span> |
| <span class="source-line-no">1797</span><span id="line-1797"> LOG.warn(msg, e);</span> |
| <span class="source-line-no">1798</span><span id="line-1798"> throw (InterruptedIOException) new InterruptedIOException(msg).initCause(e);</span> |
| <span class="source-line-no">1799</span><span id="line-1799"> }</span> |
| <span class="source-line-no">1800</span><span id="line-1800"> long elapsed = EnvironmentEdgeManager.currentTime() - start;</span> |
| <span class="source-line-no">1801</span><span id="line-1801"> elapsedWaitTime += elapsed;</span> |
| <span class="source-line-no">1802</span><span id="line-1802"> remainingWaitTime -= elapsed;</span> |
| <span class="source-line-no">1803</span><span id="line-1803"> if (canAbort && !acquired && remainingWaitTime > 0) {</span> |
| <span class="source-line-no">1804</span><span id="line-1804"> // Before we loop to wait again, interrupt all region operations that might</span> |
| <span class="source-line-no">1805</span><span id="line-1805"> // still be in progress, to encourage them to break out of waiting states or</span> |
| <span class="source-line-no">1806</span><span id="line-1806"> // inner loops, throw an exception to clients, and release the read lock via</span> |
| <span class="source-line-no">1807</span><span id="line-1807"> // endRegionOperation.</span> |
| <span class="source-line-no">1808</span><span id="line-1808"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">1809</span><span id="line-1809"> LOG.debug("Interrupting region operations after waiting for close lock for "</span> |
| <span class="source-line-no">1810</span><span id="line-1810"> + elapsedWaitTime + " ms on " + this + ", " + remainingWaitTime + " ms remaining");</span> |
| <span class="source-line-no">1811</span><span id="line-1811"> }</span> |
| <span class="source-line-no">1812</span><span id="line-1812"> interruptRegionOperations();</span> |
| <span class="source-line-no">1813</span><span id="line-1813"> }</span> |
| <span class="source-line-no">1814</span><span id="line-1814"> } while (!acquired && remainingWaitTime > 0);</span> |
| <span class="source-line-no">1815</span><span id="line-1815"></span> |
| <span class="source-line-no">1816</span><span id="line-1816"> // If we fail to acquire the lock, trigger an abort if we can; otherwise throw an IOE</span> |
| <span class="source-line-no">1817</span><span id="line-1817"> // to let the caller know we could not proceed with the close.</span> |
| <span class="source-line-no">1818</span><span id="line-1818"> if (!acquired) {</span> |
| <span class="source-line-no">1819</span><span id="line-1819"> String msg =</span> |
| <span class="source-line-no">1820</span><span id="line-1820"> "Failed to acquire close lock on " + this + " after waiting " + elapsedWaitTime + " ms";</span> |
| <span class="source-line-no">1821</span><span id="line-1821"> LOG.error(msg);</span> |
| <span class="source-line-no">1822</span><span id="line-1822"> if (canAbort) {</span> |
| <span class="source-line-no">1823</span><span id="line-1823"> // If we failed to acquire the write lock, abort the server</span> |
| <span class="source-line-no">1824</span><span id="line-1824"> rsServices.abort(msg, null);</span> |
| <span class="source-line-no">1825</span><span id="line-1825"> }</span> |
| <span class="source-line-no">1826</span><span id="line-1826"> throw new IOException(msg);</span> |
| <span class="source-line-no">1827</span><span id="line-1827"> }</span> |
| <span class="source-line-no">1828</span><span id="line-1828"></span> |
| <span class="source-line-no">1829</span><span id="line-1829"> } else {</span> |
| <span class="source-line-no">1830</span><span id="line-1830"></span> |
| <span class="source-line-no">1831</span><span id="line-1831"> long start = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">1832</span><span id="line-1832"> lock.writeLock().lock();</span> |
| <span class="source-line-no">1833</span><span id="line-1833"> elapsedWaitTime = EnvironmentEdgeManager.currentTime() - start;</span> |
| <span class="source-line-no">1834</span><span id="line-1834"></span> |
| <span class="source-line-no">1835</span><span id="line-1835"> }</span> |
| <span class="source-line-no">1836</span><span id="line-1836"></span> |
| <span class="source-line-no">1837</span><span id="line-1837"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">1838</span><span id="line-1838"> LOG.debug("Acquired close lock on " + this + " after waiting " + elapsedWaitTime + " ms");</span> |
| <span class="source-line-no">1839</span><span id="line-1839"> }</span> |
| <span class="source-line-no">1840</span><span id="line-1840"></span> |
| <span class="source-line-no">1841</span><span id="line-1841"> status.setStatus("Disabling writes for close");</span> |
| <span class="source-line-no">1842</span><span id="line-1842"> try {</span> |
| <span class="source-line-no">1843</span><span id="line-1843"> if (this.isClosed()) {</span> |
| <span class="source-line-no">1844</span><span id="line-1844"> status.abort("Already got closed by another process");</span> |
| <span class="source-line-no">1845</span><span id="line-1845"> // SplitTransaction handles the null</span> |
| <span class="source-line-no">1846</span><span id="line-1846"> return null;</span> |
| <span class="source-line-no">1847</span><span id="line-1847"> }</span> |
| <span class="source-line-no">1848</span><span id="line-1848"> LOG.debug("Updates disabled for region " + this);</span> |
| <span class="source-line-no">1849</span><span id="line-1849"> // Don't flush the cache if we are aborting</span> |
| <span class="source-line-no">1850</span><span id="line-1850"> if (!abort && canFlush) {</span> |
| <span class="source-line-no">1851</span><span id="line-1851"> int failedfFlushCount = 0;</span> |
| <span class="source-line-no">1852</span><span id="line-1852"> int flushCount = 0;</span> |
| <span class="source-line-no">1853</span><span id="line-1853"> long tmp = 0;</span> |
| <span class="source-line-no">1854</span><span id="line-1854"> long remainingSize = this.memStoreSizing.getDataSize();</span> |
| <span class="source-line-no">1855</span><span id="line-1855"> while (remainingSize > 0) {</span> |
| <span class="source-line-no">1856</span><span id="line-1856"> try {</span> |
| <span class="source-line-no">1857</span><span id="line-1857"> internalFlushcache(status);</span> |
| <span class="source-line-no">1858</span><span id="line-1858"> if (flushCount > 0) {</span> |
| <span class="source-line-no">1859</span><span id="line-1859"> LOG.info("Running extra flush, " + flushCount + " (carrying snapshot?) " + this);</span> |
| <span class="source-line-no">1860</span><span id="line-1860"> }</span> |
| <span class="source-line-no">1861</span><span id="line-1861"> flushCount++;</span> |
| <span class="source-line-no">1862</span><span id="line-1862"> tmp = this.memStoreSizing.getDataSize();</span> |
| <span class="source-line-no">1863</span><span id="line-1863"> if (tmp >= remainingSize) {</span> |
| <span class="source-line-no">1864</span><span id="line-1864"> failedfFlushCount++;</span> |
| <span class="source-line-no">1865</span><span id="line-1865"> }</span> |
| <span class="source-line-no">1866</span><span id="line-1866"> remainingSize = tmp;</span> |
| <span class="source-line-no">1867</span><span id="line-1867"> if (failedfFlushCount > 5) {</span> |
| <span class="source-line-no">1868</span><span id="line-1868"> // If we failed 5 times and are unable to clear memory, abort</span> |
| <span class="source-line-no">1869</span><span id="line-1869"> // so we do not lose data</span> |
| <span class="source-line-no">1870</span><span id="line-1870"> throw new DroppedSnapshotException("Failed clearing memory after " + flushCount</span> |
| <span class="source-line-no">1871</span><span id="line-1871"> + " attempts on region: " + Bytes.toStringBinary(getRegionInfo().getRegionName()));</span> |
| <span class="source-line-no">1872</span><span id="line-1872"> }</span> |
| <span class="source-line-no">1873</span><span id="line-1873"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">1874</span><span id="line-1874"> status.setStatus("Failed flush " + this + ", putting online again");</span> |
| <span class="source-line-no">1875</span><span id="line-1875"> synchronized (writestate) {</span> |
| <span class="source-line-no">1876</span><span id="line-1876"> writestate.writesEnabled = true;</span> |
| <span class="source-line-no">1877</span><span id="line-1877"> }</span> |
| <span class="source-line-no">1878</span><span id="line-1878"> // Have to throw to upper layers. I can't abort server from here.</span> |
| <span class="source-line-no">1879</span><span id="line-1879"> throw ioe;</span> |
| <span class="source-line-no">1880</span><span id="line-1880"> }</span> |
| <span class="source-line-no">1881</span><span id="line-1881"> }</span> |
| <span class="source-line-no">1882</span><span id="line-1882"> }</span> |
| <span class="source-line-no">1883</span><span id="line-1883"></span> |
| <span class="source-line-no">1884</span><span id="line-1884"> Map<byte[], List<HStoreFile>> result = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">1885</span><span id="line-1885"> if (!stores.isEmpty()) {</span> |
| <span class="source-line-no">1886</span><span id="line-1886"> // initialize the thread pool for closing stores in parallel.</span> |
| <span class="source-line-no">1887</span><span id="line-1887"> ThreadPoolExecutor storeCloserThreadPool =</span> |
| <span class="source-line-no">1888</span><span id="line-1888"> getStoreOpenAndCloseThreadPool("StoreCloser-" + getRegionInfo().getRegionNameAsString());</span> |
| <span class="source-line-no">1889</span><span id="line-1889"> CompletionService<Pair<byte[], Collection<HStoreFile>>> completionService =</span> |
| <span class="source-line-no">1890</span><span id="line-1890"> new ExecutorCompletionService<>(storeCloserThreadPool);</span> |
| <span class="source-line-no">1891</span><span id="line-1891"></span> |
| <span class="source-line-no">1892</span><span id="line-1892"> // close each store in parallel</span> |
| <span class="source-line-no">1893</span><span id="line-1893"> for (HStore store : stores.values()) {</span> |
| <span class="source-line-no">1894</span><span id="line-1894"> MemStoreSize mss = store.getFlushableSize();</span> |
| <span class="source-line-no">1895</span><span id="line-1895"> if (!(abort || mss.getDataSize() == 0 || writestate.readOnly)) {</span> |
| <span class="source-line-no">1896</span><span id="line-1896"> if (getRegionServerServices() != null) {</span> |
| <span class="source-line-no">1897</span><span id="line-1897"> getRegionServerServices().abort("Assertion failed while closing store "</span> |
| <span class="source-line-no">1898</span><span id="line-1898"> + getRegionInfo().getRegionNameAsString() + " " + store</span> |
| <span class="source-line-no">1899</span><span id="line-1899"> + ". flushableSize expected=0, actual={" + mss + "}. Current memStoreSize="</span> |
| <span class="source-line-no">1900</span><span id="line-1900"> + this.memStoreSizing.getMemStoreSize() + ". Maybe a coprocessor "</span> |
| <span class="source-line-no">1901</span><span id="line-1901"> + "operation failed and left the memstore in a partially updated state.", null);</span> |
| <span class="source-line-no">1902</span><span id="line-1902"> }</span> |
| <span class="source-line-no">1903</span><span id="line-1903"> }</span> |
| <span class="source-line-no">1904</span><span id="line-1904"> completionService.submit(new Callable<Pair<byte[], Collection<HStoreFile>>>() {</span> |
| <span class="source-line-no">1905</span><span id="line-1905"> @Override</span> |
| <span class="source-line-no">1906</span><span id="line-1906"> public Pair<byte[], Collection<HStoreFile>> call() throws IOException {</span> |
| <span class="source-line-no">1907</span><span id="line-1907"> return new Pair<>(store.getColumnFamilyDescriptor().getName(), store.close());</span> |
| <span class="source-line-no">1908</span><span id="line-1908"> }</span> |
| <span class="source-line-no">1909</span><span id="line-1909"> });</span> |
| <span class="source-line-no">1910</span><span id="line-1910"> }</span> |
| <span class="source-line-no">1911</span><span id="line-1911"> try {</span> |
| <span class="source-line-no">1912</span><span id="line-1912"> for (int i = 0; i < stores.size(); i++) {</span> |
| <span class="source-line-no">1913</span><span id="line-1913"> Future<Pair<byte[], Collection<HStoreFile>>> future = completionService.take();</span> |
| <span class="source-line-no">1914</span><span id="line-1914"> Pair<byte[], Collection<HStoreFile>> storeFiles = future.get();</span> |
| <span class="source-line-no">1915</span><span id="line-1915"> List<HStoreFile> familyFiles = result.get(storeFiles.getFirst());</span> |
| <span class="source-line-no">1916</span><span id="line-1916"> if (familyFiles == null) {</span> |
| <span class="source-line-no">1917</span><span id="line-1917"> familyFiles = new ArrayList<>();</span> |
| <span class="source-line-no">1918</span><span id="line-1918"> result.put(storeFiles.getFirst(), familyFiles);</span> |
| <span class="source-line-no">1919</span><span id="line-1919"> }</span> |
| <span class="source-line-no">1920</span><span id="line-1920"> familyFiles.addAll(storeFiles.getSecond());</span> |
| <span class="source-line-no">1921</span><span id="line-1921"> }</span> |
| <span class="source-line-no">1922</span><span id="line-1922"> } catch (InterruptedException e) {</span> |
| <span class="source-line-no">1923</span><span id="line-1923"> throw throwOnInterrupt(e);</span> |
| <span class="source-line-no">1924</span><span id="line-1924"> } catch (ExecutionException e) {</span> |
| <span class="source-line-no">1925</span><span id="line-1925"> Throwable cause = e.getCause();</span> |
| <span class="source-line-no">1926</span><span id="line-1926"> if (cause instanceof IOException) {</span> |
| <span class="source-line-no">1927</span><span id="line-1927"> throw (IOException) cause;</span> |
| <span class="source-line-no">1928</span><span id="line-1928"> }</span> |
| <span class="source-line-no">1929</span><span id="line-1929"> throw new IOException(cause);</span> |
| <span class="source-line-no">1930</span><span id="line-1930"> } finally {</span> |
| <span class="source-line-no">1931</span><span id="line-1931"> storeCloserThreadPool.shutdownNow();</span> |
| <span class="source-line-no">1932</span><span id="line-1932"> }</span> |
| <span class="source-line-no">1933</span><span id="line-1933"> }</span> |
| <span class="source-line-no">1934</span><span id="line-1934"></span> |
| <span class="source-line-no">1935</span><span id="line-1935"> status.setStatus("Writing region close event to WAL");</span> |
| <span class="source-line-no">1936</span><span id="line-1936"> // Always write close marker to wal even for read only table. This is not a big problem as we</span> |
| <span class="source-line-no">1937</span><span id="line-1937"> // do not write any data into the region; it is just a meta edit in the WAL file.</span> |
| <span class="source-line-no">1938</span><span id="line-1938"> if (</span> |
| <span class="source-line-no">1939</span><span id="line-1939"> !abort && wal != null && getRegionServerServices() != null</span> |
| <span class="source-line-no">1940</span><span id="line-1940"> && RegionReplicaUtil.isDefaultReplica(getRegionInfo())</span> |
| <span class="source-line-no">1941</span><span id="line-1941"> ) {</span> |
| <span class="source-line-no">1942</span><span id="line-1942"> writeRegionCloseMarker(wal);</span> |
| <span class="source-line-no">1943</span><span id="line-1943"> }</span> |
| <span class="source-line-no">1944</span><span id="line-1944"> this.closed.set(true);</span> |
| <span class="source-line-no">1945</span><span id="line-1945"></span> |
| <span class="source-line-no">1946</span><span id="line-1946"> // Decrease refCount of table latency metric registry.</span> |
| <span class="source-line-no">1947</span><span id="line-1947"> // Do this after closed#set to make sure only -1.</span> |
| <span class="source-line-no">1948</span><span id="line-1948"> if (metricsTableRequests != null) {</span> |
| <span class="source-line-no">1949</span><span id="line-1949"> metricsTableRequests.removeRegistry();</span> |
| <span class="source-line-no">1950</span><span id="line-1950"> }</span> |
| <span class="source-line-no">1951</span><span id="line-1951"></span> |
| <span class="source-line-no">1952</span><span id="line-1952"> if (!canFlush) {</span> |
| <span class="source-line-no">1953</span><span id="line-1953"> decrMemStoreSize(this.memStoreSizing.getMemStoreSize());</span> |
| <span class="source-line-no">1954</span><span id="line-1954"> } else if (this.memStoreSizing.getDataSize() != 0) {</span> |
| <span class="source-line-no">1955</span><span id="line-1955"> LOG.error("Memstore data size is {} in region {}", this.memStoreSizing.getDataSize(), this);</span> |
| <span class="source-line-no">1956</span><span id="line-1956"> }</span> |
| <span class="source-line-no">1957</span><span id="line-1957"> if (coprocessorHost != null) {</span> |
| <span class="source-line-no">1958</span><span id="line-1958"> status.setStatus("Running coprocessor post-close hooks");</span> |
| <span class="source-line-no">1959</span><span id="line-1959"> this.coprocessorHost.postClose(abort);</span> |
| <span class="source-line-no">1960</span><span id="line-1960"> }</span> |
| <span class="source-line-no">1961</span><span id="line-1961"> if (this.metricsRegion != null) {</span> |
| <span class="source-line-no">1962</span><span id="line-1962"> this.metricsRegion.close();</span> |
| <span class="source-line-no">1963</span><span id="line-1963"> }</span> |
| <span class="source-line-no">1964</span><span id="line-1964"> if (this.metricsRegionWrapper != null) {</span> |
| <span class="source-line-no">1965</span><span id="line-1965"> Closeables.close(this.metricsRegionWrapper, true);</span> |
| <span class="source-line-no">1966</span><span id="line-1966"> }</span> |
| <span class="source-line-no">1967</span><span id="line-1967"> status.markComplete("Closed");</span> |
| <span class="source-line-no">1968</span><span id="line-1968"> LOG.info("Closed {}", this);</span> |
| <span class="source-line-no">1969</span><span id="line-1969"> return result;</span> |
| <span class="source-line-no">1970</span><span id="line-1970"> } finally {</span> |
| <span class="source-line-no">1971</span><span id="line-1971"> lock.writeLock().unlock();</span> |
| <span class="source-line-no">1972</span><span id="line-1972"> }</span> |
| <span class="source-line-no">1973</span><span id="line-1973"> }</span> |
| <span class="source-line-no">1974</span><span id="line-1974"></span> |
| <span class="source-line-no">1975</span><span id="line-1975"> /** Wait for all current flushes and compactions of the region to complete */</span> |
| <span class="source-line-no">1976</span><span id="line-1976"> // TODO HBASE-18906. Check the usage (if any) in Phoenix and expose this or give alternate way for</span> |
| <span class="source-line-no">1977</span><span id="line-1977"> // Phoenix needs.</span> |
| <span class="source-line-no">1978</span><span id="line-1978"> public void waitForFlushesAndCompactions() {</span> |
| <span class="source-line-no">1979</span><span id="line-1979"> synchronized (writestate) {</span> |
| <span class="source-line-no">1980</span><span id="line-1980"> if (this.writestate.readOnly) {</span> |
| <span class="source-line-no">1981</span><span id="line-1981"> // we should not wait for replayed flushed if we are read only (for example in case the</span> |
| <span class="source-line-no">1982</span><span id="line-1982"> // region is a secondary replica).</span> |
| <span class="source-line-no">1983</span><span id="line-1983"> return;</span> |
| <span class="source-line-no">1984</span><span id="line-1984"> }</span> |
| <span class="source-line-no">1985</span><span id="line-1985"> boolean interrupted = false;</span> |
| <span class="source-line-no">1986</span><span id="line-1986"> try {</span> |
| <span class="source-line-no">1987</span><span id="line-1987"> while (writestate.compacting.get() > 0 || writestate.flushing) {</span> |
| <span class="source-line-no">1988</span><span id="line-1988"> LOG.debug("waiting for " + writestate.compacting + " compactions"</span> |
| <span class="source-line-no">1989</span><span id="line-1989"> + (writestate.flushing ? " & cache flush" : "") + " to complete for region " + this);</span> |
| <span class="source-line-no">1990</span><span id="line-1990"> try {</span> |
| <span class="source-line-no">1991</span><span id="line-1991"> writestate.wait();</span> |
| <span class="source-line-no">1992</span><span id="line-1992"> } catch (InterruptedException iex) {</span> |
| <span class="source-line-no">1993</span><span id="line-1993"> // essentially ignore and propagate the interrupt back up</span> |
| <span class="source-line-no">1994</span><span id="line-1994"> LOG.warn("Interrupted while waiting in region {}", this);</span> |
| <span class="source-line-no">1995</span><span id="line-1995"> interrupted = true;</span> |
| <span class="source-line-no">1996</span><span id="line-1996"> break;</span> |
| <span class="source-line-no">1997</span><span id="line-1997"> }</span> |
| <span class="source-line-no">1998</span><span id="line-1998"> }</span> |
| <span class="source-line-no">1999</span><span id="line-1999"> } finally {</span> |
| <span class="source-line-no">2000</span><span id="line-2000"> if (interrupted) {</span> |
| <span class="source-line-no">2001</span><span id="line-2001"> Thread.currentThread().interrupt();</span> |
| <span class="source-line-no">2002</span><span id="line-2002"> }</span> |
| <span class="source-line-no">2003</span><span id="line-2003"> }</span> |
| <span class="source-line-no">2004</span><span id="line-2004"> }</span> |
| <span class="source-line-no">2005</span><span id="line-2005"> }</span> |
| <span class="source-line-no">2006</span><span id="line-2006"></span> |
| <span class="source-line-no">2007</span><span id="line-2007"> /**</span> |
| <span class="source-line-no">2008</span><span id="line-2008"> * Wait for all current flushes of the region to complete</span> |
| <span class="source-line-no">2009</span><span id="line-2009"> */</span> |
| <span class="source-line-no">2010</span><span id="line-2010"> public void waitForFlushes() {</span> |
| <span class="source-line-no">2011</span><span id="line-2011"> waitForFlushes(0);// Unbound wait</span> |
| <span class="source-line-no">2012</span><span id="line-2012"> }</span> |
| <span class="source-line-no">2013</span><span id="line-2013"></span> |
| <span class="source-line-no">2014</span><span id="line-2014"> @Override</span> |
| <span class="source-line-no">2015</span><span id="line-2015"> public boolean waitForFlushes(long timeout) {</span> |
| <span class="source-line-no">2016</span><span id="line-2016"> synchronized (writestate) {</span> |
| <span class="source-line-no">2017</span><span id="line-2017"> if (this.writestate.readOnly) {</span> |
| <span class="source-line-no">2018</span><span id="line-2018"> // we should not wait for replayed flushed if we are read only (for example in case the</span> |
| <span class="source-line-no">2019</span><span id="line-2019"> // region is a secondary replica).</span> |
| <span class="source-line-no">2020</span><span id="line-2020"> return true;</span> |
| <span class="source-line-no">2021</span><span id="line-2021"> }</span> |
| <span class="source-line-no">2022</span><span id="line-2022"> if (!writestate.flushing) return true;</span> |
| <span class="source-line-no">2023</span><span id="line-2023"> long start = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">2024</span><span id="line-2024"> long duration = 0;</span> |
| <span class="source-line-no">2025</span><span id="line-2025"> boolean interrupted = false;</span> |
| <span class="source-line-no">2026</span><span id="line-2026"> LOG.debug("waiting for cache flush to complete for region " + this);</span> |
| <span class="source-line-no">2027</span><span id="line-2027"> try {</span> |
| <span class="source-line-no">2028</span><span id="line-2028"> while (writestate.flushing) {</span> |
| <span class="source-line-no">2029</span><span id="line-2029"> if (timeout > 0 && duration >= timeout) break;</span> |
| <span class="source-line-no">2030</span><span id="line-2030"> try {</span> |
| <span class="source-line-no">2031</span><span id="line-2031"> long toWait = timeout == 0 ? 0 : (timeout - duration);</span> |
| <span class="source-line-no">2032</span><span id="line-2032"> writestate.wait(toWait);</span> |
| <span class="source-line-no">2033</span><span id="line-2033"> } catch (InterruptedException iex) {</span> |
| <span class="source-line-no">2034</span><span id="line-2034"> // essentially ignore and propagate the interrupt back up</span> |
| <span class="source-line-no">2035</span><span id="line-2035"> LOG.warn("Interrupted while waiting in region {}", this);</span> |
| <span class="source-line-no">2036</span><span id="line-2036"> interrupted = true;</span> |
| <span class="source-line-no">2037</span><span id="line-2037"> break;</span> |
| <span class="source-line-no">2038</span><span id="line-2038"> } finally {</span> |
| <span class="source-line-no">2039</span><span id="line-2039"> duration = EnvironmentEdgeManager.currentTime() - start;</span> |
| <span class="source-line-no">2040</span><span id="line-2040"> }</span> |
| <span class="source-line-no">2041</span><span id="line-2041"> }</span> |
| <span class="source-line-no">2042</span><span id="line-2042"> } finally {</span> |
| <span class="source-line-no">2043</span><span id="line-2043"> if (interrupted) {</span> |
| <span class="source-line-no">2044</span><span id="line-2044"> Thread.currentThread().interrupt();</span> |
| <span class="source-line-no">2045</span><span id="line-2045"> }</span> |
| <span class="source-line-no">2046</span><span id="line-2046"> }</span> |
| <span class="source-line-no">2047</span><span id="line-2047"> LOG.debug("Waited {} ms for region {} flush to complete", duration, this);</span> |
| <span class="source-line-no">2048</span><span id="line-2048"> return !(writestate.flushing);</span> |
| <span class="source-line-no">2049</span><span id="line-2049"> }</span> |
| <span class="source-line-no">2050</span><span id="line-2050"> }</span> |
| <span class="source-line-no">2051</span><span id="line-2051"></span> |
| <span class="source-line-no">2052</span><span id="line-2052"> @Override</span> |
| <span class="source-line-no">2053</span><span id="line-2053"> public Configuration getReadOnlyConfiguration() {</span> |
| <span class="source-line-no">2054</span><span id="line-2054"> return new ReadOnlyConfiguration(this.conf);</span> |
| <span class="source-line-no">2055</span><span id="line-2055"> }</span> |
| <span class="source-line-no">2056</span><span id="line-2056"></span> |
| <span class="source-line-no">2057</span><span id="line-2057"> @Override</span> |
| <span class="source-line-no">2058</span><span id="line-2058"> public int getMinBlockSizeBytes() {</span> |
| <span class="source-line-no">2059</span><span id="line-2059"> return minBlockSizeBytes;</span> |
| <span class="source-line-no">2060</span><span id="line-2060"> }</span> |
| <span class="source-line-no">2061</span><span id="line-2061"></span> |
| <span class="source-line-no">2062</span><span id="line-2062"> private ThreadPoolExecutor getStoreOpenAndCloseThreadPool(final String threadNamePrefix) {</span> |
| <span class="source-line-no">2063</span><span id="line-2063"> int numStores = Math.max(1, this.htableDescriptor.getColumnFamilyCount());</span> |
| <span class="source-line-no">2064</span><span id="line-2064"> int maxThreads = Math.min(numStores, conf.getInt(HConstants.HSTORE_OPEN_AND_CLOSE_THREADS_MAX,</span> |
| <span class="source-line-no">2065</span><span id="line-2065"> HConstants.DEFAULT_HSTORE_OPEN_AND_CLOSE_THREADS_MAX));</span> |
| <span class="source-line-no">2066</span><span id="line-2066"> return getOpenAndCloseThreadPool(maxThreads, threadNamePrefix);</span> |
| <span class="source-line-no">2067</span><span id="line-2067"> }</span> |
| <span class="source-line-no">2068</span><span id="line-2068"></span> |
| <span class="source-line-no">2069</span><span id="line-2069"> ThreadPoolExecutor getStoreFileOpenAndCloseThreadPool(final String threadNamePrefix) {</span> |
| <span class="source-line-no">2070</span><span id="line-2070"> int numStores = Math.max(1, this.htableDescriptor.getColumnFamilyCount());</span> |
| <span class="source-line-no">2071</span><span id="line-2071"> int maxThreads = Math.max(1, conf.getInt(HConstants.HSTORE_OPEN_AND_CLOSE_THREADS_MAX,</span> |
| <span class="source-line-no">2072</span><span id="line-2072"> HConstants.DEFAULT_HSTORE_OPEN_AND_CLOSE_THREADS_MAX) / numStores);</span> |
| <span class="source-line-no">2073</span><span id="line-2073"> return getOpenAndCloseThreadPool(maxThreads, threadNamePrefix);</span> |
| <span class="source-line-no">2074</span><span id="line-2074"> }</span> |
| <span class="source-line-no">2075</span><span id="line-2075"></span> |
| <span class="source-line-no">2076</span><span id="line-2076"> private static ThreadPoolExecutor getOpenAndCloseThreadPool(int maxThreads,</span> |
| <span class="source-line-no">2077</span><span id="line-2077"> final String threadNamePrefix) {</span> |
| <span class="source-line-no">2078</span><span id="line-2078"> return Threads.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,</span> |
| <span class="source-line-no">2079</span><span id="line-2079"> new ThreadFactory() {</span> |
| <span class="source-line-no">2080</span><span id="line-2080"> private int count = 1;</span> |
| <span class="source-line-no">2081</span><span id="line-2081"></span> |
| <span class="source-line-no">2082</span><span id="line-2082"> @Override</span> |
| <span class="source-line-no">2083</span><span id="line-2083"> public Thread newThread(Runnable r) {</span> |
| <span class="source-line-no">2084</span><span id="line-2084"> return new Thread(r, threadNamePrefix + "-" + count++);</span> |
| <span class="source-line-no">2085</span><span id="line-2085"> }</span> |
| <span class="source-line-no">2086</span><span id="line-2086"> });</span> |
| <span class="source-line-no">2087</span><span id="line-2087"> }</span> |
| <span class="source-line-no">2088</span><span id="line-2088"></span> |
| <span class="source-line-no">2089</span><span id="line-2089"> /** Returns True if its worth doing a flush before we put up the close flag. */</span> |
| <span class="source-line-no">2090</span><span id="line-2090"> private boolean worthPreFlushing() {</span> |
| <span class="source-line-no">2091</span><span id="line-2091"> return this.memStoreSizing.getDataSize()</span> |
| <span class="source-line-no">2092</span><span id="line-2092"> > this.conf.getLong("hbase.hregion.preclose.flush.size", 1024 * 1024 * 5);</span> |
| <span class="source-line-no">2093</span><span id="line-2093"> }</span> |
| <span class="source-line-no">2094</span><span id="line-2094"></span> |
| <span class="source-line-no">2095</span><span id="line-2095"> //////////////////////////////////////////////////////////////////////////////</span> |
| <span class="source-line-no">2096</span><span id="line-2096"> // HRegion accessors</span> |
| <span class="source-line-no">2097</span><span id="line-2097"> //////////////////////////////////////////////////////////////////////////////</span> |
| <span class="source-line-no">2098</span><span id="line-2098"></span> |
| <span class="source-line-no">2099</span><span id="line-2099"> @Override</span> |
| <span class="source-line-no">2100</span><span id="line-2100"> public TableDescriptor getTableDescriptor() {</span> |
| <span class="source-line-no">2101</span><span id="line-2101"> return this.htableDescriptor;</span> |
| <span class="source-line-no">2102</span><span id="line-2102"> }</span> |
| <span class="source-line-no">2103</span><span id="line-2103"></span> |
| <span class="source-line-no">2104</span><span id="line-2104"> public void setTableDescriptor(TableDescriptor desc) {</span> |
| <span class="source-line-no">2105</span><span id="line-2105"> htableDescriptor = desc;</span> |
| <span class="source-line-no">2106</span><span id="line-2106"> }</span> |
| <span class="source-line-no">2107</span><span id="line-2107"></span> |
| <span class="source-line-no">2108</span><span id="line-2108"> /** Returns WAL in use for this region */</span> |
| <span class="source-line-no">2109</span><span id="line-2109"> public WAL getWAL() {</span> |
| <span class="source-line-no">2110</span><span id="line-2110"> return this.wal;</span> |
| <span class="source-line-no">2111</span><span id="line-2111"> }</span> |
| <span class="source-line-no">2112</span><span id="line-2112"></span> |
| <span class="source-line-no">2113</span><span id="line-2113"> public BlockCache getBlockCache() {</span> |
| <span class="source-line-no">2114</span><span id="line-2114"> return this.blockCache;</span> |
| <span class="source-line-no">2115</span><span id="line-2115"> }</span> |
| <span class="source-line-no">2116</span><span id="line-2116"></span> |
| <span class="source-line-no">2117</span><span id="line-2117"> /**</span> |
| <span class="source-line-no">2118</span><span id="line-2118"> * Only used for unit test which doesn't start region server.</span> |
| <span class="source-line-no">2119</span><span id="line-2119"> */</span> |
| <span class="source-line-no">2120</span><span id="line-2120"> public void setBlockCache(BlockCache blockCache) {</span> |
| <span class="source-line-no">2121</span><span id="line-2121"> this.blockCache = blockCache;</span> |
| <span class="source-line-no">2122</span><span id="line-2122"> }</span> |
| <span class="source-line-no">2123</span><span id="line-2123"></span> |
| <span class="source-line-no">2124</span><span id="line-2124"> public MobFileCache getMobFileCache() {</span> |
| <span class="source-line-no">2125</span><span id="line-2125"> return this.mobFileCache;</span> |
| <span class="source-line-no">2126</span><span id="line-2126"> }</span> |
| <span class="source-line-no">2127</span><span id="line-2127"></span> |
| <span class="source-line-no">2128</span><span id="line-2128"> /**</span> |
| <span class="source-line-no">2129</span><span id="line-2129"> * Only used for unit test which doesn't start region server.</span> |
| <span class="source-line-no">2130</span><span id="line-2130"> */</span> |
| <span class="source-line-no">2131</span><span id="line-2131"> public void setMobFileCache(MobFileCache mobFileCache) {</span> |
| <span class="source-line-no">2132</span><span id="line-2132"> this.mobFileCache = mobFileCache;</span> |
| <span class="source-line-no">2133</span><span id="line-2133"> }</span> |
| <span class="source-line-no">2134</span><span id="line-2134"></span> |
| <span class="source-line-no">2135</span><span id="line-2135"> /** Returns split policy for this region. */</span> |
| <span class="source-line-no">2136</span><span id="line-2136"> RegionSplitPolicy getSplitPolicy() {</span> |
| <span class="source-line-no">2137</span><span id="line-2137"> return this.splitPolicy;</span> |
| <span class="source-line-no">2138</span><span id="line-2138"> }</span> |
| <span class="source-line-no">2139</span><span id="line-2139"></span> |
| <span class="source-line-no">2140</span><span id="line-2140"> /**</span> |
| <span class="source-line-no">2141</span><span id="line-2141"> * A split takes the config from the parent region & passes it to the daughter region's</span> |
| <span class="source-line-no">2142</span><span id="line-2142"> * constructor. If 'conf' was passed, you would end up using the HTD of the parent region in</span> |
| <span class="source-line-no">2143</span><span id="line-2143"> * addition to the new daughter HTD. Pass 'baseConf' to the daughter regions to avoid this tricky</span> |
| <span class="source-line-no">2144</span><span id="line-2144"> * dedupe problem.</span> |
| <span class="source-line-no">2145</span><span id="line-2145"> * @return Configuration object</span> |
| <span class="source-line-no">2146</span><span id="line-2146"> */</span> |
| <span class="source-line-no">2147</span><span id="line-2147"> Configuration getBaseConf() {</span> |
| <span class="source-line-no">2148</span><span id="line-2148"> return this.baseConf;</span> |
| <span class="source-line-no">2149</span><span id="line-2149"> }</span> |
| <span class="source-line-no">2150</span><span id="line-2150"></span> |
| <span class="source-line-no">2151</span><span id="line-2151"> /** Returns {@link FileSystem} being used by this region */</span> |
| <span class="source-line-no">2152</span><span id="line-2152"> public FileSystem getFilesystem() {</span> |
| <span class="source-line-no">2153</span><span id="line-2153"> return fs.getFileSystem();</span> |
| <span class="source-line-no">2154</span><span id="line-2154"> }</span> |
| <span class="source-line-no">2155</span><span id="line-2155"></span> |
| <span class="source-line-no">2156</span><span id="line-2156"> /** Returns the {@link HRegionFileSystem} used by this region */</span> |
| <span class="source-line-no">2157</span><span id="line-2157"> public HRegionFileSystem getRegionFileSystem() {</span> |
| <span class="source-line-no">2158</span><span id="line-2158"> return this.fs;</span> |
| <span class="source-line-no">2159</span><span id="line-2159"> }</span> |
| <span class="source-line-no">2160</span><span id="line-2160"></span> |
| <span class="source-line-no">2161</span><span id="line-2161"> /** Returns the WAL {@link HRegionFileSystem} used by this region */</span> |
| <span class="source-line-no">2162</span><span id="line-2162"> HRegionWALFileSystem getRegionWALFileSystem() throws IOException {</span> |
| <span class="source-line-no">2163</span><span id="line-2163"> return new HRegionWALFileSystem(conf, getWalFileSystem(),</span> |
| <span class="source-line-no">2164</span><span id="line-2164"> CommonFSUtils.getWALTableDir(conf, htableDescriptor.getTableName()), fs.getRegionInfo());</span> |
| <span class="source-line-no">2165</span><span id="line-2165"> }</span> |
| <span class="source-line-no">2166</span><span id="line-2166"></span> |
| <span class="source-line-no">2167</span><span id="line-2167"> /** Returns the WAL {@link FileSystem} being used by this region */</span> |
| <span class="source-line-no">2168</span><span id="line-2168"> FileSystem getWalFileSystem() throws IOException {</span> |
| <span class="source-line-no">2169</span><span id="line-2169"> if (walFS == null) {</span> |
| <span class="source-line-no">2170</span><span id="line-2170"> walFS = CommonFSUtils.getWALFileSystem(conf);</span> |
| <span class="source-line-no">2171</span><span id="line-2171"> }</span> |
| <span class="source-line-no">2172</span><span id="line-2172"> return walFS;</span> |
| <span class="source-line-no">2173</span><span id="line-2173"> }</span> |
| <span class="source-line-no">2174</span><span id="line-2174"></span> |
| <span class="source-line-no">2175</span><span id="line-2175"> /**</span> |
| <span class="source-line-no">2176</span><span id="line-2176"> * @return the Region directory under WALRootDirectory</span> |
| <span class="source-line-no">2177</span><span id="line-2177"> * @throws IOException if there is an error getting WALRootDir</span> |
| <span class="source-line-no">2178</span><span id="line-2178"> */</span> |
| <span class="source-line-no">2179</span><span id="line-2179"> public Path getWALRegionDir() throws IOException {</span> |
| <span class="source-line-no">2180</span><span id="line-2180"> if (regionWalDir == null) {</span> |
| <span class="source-line-no">2181</span><span id="line-2181"> regionWalDir = CommonFSUtils.getWALRegionDir(conf, getRegionInfo().getTable(),</span> |
| <span class="source-line-no">2182</span><span id="line-2182"> getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">2183</span><span id="line-2183"> }</span> |
| <span class="source-line-no">2184</span><span id="line-2184"> return regionWalDir;</span> |
| <span class="source-line-no">2185</span><span id="line-2185"> }</span> |
| <span class="source-line-no">2186</span><span id="line-2186"></span> |
| <span class="source-line-no">2187</span><span id="line-2187"> @Override</span> |
| <span class="source-line-no">2188</span><span id="line-2188"> public long getEarliestFlushTimeForAllStores() {</span> |
| <span class="source-line-no">2189</span><span id="line-2189"> return Collections.min(lastStoreFlushTimeMap.values());</span> |
| <span class="source-line-no">2190</span><span id="line-2190"> }</span> |
| <span class="source-line-no">2191</span><span id="line-2191"></span> |
| <span class="source-line-no">2192</span><span id="line-2192"> @Override</span> |
| <span class="source-line-no">2193</span><span id="line-2193"> public long getOldestHfileTs(boolean majorCompactionOnly) throws IOException {</span> |
| <span class="source-line-no">2194</span><span id="line-2194"> long result = Long.MAX_VALUE;</span> |
| <span class="source-line-no">2195</span><span id="line-2195"> for (HStore store : stores.values()) {</span> |
| <span class="source-line-no">2196</span><span id="line-2196"> Collection<HStoreFile> storeFiles = store.getStorefiles();</span> |
| <span class="source-line-no">2197</span><span id="line-2197"> if (storeFiles == null) {</span> |
| <span class="source-line-no">2198</span><span id="line-2198"> continue;</span> |
| <span class="source-line-no">2199</span><span id="line-2199"> }</span> |
| <span class="source-line-no">2200</span><span id="line-2200"> for (HStoreFile file : storeFiles) {</span> |
| <span class="source-line-no">2201</span><span id="line-2201"> StoreFileReader sfReader = file.getReader();</span> |
| <span class="source-line-no">2202</span><span id="line-2202"> if (sfReader == null) {</span> |
| <span class="source-line-no">2203</span><span id="line-2203"> continue;</span> |
| <span class="source-line-no">2204</span><span id="line-2204"> }</span> |
| <span class="source-line-no">2205</span><span id="line-2205"> HFile.Reader reader = sfReader.getHFileReader();</span> |
| <span class="source-line-no">2206</span><span id="line-2206"> if (reader == null) {</span> |
| <span class="source-line-no">2207</span><span id="line-2207"> continue;</span> |
| <span class="source-line-no">2208</span><span id="line-2208"> }</span> |
| <span class="source-line-no">2209</span><span id="line-2209"> if (majorCompactionOnly) {</span> |
| <span class="source-line-no">2210</span><span id="line-2210"> byte[] val = reader.getHFileInfo().get(MAJOR_COMPACTION_KEY);</span> |
| <span class="source-line-no">2211</span><span id="line-2211"> if (val == null || !Bytes.toBoolean(val)) {</span> |
| <span class="source-line-no">2212</span><span id="line-2212"> continue;</span> |
| <span class="source-line-no">2213</span><span id="line-2213"> }</span> |
| <span class="source-line-no">2214</span><span id="line-2214"> }</span> |
| <span class="source-line-no">2215</span><span id="line-2215"> result = Math.min(result, reader.getFileContext().getFileCreateTime());</span> |
| <span class="source-line-no">2216</span><span id="line-2216"> }</span> |
| <span class="source-line-no">2217</span><span id="line-2217"> }</span> |
| <span class="source-line-no">2218</span><span id="line-2218"> return result == Long.MAX_VALUE ? 0 : result;</span> |
| <span class="source-line-no">2219</span><span id="line-2219"> }</span> |
| <span class="source-line-no">2220</span><span id="line-2220"></span> |
| <span class="source-line-no">2221</span><span id="line-2221"> RegionLoad.Builder setCompleteSequenceId(RegionLoad.Builder regionLoadBldr) {</span> |
| <span class="source-line-no">2222</span><span id="line-2222"> long lastFlushOpSeqIdLocal = this.lastFlushOpSeqId;</span> |
| <span class="source-line-no">2223</span><span id="line-2223"> byte[] encodedRegionName = this.getRegionInfo().getEncodedNameAsBytes();</span> |
| <span class="source-line-no">2224</span><span id="line-2224"> regionLoadBldr.clearStoreCompleteSequenceId();</span> |
| <span class="source-line-no">2225</span><span id="line-2225"> for (byte[] familyName : this.stores.keySet()) {</span> |
| <span class="source-line-no">2226</span><span id="line-2226"> long earliest = this.wal.getEarliestMemStoreSeqNum(encodedRegionName, familyName);</span> |
| <span class="source-line-no">2227</span><span id="line-2227"> // Subtract - 1 to go earlier than the current oldest, unflushed edit in memstore; this will</span> |
| <span class="source-line-no">2228</span><span id="line-2228"> // give us a sequence id that is for sure flushed. We want edit replay to start after this</span> |
| <span class="source-line-no">2229</span><span id="line-2229"> // sequence id in this region. If NO_SEQNUM, use the regions maximum flush id.</span> |
| <span class="source-line-no">2230</span><span id="line-2230"> long csid = (earliest == HConstants.NO_SEQNUM) ? lastFlushOpSeqIdLocal : earliest - 1;</span> |
| <span class="source-line-no">2231</span><span id="line-2231"> regionLoadBldr.addStoreCompleteSequenceId(StoreSequenceId.newBuilder()</span> |
| <span class="source-line-no">2232</span><span id="line-2232"> .setFamilyName(UnsafeByteOperations.unsafeWrap(familyName)).setSequenceId(csid).build());</span> |
| <span class="source-line-no">2233</span><span id="line-2233"> }</span> |
| <span class="source-line-no">2234</span><span id="line-2234"> return regionLoadBldr.setCompleteSequenceId(getMaxFlushedSeqId());</span> |
| <span class="source-line-no">2235</span><span id="line-2235"> }</span> |
| <span class="source-line-no">2236</span><span id="line-2236"></span> |
| <span class="source-line-no">2237</span><span id="line-2237"> //////////////////////////////////////////////////////////////////////////////</span> |
| <span class="source-line-no">2238</span><span id="line-2238"> // HRegion maintenance.</span> |
| <span class="source-line-no">2239</span><span id="line-2239"> //</span> |
| <span class="source-line-no">2240</span><span id="line-2240"> // These methods are meant to be called periodically by the HRegionServer for</span> |
| <span class="source-line-no">2241</span><span id="line-2241"> // upkeep.</span> |
| <span class="source-line-no">2242</span><span id="line-2242"> //////////////////////////////////////////////////////////////////////////////</span> |
| <span class="source-line-no">2243</span><span id="line-2243"> /**</span> |
| <span class="source-line-no">2244</span><span id="line-2244"> * Do preparation for pending compaction.</span> |
| <span class="source-line-no">2245</span><span id="line-2245"> */</span> |
| <span class="source-line-no">2246</span><span id="line-2246"> protected void doRegionCompactionPrep() throws IOException {</span> |
| <span class="source-line-no">2247</span><span id="line-2247"> }</span> |
| <span class="source-line-no">2248</span><span id="line-2248"></span> |
| <span class="source-line-no">2249</span><span id="line-2249"> /**</span> |
| <span class="source-line-no">2250</span><span id="line-2250"> * Synchronously compact all stores in the region.</span> |
| <span class="source-line-no">2251</span><span id="line-2251"> * <p></span> |
| <span class="source-line-no">2252</span><span id="line-2252"> * This operation could block for a long time, so don't call it from a time-sensitive thread.</span> |
| <span class="source-line-no">2253</span><span id="line-2253"> * <p></span> |
| <span class="source-line-no">2254</span><span id="line-2254"> * Note that no locks are taken to prevent possible conflicts between compaction and splitting</span> |
| <span class="source-line-no">2255</span><span id="line-2255"> * activities. The regionserver does not normally compact and split in parallel. However by</span> |
| <span class="source-line-no">2256</span><span id="line-2256"> * calling this method you may introduce unexpected and unhandled concurrency. Don't do this</span> |
| <span class="source-line-no">2257</span><span id="line-2257"> * unless you know what you are doing.</span> |
| <span class="source-line-no">2258</span><span id="line-2258"> * @param majorCompaction True to force a major compaction regardless of thresholds</span> |
| <span class="source-line-no">2259</span><span id="line-2259"> */</span> |
| <span class="source-line-no">2260</span><span id="line-2260"> public void compact(boolean majorCompaction) throws IOException {</span> |
| <span class="source-line-no">2261</span><span id="line-2261"> if (majorCompaction) {</span> |
| <span class="source-line-no">2262</span><span id="line-2262"> stores.values().forEach(HStore::triggerMajorCompaction);</span> |
| <span class="source-line-no">2263</span><span id="line-2263"> }</span> |
| <span class="source-line-no">2264</span><span id="line-2264"> for (HStore s : stores.values()) {</span> |
| <span class="source-line-no">2265</span><span id="line-2265"> Optional<CompactionContext> compaction = s.requestCompaction();</span> |
| <span class="source-line-no">2266</span><span id="line-2266"> if (compaction.isPresent()) {</span> |
| <span class="source-line-no">2267</span><span id="line-2267"> ThroughputController controller = null;</span> |
| <span class="source-line-no">2268</span><span id="line-2268"> if (rsServices != null) {</span> |
| <span class="source-line-no">2269</span><span id="line-2269"> controller = CompactionThroughputControllerFactory.create(rsServices, conf);</span> |
| <span class="source-line-no">2270</span><span id="line-2270"> }</span> |
| <span class="source-line-no">2271</span><span id="line-2271"> if (controller == null) {</span> |
| <span class="source-line-no">2272</span><span id="line-2272"> controller = NoLimitThroughputController.INSTANCE;</span> |
| <span class="source-line-no">2273</span><span id="line-2273"> }</span> |
| <span class="source-line-no">2274</span><span id="line-2274"> compact(compaction.get(), s, controller, null);</span> |
| <span class="source-line-no">2275</span><span id="line-2275"> }</span> |
| <span class="source-line-no">2276</span><span id="line-2276"> }</span> |
| <span class="source-line-no">2277</span><span id="line-2277"> }</span> |
| <span class="source-line-no">2278</span><span id="line-2278"></span> |
| <span class="source-line-no">2279</span><span id="line-2279"> /**</span> |
| <span class="source-line-no">2280</span><span id="line-2280"> * This is a helper function that compact all the stores synchronously.</span> |
| <span class="source-line-no">2281</span><span id="line-2281"> * <p></span> |
| <span class="source-line-no">2282</span><span id="line-2282"> * It is used by utilities and testing</span> |
| <span class="source-line-no">2283</span><span id="line-2283"> */</span> |
| <span class="source-line-no">2284</span><span id="line-2284"> public void compactStores() throws IOException {</span> |
| <span class="source-line-no">2285</span><span id="line-2285"> for (HStore s : stores.values()) {</span> |
| <span class="source-line-no">2286</span><span id="line-2286"> Optional<CompactionContext> compaction = s.requestCompaction();</span> |
| <span class="source-line-no">2287</span><span id="line-2287"> if (compaction.isPresent()) {</span> |
| <span class="source-line-no">2288</span><span id="line-2288"> compact(compaction.get(), s, NoLimitThroughputController.INSTANCE, null);</span> |
| <span class="source-line-no">2289</span><span id="line-2289"> }</span> |
| <span class="source-line-no">2290</span><span id="line-2290"> }</span> |
| <span class="source-line-no">2291</span><span id="line-2291"> }</span> |
| <span class="source-line-no">2292</span><span id="line-2292"></span> |
| <span class="source-line-no">2293</span><span id="line-2293"> /**</span> |
| <span class="source-line-no">2294</span><span id="line-2294"> * This is a helper function that compact the given store.</span> |
| <span class="source-line-no">2295</span><span id="line-2295"> * <p></span> |
| <span class="source-line-no">2296</span><span id="line-2296"> * It is used by utilities and testing</span> |
| <span class="source-line-no">2297</span><span id="line-2297"> */</span> |
| <span class="source-line-no">2298</span><span id="line-2298"> void compactStore(byte[] family, ThroughputController throughputController) throws IOException {</span> |
| <span class="source-line-no">2299</span><span id="line-2299"> HStore s = getStore(family);</span> |
| <span class="source-line-no">2300</span><span id="line-2300"> Optional<CompactionContext> compaction = s.requestCompaction();</span> |
| <span class="source-line-no">2301</span><span id="line-2301"> if (compaction.isPresent()) {</span> |
| <span class="source-line-no">2302</span><span id="line-2302"> compact(compaction.get(), s, throughputController, null);</span> |
| <span class="source-line-no">2303</span><span id="line-2303"> }</span> |
| <span class="source-line-no">2304</span><span id="line-2304"> }</span> |
| <span class="source-line-no">2305</span><span id="line-2305"></span> |
| <span class="source-line-no">2306</span><span id="line-2306"> /**</span> |
| <span class="source-line-no">2307</span><span id="line-2307"> * Called by compaction thread and after region is opened to compact the HStores if necessary.</span> |
| <span class="source-line-no">2308</span><span id="line-2308"> * <p></span> |
| <span class="source-line-no">2309</span><span id="line-2309"> * This operation could block for a long time, so don't call it from a time-sensitive thread. Note</span> |
| <span class="source-line-no">2310</span><span id="line-2310"> * that no locking is necessary at this level because compaction only conflicts with a region</span> |
| <span class="source-line-no">2311</span><span id="line-2311"> * split, and that cannot happen because the region server does them sequentially and not in</span> |
| <span class="source-line-no">2312</span><span id="line-2312"> * parallel.</span> |
| <span class="source-line-no">2313</span><span id="line-2313"> * @param compaction Compaction details, obtained by requestCompaction()</span> |
| <span class="source-line-no">2314</span><span id="line-2314"> * @return whether the compaction completed</span> |
| <span class="source-line-no">2315</span><span id="line-2315"> */</span> |
| <span class="source-line-no">2316</span><span id="line-2316"> public boolean compact(CompactionContext compaction, HStore store,</span> |
| <span class="source-line-no">2317</span><span id="line-2317"> ThroughputController throughputController) throws IOException {</span> |
| <span class="source-line-no">2318</span><span id="line-2318"> return compact(compaction, store, throughputController, null);</span> |
| <span class="source-line-no">2319</span><span id="line-2319"> }</span> |
| <span class="source-line-no">2320</span><span id="line-2320"></span> |
| <span class="source-line-no">2321</span><span id="line-2321"> private boolean shouldForbidMajorCompaction() {</span> |
| <span class="source-line-no">2322</span><span id="line-2322"> if (rsServices != null && rsServices.getReplicationSourceService() != null) {</span> |
| <span class="source-line-no">2323</span><span id="line-2323"> return rsServices.getReplicationSourceService().getSyncReplicationPeerInfoProvider()</span> |
| <span class="source-line-no">2324</span><span id="line-2324"> .checkState(getRegionInfo().getTable(), ForbidMajorCompactionChecker.get());</span> |
| <span class="source-line-no">2325</span><span id="line-2325"> }</span> |
| <span class="source-line-no">2326</span><span id="line-2326"> return false;</span> |
| <span class="source-line-no">2327</span><span id="line-2327"> }</span> |
| <span class="source-line-no">2328</span><span id="line-2328"></span> |
| <span class="source-line-no">2329</span><span id="line-2329"> /**</span> |
| <span class="source-line-no">2330</span><span id="line-2330"> * We are trying to remove / relax the region read lock for compaction. Let's see what are the</span> |
| <span class="source-line-no">2331</span><span id="line-2331"> * potential race conditions among the operations (user scan, region split, region close and</span> |
| <span class="source-line-no">2332</span><span id="line-2332"> * region bulk load). user scan ---> region read lock region split --> region close first --></span> |
| <span class="source-line-no">2333</span><span id="line-2333"> * region write lock region close --> region write lock region bulk load --> region write lock</span> |
| <span class="source-line-no">2334</span><span id="line-2334"> * read lock is compatible with read lock. ---> no problem with user scan/read region bulk load</span> |
| <span class="source-line-no">2335</span><span id="line-2335"> * does not cause problem for compaction (no consistency problem, store lock will help the store</span> |
| <span class="source-line-no">2336</span><span id="line-2336"> * file accounting). They can run almost concurrently at the region level. The only remaining race</span> |
| <span class="source-line-no">2337</span><span id="line-2337"> * condition is between the region close and compaction. So we will evaluate, below, how region</span> |
| <span class="source-line-no">2338</span><span id="line-2338"> * close intervenes with compaction if compaction does not acquire region read lock. Here are the</span> |
| <span class="source-line-no">2339</span><span id="line-2339"> * steps for compaction: 1. obtain list of StoreFile's 2. create StoreFileScanner's based on list</span> |
| <span class="source-line-no">2340</span><span id="line-2340"> * from #1 3. perform compaction and save resulting files under tmp dir 4. swap in compacted files</span> |
| <span class="source-line-no">2341</span><span id="line-2341"> * #1 is guarded by store lock. This patch does not change this --> no worse or better For #2, we</span> |
| <span class="source-line-no">2342</span><span id="line-2342"> * obtain smallest read point (for region) across all the Scanners (for both default compactor and</span> |
| <span class="source-line-no">2343</span><span id="line-2343"> * stripe compactor). The read points are for user scans. Region keeps the read points for all</span> |
| <span class="source-line-no">2344</span><span id="line-2344"> * currently open user scanners. Compaction needs to know the smallest read point so that during</span> |
| <span class="source-line-no">2345</span><span id="line-2345"> * re-write of the hfiles, it can remove the mvcc points for the cells if their mvccs are older</span> |
| <span class="source-line-no">2346</span><span id="line-2346"> * than the smallest since they are not needed anymore. This will not conflict with compaction.</span> |
| <span class="source-line-no">2347</span><span id="line-2347"> * For #3, it can be performed in parallel to other operations. For #4 bulk load and compaction</span> |
| <span class="source-line-no">2348</span><span id="line-2348"> * don't conflict with each other on the region level (for multi-family atomicy). Region close and</span> |
| <span class="source-line-no">2349</span><span id="line-2349"> * compaction are guarded pretty well by the 'writestate'. In HRegion#doClose(), we have :</span> |
| <span class="source-line-no">2350</span><span id="line-2350"> * synchronized (writestate) { // Disable compacting and flushing by background threads for this</span> |
| <span class="source-line-no">2351</span><span id="line-2351"> * // region. canFlush = !writestate.readOnly; writestate.writesEnabled = false;</span> |
| <span class="source-line-no">2352</span><span id="line-2352"> * LOG.debug("Closing " + this + ": disabling compactions & flushes");</span> |
| <span class="source-line-no">2353</span><span id="line-2353"> * waitForFlushesAndCompactions(); } waitForFlushesAndCompactions() would wait for</span> |
| <span class="source-line-no">2354</span><span id="line-2354"> * writestate.compacting to come down to 0. and in HRegion.compact() try { synchronized</span> |
| <span class="source-line-no">2355</span><span id="line-2355"> * (writestate) { if (writestate.writesEnabled) { wasStateSet = true; ++writestate.compacting; }</span> |
| <span class="source-line-no">2356</span><span id="line-2356"> * else { String msg = "NOT compacting region " + this + ". Writes disabled."; LOG.info(msg);</span> |
| <span class="source-line-no">2357</span><span id="line-2357"> * status.abort(msg); return false; } } Also in compactor.performCompaction(): check periodically</span> |
| <span class="source-line-no">2358</span><span id="line-2358"> * to see if a system stop is requested if (closeChecker != null &&</span> |
| <span class="source-line-no">2359</span><span id="line-2359"> * closeChecker.isTimeLimit(store, now)) { progress.cancel(); return false; } if (closeChecker !=</span> |
| <span class="source-line-no">2360</span><span id="line-2360"> * null && closeChecker.isSizeLimit(store, len)) { progress.cancel(); return false; }</span> |
| <span class="source-line-no">2361</span><span id="line-2361"> */</span> |
| <span class="source-line-no">2362</span><span id="line-2362"> public boolean compact(CompactionContext compaction, HStore store,</span> |
| <span class="source-line-no">2363</span><span id="line-2363"> ThroughputController throughputController, User user) throws IOException {</span> |
| <span class="source-line-no">2364</span><span id="line-2364"> assert compaction != null && compaction.hasSelection();</span> |
| <span class="source-line-no">2365</span><span id="line-2365"> assert !compaction.getRequest().getFiles().isEmpty();</span> |
| <span class="source-line-no">2366</span><span id="line-2366"> if (this.closing.get() || this.closed.get()) {</span> |
| <span class="source-line-no">2367</span><span id="line-2367"> LOG.debug("Skipping compaction on " + this + " because closing/closed");</span> |
| <span class="source-line-no">2368</span><span id="line-2368"> store.cancelRequestedCompaction(compaction);</span> |
| <span class="source-line-no">2369</span><span id="line-2369"> return false;</span> |
| <span class="source-line-no">2370</span><span id="line-2370"> }</span> |
| <span class="source-line-no">2371</span><span id="line-2371"></span> |
| <span class="source-line-no">2372</span><span id="line-2372"> if (compaction.getRequest().isAllFiles() && shouldForbidMajorCompaction()) {</span> |
| <span class="source-line-no">2373</span><span id="line-2373"> LOG.warn("Skipping major compaction on " + this</span> |
| <span class="source-line-no">2374</span><span id="line-2374"> + " because this cluster is transiting sync replication state"</span> |
| <span class="source-line-no">2375</span><span id="line-2375"> + " from STANDBY to DOWNGRADE_ACTIVE");</span> |
| <span class="source-line-no">2376</span><span id="line-2376"> store.cancelRequestedCompaction(compaction);</span> |
| <span class="source-line-no">2377</span><span id="line-2377"> return false;</span> |
| <span class="source-line-no">2378</span><span id="line-2378"> }</span> |
| <span class="source-line-no">2379</span><span id="line-2379"></span> |
| <span class="source-line-no">2380</span><span id="line-2380"> MonitoredTask status = null;</span> |
| <span class="source-line-no">2381</span><span id="line-2381"> boolean requestNeedsCancellation = true;</span> |
| <span class="source-line-no">2382</span><span id="line-2382"> try {</span> |
| <span class="source-line-no">2383</span><span id="line-2383"> byte[] cf = Bytes.toBytes(store.getColumnFamilyName());</span> |
| <span class="source-line-no">2384</span><span id="line-2384"> if (stores.get(cf) != store) {</span> |
| <span class="source-line-no">2385</span><span id="line-2385"> LOG.warn("Store " + store.getColumnFamilyName() + " on region " + this</span> |
| <span class="source-line-no">2386</span><span id="line-2386"> + " has been re-instantiated, cancel this compaction request. "</span> |
| <span class="source-line-no">2387</span><span id="line-2387"> + " It may be caused by the roll back of split transaction");</span> |
| <span class="source-line-no">2388</span><span id="line-2388"> return false;</span> |
| <span class="source-line-no">2389</span><span id="line-2389"> }</span> |
| <span class="source-line-no">2390</span><span id="line-2390"></span> |
| <span class="source-line-no">2391</span><span id="line-2391"> status = TaskMonitor.get().createStatus("Compacting " + store + " in " + this);</span> |
| <span class="source-line-no">2392</span><span id="line-2392"> if (this.closed.get()) {</span> |
| <span class="source-line-no">2393</span><span id="line-2393"> String msg = "Skipping compaction on " + this + " because closed";</span> |
| <span class="source-line-no">2394</span><span id="line-2394"> LOG.debug(msg);</span> |
| <span class="source-line-no">2395</span><span id="line-2395"> status.abort(msg);</span> |
| <span class="source-line-no">2396</span><span id="line-2396"> return false;</span> |
| <span class="source-line-no">2397</span><span id="line-2397"> }</span> |
| <span class="source-line-no">2398</span><span id="line-2398"> boolean wasStateSet = false;</span> |
| <span class="source-line-no">2399</span><span id="line-2399"> try {</span> |
| <span class="source-line-no">2400</span><span id="line-2400"> synchronized (writestate) {</span> |
| <span class="source-line-no">2401</span><span id="line-2401"> if (writestate.writesEnabled) {</span> |
| <span class="source-line-no">2402</span><span id="line-2402"> wasStateSet = true;</span> |
| <span class="source-line-no">2403</span><span id="line-2403"> writestate.compacting.incrementAndGet();</span> |
| <span class="source-line-no">2404</span><span id="line-2404"> } else {</span> |
| <span class="source-line-no">2405</span><span id="line-2405"> String msg = "NOT compacting region " + this + ". Writes disabled.";</span> |
| <span class="source-line-no">2406</span><span id="line-2406"> LOG.info(msg);</span> |
| <span class="source-line-no">2407</span><span id="line-2407"> status.abort(msg);</span> |
| <span class="source-line-no">2408</span><span id="line-2408"> return false;</span> |
| <span class="source-line-no">2409</span><span id="line-2409"> }</span> |
| <span class="source-line-no">2410</span><span id="line-2410"> }</span> |
| <span class="source-line-no">2411</span><span id="line-2411"> LOG.info("Starting compaction of {} in {}{}", store, this,</span> |
| <span class="source-line-no">2412</span><span id="line-2412"> (compaction.getRequest().isOffPeak() ? " as an off-peak compaction" : ""));</span> |
| <span class="source-line-no">2413</span><span id="line-2413"> doRegionCompactionPrep();</span> |
| <span class="source-line-no">2414</span><span id="line-2414"> try {</span> |
| <span class="source-line-no">2415</span><span id="line-2415"> status.setStatus("Compacting store " + store);</span> |
| <span class="source-line-no">2416</span><span id="line-2416"> // We no longer need to cancel the request on the way out of this</span> |
| <span class="source-line-no">2417</span><span id="line-2417"> // method because Store#compact will clean up unconditionally</span> |
| <span class="source-line-no">2418</span><span id="line-2418"> requestNeedsCancellation = false;</span> |
| <span class="source-line-no">2419</span><span id="line-2419"> store.compact(compaction, throughputController, user);</span> |
| <span class="source-line-no">2420</span><span id="line-2420"> } catch (InterruptedIOException iioe) {</span> |
| <span class="source-line-no">2421</span><span id="line-2421"> String msg = "region " + this + " compaction interrupted";</span> |
| <span class="source-line-no">2422</span><span id="line-2422"> LOG.info(msg, iioe);</span> |
| <span class="source-line-no">2423</span><span id="line-2423"> status.abort(msg);</span> |
| <span class="source-line-no">2424</span><span id="line-2424"> return false;</span> |
| <span class="source-line-no">2425</span><span id="line-2425"> }</span> |
| <span class="source-line-no">2426</span><span id="line-2426"> } finally {</span> |
| <span class="source-line-no">2427</span><span id="line-2427"> if (wasStateSet) {</span> |
| <span class="source-line-no">2428</span><span id="line-2428"> synchronized (writestate) {</span> |
| <span class="source-line-no">2429</span><span id="line-2429"> writestate.compacting.decrementAndGet();</span> |
| <span class="source-line-no">2430</span><span id="line-2430"> if (writestate.compacting.get() <= 0) {</span> |
| <span class="source-line-no">2431</span><span id="line-2431"> writestate.notifyAll();</span> |
| <span class="source-line-no">2432</span><span id="line-2432"> }</span> |
| <span class="source-line-no">2433</span><span id="line-2433"> }</span> |
| <span class="source-line-no">2434</span><span id="line-2434"> }</span> |
| <span class="source-line-no">2435</span><span id="line-2435"> }</span> |
| <span class="source-line-no">2436</span><span id="line-2436"> status.markComplete("Compaction complete");</span> |
| <span class="source-line-no">2437</span><span id="line-2437"> return true;</span> |
| <span class="source-line-no">2438</span><span id="line-2438"> } finally {</span> |
| <span class="source-line-no">2439</span><span id="line-2439"> if (requestNeedsCancellation) store.cancelRequestedCompaction(compaction);</span> |
| <span class="source-line-no">2440</span><span id="line-2440"> if (status != null) {</span> |
| <span class="source-line-no">2441</span><span id="line-2441"> LOG.debug("Compaction status journal for {}:\n{}", this.getRegionInfo().getEncodedName(),</span> |
| <span class="source-line-no">2442</span><span id="line-2442"> status.prettyPrintJournal());</span> |
| <span class="source-line-no">2443</span><span id="line-2443"> status.cleanup();</span> |
| <span class="source-line-no">2444</span><span id="line-2444"> }</span> |
| <span class="source-line-no">2445</span><span id="line-2445"> }</span> |
| <span class="source-line-no">2446</span><span id="line-2446"> }</span> |
| <span class="source-line-no">2447</span><span id="line-2447"></span> |
| <span class="source-line-no">2448</span><span id="line-2448"> /**</span> |
| <span class="source-line-no">2449</span><span id="line-2449"> * Flush the cache.</span> |
| <span class="source-line-no">2450</span><span id="line-2450"> * <p></span> |
| <span class="source-line-no">2451</span><span id="line-2451"> * When this method is called the cache will be flushed unless:</span> |
| <span class="source-line-no">2452</span><span id="line-2452"> * <ol></span> |
| <span class="source-line-no">2453</span><span id="line-2453"> * <li>the cache is empty</li></span> |
| <span class="source-line-no">2454</span><span id="line-2454"> * <li>the region is closed.</li></span> |
| <span class="source-line-no">2455</span><span id="line-2455"> * <li>a flush is already in progress</li></span> |
| <span class="source-line-no">2456</span><span id="line-2456"> * <li>writes are disabled</li></span> |
| <span class="source-line-no">2457</span><span id="line-2457"> * </ol></span> |
| <span class="source-line-no">2458</span><span id="line-2458"> * <p></span> |
| <span class="source-line-no">2459</span><span id="line-2459"> * This method may block for some time, so it should not be called from a time-sensitive thread.</span> |
| <span class="source-line-no">2460</span><span id="line-2460"> * @param flushAllStores whether we want to force a flush of all stores</span> |
| <span class="source-line-no">2461</span><span id="line-2461"> * @return FlushResult indicating whether the flush was successful or not and if the region needs</span> |
| <span class="source-line-no">2462</span><span id="line-2462"> * compacting</span> |
| <span class="source-line-no">2463</span><span id="line-2463"> * @throws IOException general io exceptions because a snapshot was not properly persisted.</span> |
| <span class="source-line-no">2464</span><span id="line-2464"> */</span> |
| <span class="source-line-no">2465</span><span id="line-2465"> // TODO HBASE-18905. We might have to expose a requestFlush API for CPs</span> |
| <span class="source-line-no">2466</span><span id="line-2466"> public FlushResult flush(boolean flushAllStores) throws IOException {</span> |
| <span class="source-line-no">2467</span><span id="line-2467"> return flushcache(flushAllStores, false, FlushLifeCycleTracker.DUMMY);</span> |
| <span class="source-line-no">2468</span><span id="line-2468"> }</span> |
| <span class="source-line-no">2469</span><span id="line-2469"></span> |
| <span class="source-line-no">2470</span><span id="line-2470"> public interface FlushResult {</span> |
| <span class="source-line-no">2471</span><span id="line-2471"> enum Result {</span> |
| <span class="source-line-no">2472</span><span id="line-2472"> FLUSHED_NO_COMPACTION_NEEDED,</span> |
| <span class="source-line-no">2473</span><span id="line-2473"> FLUSHED_COMPACTION_NEEDED,</span> |
| <span class="source-line-no">2474</span><span id="line-2474"> // Special case where a flush didn't run because there's nothing in the memstores. Used when</span> |
| <span class="source-line-no">2475</span><span id="line-2475"> // bulk loading to know when we can still load even if a flush didn't happen.</span> |
| <span class="source-line-no">2476</span><span id="line-2476"> CANNOT_FLUSH_MEMSTORE_EMPTY,</span> |
| <span class="source-line-no">2477</span><span id="line-2477"> CANNOT_FLUSH</span> |
| <span class="source-line-no">2478</span><span id="line-2478"> }</span> |
| <span class="source-line-no">2479</span><span id="line-2479"></span> |
| <span class="source-line-no">2480</span><span id="line-2480"> /** Returns the detailed result code */</span> |
| <span class="source-line-no">2481</span><span id="line-2481"> Result getResult();</span> |
| <span class="source-line-no">2482</span><span id="line-2482"></span> |
| <span class="source-line-no">2483</span><span id="line-2483"> /** Returns true if the memstores were flushed, else false */</span> |
| <span class="source-line-no">2484</span><span id="line-2484"> boolean isFlushSucceeded();</span> |
| <span class="source-line-no">2485</span><span id="line-2485"></span> |
| <span class="source-line-no">2486</span><span id="line-2486"> /** Returns True if the flush requested a compaction, else false */</span> |
| <span class="source-line-no">2487</span><span id="line-2487"> boolean isCompactionNeeded();</span> |
| <span class="source-line-no">2488</span><span id="line-2488"> }</span> |
| <span class="source-line-no">2489</span><span id="line-2489"></span> |
| <span class="source-line-no">2490</span><span id="line-2490"> public FlushResultImpl flushcache(boolean flushAllStores, boolean writeFlushRequestWalMarker,</span> |
| <span class="source-line-no">2491</span><span id="line-2491"> FlushLifeCycleTracker tracker) throws IOException {</span> |
| <span class="source-line-no">2492</span><span id="line-2492"> List<byte[]> families = null;</span> |
| <span class="source-line-no">2493</span><span id="line-2493"> if (flushAllStores) {</span> |
| <span class="source-line-no">2494</span><span id="line-2494"> families = new ArrayList<>();</span> |
| <span class="source-line-no">2495</span><span id="line-2495"> families.addAll(this.getTableDescriptor().getColumnFamilyNames());</span> |
| <span class="source-line-no">2496</span><span id="line-2496"> }</span> |
| <span class="source-line-no">2497</span><span id="line-2497"> return this.flushcache(families, writeFlushRequestWalMarker, tracker);</span> |
| <span class="source-line-no">2498</span><span id="line-2498"> }</span> |
| <span class="source-line-no">2499</span><span id="line-2499"></span> |
| <span class="source-line-no">2500</span><span id="line-2500"> /**</span> |
| <span class="source-line-no">2501</span><span id="line-2501"> * Flush the cache. When this method is called the cache will be flushed unless:</span> |
| <span class="source-line-no">2502</span><span id="line-2502"> * <ol></span> |
| <span class="source-line-no">2503</span><span id="line-2503"> * <li>the cache is empty</li></span> |
| <span class="source-line-no">2504</span><span id="line-2504"> * <li>the region is closed.</li></span> |
| <span class="source-line-no">2505</span><span id="line-2505"> * <li>a flush is already in progress</li></span> |
| <span class="source-line-no">2506</span><span id="line-2506"> * <li>writes are disabled</li></span> |
| <span class="source-line-no">2507</span><span id="line-2507"> * </ol></span> |
| <span class="source-line-no">2508</span><span id="line-2508"> * <p></span> |
| <span class="source-line-no">2509</span><span id="line-2509"> * This method may block for some time, so it should not be called from a time-sensitive thread.</span> |
| <span class="source-line-no">2510</span><span id="line-2510"> * @param families stores of region to flush.</span> |
| <span class="source-line-no">2511</span><span id="line-2511"> * @param writeFlushRequestWalMarker whether to write the flush request marker to WAL</span> |
| <span class="source-line-no">2512</span><span id="line-2512"> * @param tracker used to track the life cycle of this flush</span> |
| <span class="source-line-no">2513</span><span id="line-2513"> * @return whether the flush is success and whether the region needs compacting</span> |
| <span class="source-line-no">2514</span><span id="line-2514"> * @throws IOException general io exceptions</span> |
| <span class="source-line-no">2515</span><span id="line-2515"> * @throws DroppedSnapshotException Thrown when replay of wal is required because a Snapshot was</span> |
| <span class="source-line-no">2516</span><span id="line-2516"> * not properly persisted. The region is put in closing mode, and</span> |
| <span class="source-line-no">2517</span><span id="line-2517"> * the caller MUST abort after this.</span> |
| <span class="source-line-no">2518</span><span id="line-2518"> */</span> |
| <span class="source-line-no">2519</span><span id="line-2519"> public FlushResultImpl flushcache(List<byte[]> families, boolean writeFlushRequestWalMarker,</span> |
| <span class="source-line-no">2520</span><span id="line-2520"> FlushLifeCycleTracker tracker) throws IOException {</span> |
| <span class="source-line-no">2521</span><span id="line-2521"> // fail-fast instead of waiting on the lock</span> |
| <span class="source-line-no">2522</span><span id="line-2522"> if (this.closing.get()) {</span> |
| <span class="source-line-no">2523</span><span id="line-2523"> String msg = "Skipping flush on " + this + " because closing";</span> |
| <span class="source-line-no">2524</span><span id="line-2524"> LOG.debug(msg);</span> |
| <span class="source-line-no">2525</span><span id="line-2525"> return new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, msg, false);</span> |
| <span class="source-line-no">2526</span><span id="line-2526"> }</span> |
| <span class="source-line-no">2527</span><span id="line-2527"> MonitoredTask status = TaskMonitor.get().createStatus("Flushing " + this);</span> |
| <span class="source-line-no">2528</span><span id="line-2528"> status.setStatus("Acquiring readlock on region");</span> |
| <span class="source-line-no">2529</span><span id="line-2529"> // block waiting for the lock for flushing cache</span> |
| <span class="source-line-no">2530</span><span id="line-2530"> lock.readLock().lock();</span> |
| <span class="source-line-no">2531</span><span id="line-2531"> boolean flushed = true;</span> |
| <span class="source-line-no">2532</span><span id="line-2532"> try {</span> |
| <span class="source-line-no">2533</span><span id="line-2533"> if (this.closed.get()) {</span> |
| <span class="source-line-no">2534</span><span id="line-2534"> String msg = "Skipping flush on " + this + " because closed";</span> |
| <span class="source-line-no">2535</span><span id="line-2535"> LOG.debug(msg);</span> |
| <span class="source-line-no">2536</span><span id="line-2536"> status.abort(msg);</span> |
| <span class="source-line-no">2537</span><span id="line-2537"> flushed = false;</span> |
| <span class="source-line-no">2538</span><span id="line-2538"> return new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, msg, false);</span> |
| <span class="source-line-no">2539</span><span id="line-2539"> }</span> |
| <span class="source-line-no">2540</span><span id="line-2540"> if (coprocessorHost != null) {</span> |
| <span class="source-line-no">2541</span><span id="line-2541"> status.setStatus("Running coprocessor pre-flush hooks");</span> |
| <span class="source-line-no">2542</span><span id="line-2542"> coprocessorHost.preFlush(tracker);</span> |
| <span class="source-line-no">2543</span><span id="line-2543"> }</span> |
| <span class="source-line-no">2544</span><span id="line-2544"> // TODO: this should be managed within memstore with the snapshot, updated only after flush</span> |
| <span class="source-line-no">2545</span><span id="line-2545"> // successful</span> |
| <span class="source-line-no">2546</span><span id="line-2546"> if (numMutationsWithoutWAL.sum() > 0) {</span> |
| <span class="source-line-no">2547</span><span id="line-2547"> numMutationsWithoutWAL.reset();</span> |
| <span class="source-line-no">2548</span><span id="line-2548"> dataInMemoryWithoutWAL.reset();</span> |
| <span class="source-line-no">2549</span><span id="line-2549"> }</span> |
| <span class="source-line-no">2550</span><span id="line-2550"> synchronized (writestate) {</span> |
| <span class="source-line-no">2551</span><span id="line-2551"> if (!writestate.flushing && writestate.writesEnabled) {</span> |
| <span class="source-line-no">2552</span><span id="line-2552"> this.writestate.flushing = true;</span> |
| <span class="source-line-no">2553</span><span id="line-2553"> } else {</span> |
| <span class="source-line-no">2554</span><span id="line-2554"> String msg = "NOT flushing " + this + " as "</span> |
| <span class="source-line-no">2555</span><span id="line-2555"> + (writestate.flushing ? "already flushing" : "writes are not enabled");</span> |
| <span class="source-line-no">2556</span><span id="line-2556"> LOG.debug(msg);</span> |
| <span class="source-line-no">2557</span><span id="line-2557"> status.abort(msg);</span> |
| <span class="source-line-no">2558</span><span id="line-2558"> flushed = false;</span> |
| <span class="source-line-no">2559</span><span id="line-2559"> return new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, msg, false);</span> |
| <span class="source-line-no">2560</span><span id="line-2560"> }</span> |
| <span class="source-line-no">2561</span><span id="line-2561"> }</span> |
| <span class="source-line-no">2562</span><span id="line-2562"></span> |
| <span class="source-line-no">2563</span><span id="line-2563"> try {</span> |
| <span class="source-line-no">2564</span><span id="line-2564"> // The reason that we do not always use flushPolicy is, when the flush is</span> |
| <span class="source-line-no">2565</span><span id="line-2565"> // caused by logRoller, we should select stores which must be flushed</span> |
| <span class="source-line-no">2566</span><span id="line-2566"> // rather than could be flushed.</span> |
| <span class="source-line-no">2567</span><span id="line-2567"> Collection<HStore> specificStoresToFlush = null;</span> |
| <span class="source-line-no">2568</span><span id="line-2568"> if (families != null) {</span> |
| <span class="source-line-no">2569</span><span id="line-2569"> specificStoresToFlush = getSpecificStores(families);</span> |
| <span class="source-line-no">2570</span><span id="line-2570"> } else {</span> |
| <span class="source-line-no">2571</span><span id="line-2571"> specificStoresToFlush = flushPolicy.selectStoresToFlush();</span> |
| <span class="source-line-no">2572</span><span id="line-2572"> }</span> |
| <span class="source-line-no">2573</span><span id="line-2573"> FlushResultImpl fs =</span> |
| <span class="source-line-no">2574</span><span id="line-2574"> internalFlushcache(specificStoresToFlush, status, writeFlushRequestWalMarker, tracker);</span> |
| <span class="source-line-no">2575</span><span id="line-2575"></span> |
| <span class="source-line-no">2576</span><span id="line-2576"> if (coprocessorHost != null) {</span> |
| <span class="source-line-no">2577</span><span id="line-2577"> status.setStatus("Running post-flush coprocessor hooks");</span> |
| <span class="source-line-no">2578</span><span id="line-2578"> coprocessorHost.postFlush(tracker);</span> |
| <span class="source-line-no">2579</span><span id="line-2579"> }</span> |
| <span class="source-line-no">2580</span><span id="line-2580"></span> |
| <span class="source-line-no">2581</span><span id="line-2581"> if (fs.isFlushSucceeded()) {</span> |
| <span class="source-line-no">2582</span><span id="line-2582"> flushesQueued.reset();</span> |
| <span class="source-line-no">2583</span><span id="line-2583"> }</span> |
| <span class="source-line-no">2584</span><span id="line-2584"></span> |
| <span class="source-line-no">2585</span><span id="line-2585"> status.markComplete("Flush successful " + fs.toString());</span> |
| <span class="source-line-no">2586</span><span id="line-2586"> return fs;</span> |
| <span class="source-line-no">2587</span><span id="line-2587"> } finally {</span> |
| <span class="source-line-no">2588</span><span id="line-2588"> synchronized (writestate) {</span> |
| <span class="source-line-no">2589</span><span id="line-2589"> writestate.flushing = false;</span> |
| <span class="source-line-no">2590</span><span id="line-2590"> this.writestate.flushRequested = false;</span> |
| <span class="source-line-no">2591</span><span id="line-2591"> writestate.notifyAll();</span> |
| <span class="source-line-no">2592</span><span id="line-2592"> }</span> |
| <span class="source-line-no">2593</span><span id="line-2593"> }</span> |
| <span class="source-line-no">2594</span><span id="line-2594"> } finally {</span> |
| <span class="source-line-no">2595</span><span id="line-2595"> lock.readLock().unlock();</span> |
| <span class="source-line-no">2596</span><span id="line-2596"> if (flushed) {</span> |
| <span class="source-line-no">2597</span><span id="line-2597"> // Don't log this journal stuff if no flush -- confusing.</span> |
| <span class="source-line-no">2598</span><span id="line-2598"> LOG.debug("Flush status journal for {}:\n{}", this.getRegionInfo().getEncodedName(),</span> |
| <span class="source-line-no">2599</span><span id="line-2599"> status.prettyPrintJournal());</span> |
| <span class="source-line-no">2600</span><span id="line-2600"> }</span> |
| <span class="source-line-no">2601</span><span id="line-2601"> status.cleanup();</span> |
| <span class="source-line-no">2602</span><span id="line-2602"> }</span> |
| <span class="source-line-no">2603</span><span id="line-2603"> }</span> |
| <span class="source-line-no">2604</span><span id="line-2604"></span> |
| <span class="source-line-no">2605</span><span id="line-2605"> /**</span> |
| <span class="source-line-no">2606</span><span id="line-2606"> * get stores which matches the specified families</span> |
| <span class="source-line-no">2607</span><span id="line-2607"> * @return the stores need to be flushed.</span> |
| <span class="source-line-no">2608</span><span id="line-2608"> */</span> |
| <span class="source-line-no">2609</span><span id="line-2609"> private Collection<HStore> getSpecificStores(List<byte[]> families) {</span> |
| <span class="source-line-no">2610</span><span id="line-2610"> Collection<HStore> specificStoresToFlush = new ArrayList<>();</span> |
| <span class="source-line-no">2611</span><span id="line-2611"> for (byte[] family : families) {</span> |
| <span class="source-line-no">2612</span><span id="line-2612"> specificStoresToFlush.add(stores.get(family));</span> |
| <span class="source-line-no">2613</span><span id="line-2613"> }</span> |
| <span class="source-line-no">2614</span><span id="line-2614"> return specificStoresToFlush;</span> |
| <span class="source-line-no">2615</span><span id="line-2615"> }</span> |
| <span class="source-line-no">2616</span><span id="line-2616"></span> |
| <span class="source-line-no">2617</span><span id="line-2617"> /**</span> |
| <span class="source-line-no">2618</span><span id="line-2618"> * Should the store be flushed because it is old enough.</span> |
| <span class="source-line-no">2619</span><span id="line-2619"> * <p></span> |
| <span class="source-line-no">2620</span><span id="line-2620"> * Every FlushPolicy should call this to determine whether a store is old enough to flush (except</span> |
| <span class="source-line-no">2621</span><span id="line-2621"> * that you always flush all stores). Otherwise the method will always returns true which will</span> |
| <span class="source-line-no">2622</span><span id="line-2622"> * make a lot of flush requests.</span> |
| <span class="source-line-no">2623</span><span id="line-2623"> */</span> |
| <span class="source-line-no">2624</span><span id="line-2624"> boolean shouldFlushStore(HStore store) {</span> |
| <span class="source-line-no">2625</span><span id="line-2625"> long earliest = this.wal.getEarliestMemStoreSeqNum(getRegionInfo().getEncodedNameAsBytes(),</span> |
| <span class="source-line-no">2626</span><span id="line-2626"> store.getColumnFamilyDescriptor().getName()) - 1;</span> |
| <span class="source-line-no">2627</span><span id="line-2627"> if (earliest > 0 && earliest + flushPerChanges < mvcc.getReadPoint()) {</span> |
| <span class="source-line-no">2628</span><span id="line-2628"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">2629</span><span id="line-2629"> LOG.debug("Flush column family " + store.getColumnFamilyName() + " of "</span> |
| <span class="source-line-no">2630</span><span id="line-2630"> + getRegionInfo().getEncodedName() + " because unflushed sequenceid=" + earliest</span> |
| <span class="source-line-no">2631</span><span id="line-2631"> + " is > " + this.flushPerChanges + " from current=" + mvcc.getReadPoint());</span> |
| <span class="source-line-no">2632</span><span id="line-2632"> }</span> |
| <span class="source-line-no">2633</span><span id="line-2633"> return true;</span> |
| <span class="source-line-no">2634</span><span id="line-2634"> }</span> |
| <span class="source-line-no">2635</span><span id="line-2635"> if (this.flushCheckInterval <= 0) {</span> |
| <span class="source-line-no">2636</span><span id="line-2636"> return false;</span> |
| <span class="source-line-no">2637</span><span id="line-2637"> }</span> |
| <span class="source-line-no">2638</span><span id="line-2638"> long now = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">2639</span><span id="line-2639"> if (store.timeOfOldestEdit() < now - this.flushCheckInterval) {</span> |
| <span class="source-line-no">2640</span><span id="line-2640"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">2641</span><span id="line-2641"> LOG.debug("Flush column family: " + store.getColumnFamilyName() + " of "</span> |
| <span class="source-line-no">2642</span><span id="line-2642"> + getRegionInfo().getEncodedName() + " because time of oldest edit="</span> |
| <span class="source-line-no">2643</span><span id="line-2643"> + store.timeOfOldestEdit() + " is > " + this.flushCheckInterval + " from now =" + now);</span> |
| <span class="source-line-no">2644</span><span id="line-2644"> }</span> |
| <span class="source-line-no">2645</span><span id="line-2645"> return true;</span> |
| <span class="source-line-no">2646</span><span id="line-2646"> }</span> |
| <span class="source-line-no">2647</span><span id="line-2647"> return false;</span> |
| <span class="source-line-no">2648</span><span id="line-2648"> }</span> |
| <span class="source-line-no">2649</span><span id="line-2649"></span> |
| <span class="source-line-no">2650</span><span id="line-2650"> /**</span> |
| <span class="source-line-no">2651</span><span id="line-2651"> * Should the memstore be flushed now</span> |
| <span class="source-line-no">2652</span><span id="line-2652"> */</span> |
| <span class="source-line-no">2653</span><span id="line-2653"> boolean shouldFlush(final StringBuilder whyFlush) {</span> |
| <span class="source-line-no">2654</span><span id="line-2654"> whyFlush.setLength(0);</span> |
| <span class="source-line-no">2655</span><span id="line-2655"> // This is a rough measure.</span> |
| <span class="source-line-no">2656</span><span id="line-2656"> if (</span> |
| <span class="source-line-no">2657</span><span id="line-2657"> this.maxFlushedSeqId > 0</span> |
| <span class="source-line-no">2658</span><span id="line-2658"> && (this.maxFlushedSeqId + this.flushPerChanges < this.mvcc.getReadPoint())</span> |
| <span class="source-line-no">2659</span><span id="line-2659"> ) {</span> |
| <span class="source-line-no">2660</span><span id="line-2660"> whyFlush.append("more than max edits, " + this.flushPerChanges + ", since last flush");</span> |
| <span class="source-line-no">2661</span><span id="line-2661"> return true;</span> |
| <span class="source-line-no">2662</span><span id="line-2662"> }</span> |
| <span class="source-line-no">2663</span><span id="line-2663"> long modifiedFlushCheckInterval = flushCheckInterval;</span> |
| <span class="source-line-no">2664</span><span id="line-2664"> if (</span> |
| <span class="source-line-no">2665</span><span id="line-2665"> getRegionInfo().getTable().isSystemTable()</span> |
| <span class="source-line-no">2666</span><span id="line-2666"> && getRegionInfo().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID</span> |
| <span class="source-line-no">2667</span><span id="line-2667"> ) {</span> |
| <span class="source-line-no">2668</span><span id="line-2668"> modifiedFlushCheckInterval = SYSTEM_CACHE_FLUSH_INTERVAL;</span> |
| <span class="source-line-no">2669</span><span id="line-2669"> }</span> |
| <span class="source-line-no">2670</span><span id="line-2670"> if (modifiedFlushCheckInterval <= 0) { // disabled</span> |
| <span class="source-line-no">2671</span><span id="line-2671"> return false;</span> |
| <span class="source-line-no">2672</span><span id="line-2672"> }</span> |
| <span class="source-line-no">2673</span><span id="line-2673"> long now = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">2674</span><span id="line-2674"> // if we flushed in the recent past, we don't need to do again now</span> |
| <span class="source-line-no">2675</span><span id="line-2675"> if ((now - getEarliestFlushTimeForAllStores() < modifiedFlushCheckInterval)) {</span> |
| <span class="source-line-no">2676</span><span id="line-2676"> return false;</span> |
| <span class="source-line-no">2677</span><span id="line-2677"> }</span> |
| <span class="source-line-no">2678</span><span id="line-2678"> // since we didn't flush in the recent past, flush now if certain conditions</span> |
| <span class="source-line-no">2679</span><span id="line-2679"> // are met. Return true on first such memstore hit.</span> |
| <span class="source-line-no">2680</span><span id="line-2680"> for (HStore s : stores.values()) {</span> |
| <span class="source-line-no">2681</span><span id="line-2681"> if (s.timeOfOldestEdit() < now - modifiedFlushCheckInterval) {</span> |
| <span class="source-line-no">2682</span><span id="line-2682"> // we have an old enough edit in the memstore, flush</span> |
| <span class="source-line-no">2683</span><span id="line-2683"> whyFlush.append(s.toString() + " has an old edit so flush to free WALs");</span> |
| <span class="source-line-no">2684</span><span id="line-2684"> return true;</span> |
| <span class="source-line-no">2685</span><span id="line-2685"> }</span> |
| <span class="source-line-no">2686</span><span id="line-2686"> }</span> |
| <span class="source-line-no">2687</span><span id="line-2687"> return false;</span> |
| <span class="source-line-no">2688</span><span id="line-2688"> }</span> |
| <span class="source-line-no">2689</span><span id="line-2689"></span> |
| <span class="source-line-no">2690</span><span id="line-2690"> /**</span> |
| <span class="source-line-no">2691</span><span id="line-2691"> * Flushing all stores.</span> |
| <span class="source-line-no">2692</span><span id="line-2692"> * @see #internalFlushcache(Collection, MonitoredTask, boolean, FlushLifeCycleTracker)</span> |
| <span class="source-line-no">2693</span><span id="line-2693"> */</span> |
| <span class="source-line-no">2694</span><span id="line-2694"> private FlushResult internalFlushcache(MonitoredTask status) throws IOException {</span> |
| <span class="source-line-no">2695</span><span id="line-2695"> return internalFlushcache(stores.values(), status, false, FlushLifeCycleTracker.DUMMY);</span> |
| <span class="source-line-no">2696</span><span id="line-2696"> }</span> |
| <span class="source-line-no">2697</span><span id="line-2697"></span> |
| <span class="source-line-no">2698</span><span id="line-2698"> /**</span> |
| <span class="source-line-no">2699</span><span id="line-2699"> * Flushing given stores.</span> |
| <span class="source-line-no">2700</span><span id="line-2700"> * @see #internalFlushcache(WAL, long, Collection, MonitoredTask, boolean, FlushLifeCycleTracker)</span> |
| <span class="source-line-no">2701</span><span id="line-2701"> */</span> |
| <span class="source-line-no">2702</span><span id="line-2702"> private FlushResultImpl internalFlushcache(Collection<HStore> storesToFlush, MonitoredTask status,</span> |
| <span class="source-line-no">2703</span><span id="line-2703"> boolean writeFlushWalMarker, FlushLifeCycleTracker tracker) throws IOException {</span> |
| <span class="source-line-no">2704</span><span id="line-2704"> return internalFlushcache(this.wal, HConstants.NO_SEQNUM, storesToFlush, status,</span> |
| <span class="source-line-no">2705</span><span id="line-2705"> writeFlushWalMarker, tracker);</span> |
| <span class="source-line-no">2706</span><span id="line-2706"> }</span> |
| <span class="source-line-no">2707</span><span id="line-2707"></span> |
| <span class="source-line-no">2708</span><span id="line-2708"> /**</span> |
| <span class="source-line-no">2709</span><span id="line-2709"> * Flush the memstore. Flushing the memstore is a little tricky. We have a lot of updates in the</span> |
| <span class="source-line-no">2710</span><span id="line-2710"> * memstore, all of which have also been written to the wal. We need to write those updates in the</span> |
| <span class="source-line-no">2711</span><span id="line-2711"> * memstore out to disk, while being able to process reads/writes as much as possible during the</span> |
| <span class="source-line-no">2712</span><span id="line-2712"> * flush operation.</span> |
| <span class="source-line-no">2713</span><span id="line-2713"> * <p></span> |
| <span class="source-line-no">2714</span><span id="line-2714"> * This method may block for some time. Every time you call it, we up the regions sequence id even</span> |
| <span class="source-line-no">2715</span><span id="line-2715"> * if we don't flush; i.e. the returned region id will be at least one larger than the last edit</span> |
| <span class="source-line-no">2716</span><span id="line-2716"> * applied to this region. The returned id does not refer to an actual edit. The returned id can</span> |
| <span class="source-line-no">2717</span><span id="line-2717"> * be used for say installing a bulk loaded file just ahead of the last hfile that was the result</span> |
| <span class="source-line-no">2718</span><span id="line-2718"> * of this flush, etc.</span> |
| <span class="source-line-no">2719</span><span id="line-2719"> * @param wal Null if we're NOT to go via wal.</span> |
| <span class="source-line-no">2720</span><span id="line-2720"> * @param myseqid The seqid to use if <code>wal</code> is null writing out flush file.</span> |
| <span class="source-line-no">2721</span><span id="line-2721"> * @param storesToFlush The list of stores to flush.</span> |
| <span class="source-line-no">2722</span><span id="line-2722"> * @return object describing the flush's state</span> |
| <span class="source-line-no">2723</span><span id="line-2723"> * @throws IOException general io exceptions</span> |
| <span class="source-line-no">2724</span><span id="line-2724"> * @throws DroppedSnapshotException Thrown when replay of WAL is required.</span> |
| <span class="source-line-no">2725</span><span id="line-2725"> */</span> |
| <span class="source-line-no">2726</span><span id="line-2726"> protected FlushResultImpl internalFlushcache(WAL wal, long myseqid,</span> |
| <span class="source-line-no">2727</span><span id="line-2727"> Collection<HStore> storesToFlush, MonitoredTask status, boolean writeFlushWalMarker,</span> |
| <span class="source-line-no">2728</span><span id="line-2728"> FlushLifeCycleTracker tracker) throws IOException {</span> |
| <span class="source-line-no">2729</span><span id="line-2729"> PrepareFlushResult result =</span> |
| <span class="source-line-no">2730</span><span id="line-2730"> internalPrepareFlushCache(wal, myseqid, storesToFlush, status, writeFlushWalMarker, tracker);</span> |
| <span class="source-line-no">2731</span><span id="line-2731"> if (result.result == null) {</span> |
| <span class="source-line-no">2732</span><span id="line-2732"> return internalFlushCacheAndCommit(wal, status, result, storesToFlush);</span> |
| <span class="source-line-no">2733</span><span id="line-2733"> } else {</span> |
| <span class="source-line-no">2734</span><span id="line-2734"> return result.result; // early exit due to failure from prepare stage</span> |
| <span class="source-line-no">2735</span><span id="line-2735"> }</span> |
| <span class="source-line-no">2736</span><span id="line-2736"> }</span> |
| <span class="source-line-no">2737</span><span id="line-2737"></span> |
| <span class="source-line-no">2738</span><span id="line-2738"> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DLS_DEAD_LOCAL_STORE",</span> |
| <span class="source-line-no">2739</span><span id="line-2739"> justification = "FindBugs seems confused about trxId")</span> |
| <span class="source-line-no">2740</span><span id="line-2740"> protected PrepareFlushResult internalPrepareFlushCache(WAL wal, long myseqid,</span> |
| <span class="source-line-no">2741</span><span id="line-2741"> Collection<HStore> storesToFlush, MonitoredTask status, boolean writeFlushWalMarker,</span> |
| <span class="source-line-no">2742</span><span id="line-2742"> FlushLifeCycleTracker tracker) throws IOException {</span> |
| <span class="source-line-no">2743</span><span id="line-2743"> if (this.rsServices != null && this.rsServices.isAborted()) {</span> |
| <span class="source-line-no">2744</span><span id="line-2744"> // Don't flush when server aborting, it's unsafe</span> |
| <span class="source-line-no">2745</span><span id="line-2745"> throw new IOException("Aborting flush because server is aborted...");</span> |
| <span class="source-line-no">2746</span><span id="line-2746"> }</span> |
| <span class="source-line-no">2747</span><span id="line-2747"> final long startTime = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">2748</span><span id="line-2748"> // If nothing to flush, return, but return with a valid unused sequenceId.</span> |
| <span class="source-line-no">2749</span><span id="line-2749"> // Its needed by bulk upload IIRC. It flushes until no edits in memory so it can insert a</span> |
| <span class="source-line-no">2750</span><span id="line-2750"> // bulk loaded file between memory and existing hfiles. It wants a good seqeunceId that belongs</span> |
| <span class="source-line-no">2751</span><span id="line-2751"> // to no other that it can use to associate with the bulk load. Hence this little dance below</span> |
| <span class="source-line-no">2752</span><span id="line-2752"> // to go get one.</span> |
| <span class="source-line-no">2753</span><span id="line-2753"> if (this.memStoreSizing.getDataSize() <= 0) {</span> |
| <span class="source-line-no">2754</span><span id="line-2754"> // Take an update lock so no edits can come into memory just yet.</span> |
| <span class="source-line-no">2755</span><span id="line-2755"> this.updatesLock.writeLock().lock();</span> |
| <span class="source-line-no">2756</span><span id="line-2756"> WriteEntry writeEntry = null;</span> |
| <span class="source-line-no">2757</span><span id="line-2757"> try {</span> |
| <span class="source-line-no">2758</span><span id="line-2758"> if (this.memStoreSizing.getDataSize() <= 0) {</span> |
| <span class="source-line-no">2759</span><span id="line-2759"> // Presume that if there are still no edits in the memstore, then there are no edits for</span> |
| <span class="source-line-no">2760</span><span id="line-2760"> // this region out in the WAL subsystem so no need to do any trickery clearing out</span> |
| <span class="source-line-no">2761</span><span id="line-2761"> // edits in the WAL sub-system. Up the sequence number so the resulting flush id is for</span> |
| <span class="source-line-no">2762</span><span id="line-2762"> // sure just beyond the last appended region edit and not associated with any edit</span> |
| <span class="source-line-no">2763</span><span id="line-2763"> // (useful as marker when bulk loading, etc.).</span> |
| <span class="source-line-no">2764</span><span id="line-2764"> if (wal != null) {</span> |
| <span class="source-line-no">2765</span><span id="line-2765"> writeEntry = mvcc.begin();</span> |
| <span class="source-line-no">2766</span><span id="line-2766"> long flushOpSeqId = writeEntry.getWriteNumber();</span> |
| <span class="source-line-no">2767</span><span id="line-2767"> FlushResultImpl flushResult = new FlushResultImpl(</span> |
| <span class="source-line-no">2768</span><span id="line-2768"> FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY, flushOpSeqId, "Nothing to flush",</span> |
| <span class="source-line-no">2769</span><span id="line-2769"> writeCanNotFlushMarkerToWAL(writeEntry, wal, writeFlushWalMarker));</span> |
| <span class="source-line-no">2770</span><span id="line-2770"> mvcc.completeAndWait(writeEntry);</span> |
| <span class="source-line-no">2771</span><span id="line-2771"> // Set to null so we don't complete it again down in finally block.</span> |
| <span class="source-line-no">2772</span><span id="line-2772"> writeEntry = null;</span> |
| <span class="source-line-no">2773</span><span id="line-2773"> return new PrepareFlushResult(flushResult, myseqid);</span> |
| <span class="source-line-no">2774</span><span id="line-2774"> } else {</span> |
| <span class="source-line-no">2775</span><span id="line-2775"> return new PrepareFlushResult(new FlushResultImpl(</span> |
| <span class="source-line-no">2776</span><span id="line-2776"> FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY, "Nothing to flush", false), myseqid);</span> |
| <span class="source-line-no">2777</span><span id="line-2777"> }</span> |
| <span class="source-line-no">2778</span><span id="line-2778"> }</span> |
| <span class="source-line-no">2779</span><span id="line-2779"> } finally {</span> |
| <span class="source-line-no">2780</span><span id="line-2780"> if (writeEntry != null) {</span> |
| <span class="source-line-no">2781</span><span id="line-2781"> // If writeEntry is non-null, this operation failed; the mvcc transaction failed...</span> |
| <span class="source-line-no">2782</span><span id="line-2782"> // but complete it anyways so it doesn't block the mvcc queue.</span> |
| <span class="source-line-no">2783</span><span id="line-2783"> mvcc.complete(writeEntry);</span> |
| <span class="source-line-no">2784</span><span id="line-2784"> }</span> |
| <span class="source-line-no">2785</span><span id="line-2785"> this.updatesLock.writeLock().unlock();</span> |
| <span class="source-line-no">2786</span><span id="line-2786"> }</span> |
| <span class="source-line-no">2787</span><span id="line-2787"> }</span> |
| <span class="source-line-no">2788</span><span id="line-2788"> logFatLineOnFlush(storesToFlush, myseqid);</span> |
| <span class="source-line-no">2789</span><span id="line-2789"> // Stop updates while we snapshot the memstore of all of these regions' stores. We only have</span> |
| <span class="source-line-no">2790</span><span id="line-2790"> // to do this for a moment. It is quick. We also set the memstore size to zero here before we</span> |
| <span class="source-line-no">2791</span><span id="line-2791"> // allow updates again so its value will represent the size of the updates received</span> |
| <span class="source-line-no">2792</span><span id="line-2792"> // during flush</span> |
| <span class="source-line-no">2793</span><span id="line-2793"></span> |
| <span class="source-line-no">2794</span><span id="line-2794"> // We have to take an update lock during snapshot, or else a write could end up in both snapshot</span> |
| <span class="source-line-no">2795</span><span id="line-2795"> // and memstore (makes it difficult to do atomic rows then)</span> |
| <span class="source-line-no">2796</span><span id="line-2796"> status.setStatus("Obtaining lock to block concurrent updates");</span> |
| <span class="source-line-no">2797</span><span id="line-2797"> // block waiting for the lock for internal flush</span> |
| <span class="source-line-no">2798</span><span id="line-2798"> this.updatesLock.writeLock().lock();</span> |
| <span class="source-line-no">2799</span><span id="line-2799"> status.setStatus("Preparing flush snapshotting stores in " + getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">2800</span><span id="line-2800"> MemStoreSizing totalSizeOfFlushableStores = new NonThreadSafeMemStoreSizing();</span> |
| <span class="source-line-no">2801</span><span id="line-2801"></span> |
| <span class="source-line-no">2802</span><span id="line-2802"> Map<byte[], Long> flushedFamilyNamesToSeq = new HashMap<>();</span> |
| <span class="source-line-no">2803</span><span id="line-2803"> for (HStore store : storesToFlush) {</span> |
| <span class="source-line-no">2804</span><span id="line-2804"> flushedFamilyNamesToSeq.put(store.getColumnFamilyDescriptor().getName(),</span> |
| <span class="source-line-no">2805</span><span id="line-2805"> store.preFlushSeqIDEstimation());</span> |
| <span class="source-line-no">2806</span><span id="line-2806"> }</span> |
| <span class="source-line-no">2807</span><span id="line-2807"></span> |
| <span class="source-line-no">2808</span><span id="line-2808"> TreeMap<byte[], StoreFlushContext> storeFlushCtxs = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">2809</span><span id="line-2809"> TreeMap<byte[], List<Path>> committedFiles = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">2810</span><span id="line-2810"> TreeMap<byte[], MemStoreSize> storeFlushableSize = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">2811</span><span id="line-2811"> // The sequence id of this flush operation which is used to log FlushMarker and pass to</span> |
| <span class="source-line-no">2812</span><span id="line-2812"> // createFlushContext to use as the store file's sequence id. It can be in advance of edits</span> |
| <span class="source-line-no">2813</span><span id="line-2813"> // still in the memstore, edits that are in other column families yet to be flushed.</span> |
| <span class="source-line-no">2814</span><span id="line-2814"> long flushOpSeqId = HConstants.NO_SEQNUM;</span> |
| <span class="source-line-no">2815</span><span id="line-2815"> // The max flushed sequence id after this flush operation completes. All edits in memstore</span> |
| <span class="source-line-no">2816</span><span id="line-2816"> // will be in advance of this sequence id.</span> |
| <span class="source-line-no">2817</span><span id="line-2817"> long flushedSeqId = HConstants.NO_SEQNUM;</span> |
| <span class="source-line-no">2818</span><span id="line-2818"> byte[] encodedRegionName = getRegionInfo().getEncodedNameAsBytes();</span> |
| <span class="source-line-no">2819</span><span id="line-2819"> try {</span> |
| <span class="source-line-no">2820</span><span id="line-2820"> if (wal != null) {</span> |
| <span class="source-line-no">2821</span><span id="line-2821"> Long earliestUnflushedSequenceIdForTheRegion =</span> |
| <span class="source-line-no">2822</span><span id="line-2822"> wal.startCacheFlush(encodedRegionName, flushedFamilyNamesToSeq);</span> |
| <span class="source-line-no">2823</span><span id="line-2823"> if (earliestUnflushedSequenceIdForTheRegion == null) {</span> |
| <span class="source-line-no">2824</span><span id="line-2824"> // This should never happen. This is how startCacheFlush signals flush cannot proceed.</span> |
| <span class="source-line-no">2825</span><span id="line-2825"> String msg = this.getRegionInfo().getEncodedName() + " flush aborted; WAL closing.";</span> |
| <span class="source-line-no">2826</span><span id="line-2826"> status.setStatus(msg);</span> |
| <span class="source-line-no">2827</span><span id="line-2827"> return new PrepareFlushResult(</span> |
| <span class="source-line-no">2828</span><span id="line-2828"> new FlushResultImpl(FlushResult.Result.CANNOT_FLUSH, msg, false), myseqid);</span> |
| <span class="source-line-no">2829</span><span id="line-2829"> }</span> |
| <span class="source-line-no">2830</span><span id="line-2830"> flushOpSeqId = getNextSequenceId(wal);</span> |
| <span class="source-line-no">2831</span><span id="line-2831"> // Back up 1, minus 1 from oldest sequence id in memstore to get last 'flushed' edit</span> |
| <span class="source-line-no">2832</span><span id="line-2832"> flushedSeqId = earliestUnflushedSequenceIdForTheRegion.longValue() == HConstants.NO_SEQNUM</span> |
| <span class="source-line-no">2833</span><span id="line-2833"> ? flushOpSeqId</span> |
| <span class="source-line-no">2834</span><span id="line-2834"> : earliestUnflushedSequenceIdForTheRegion.longValue() - 1;</span> |
| <span class="source-line-no">2835</span><span id="line-2835"> } else {</span> |
| <span class="source-line-no">2836</span><span id="line-2836"> // use the provided sequence Id as WAL is not being used for this flush.</span> |
| <span class="source-line-no">2837</span><span id="line-2837"> flushedSeqId = flushOpSeqId = myseqid;</span> |
| <span class="source-line-no">2838</span><span id="line-2838"> }</span> |
| <span class="source-line-no">2839</span><span id="line-2839"></span> |
| <span class="source-line-no">2840</span><span id="line-2840"> for (HStore s : storesToFlush) {</span> |
| <span class="source-line-no">2841</span><span id="line-2841"> storeFlushCtxs.put(s.getColumnFamilyDescriptor().getName(),</span> |
| <span class="source-line-no">2842</span><span id="line-2842"> s.createFlushContext(flushOpSeqId, tracker));</span> |
| <span class="source-line-no">2843</span><span id="line-2843"> // for writing stores to WAL</span> |
| <span class="source-line-no">2844</span><span id="line-2844"> committedFiles.put(s.getColumnFamilyDescriptor().getName(), null);</span> |
| <span class="source-line-no">2845</span><span id="line-2845"> }</span> |
| <span class="source-line-no">2846</span><span id="line-2846"></span> |
| <span class="source-line-no">2847</span><span id="line-2847"> // write the snapshot start to WAL</span> |
| <span class="source-line-no">2848</span><span id="line-2848"> if (wal != null && !writestate.readOnly) {</span> |
| <span class="source-line-no">2849</span><span id="line-2849"> FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.START_FLUSH,</span> |
| <span class="source-line-no">2850</span><span id="line-2850"> getRegionInfo(), flushOpSeqId, committedFiles);</span> |
| <span class="source-line-no">2851</span><span id="line-2851"> // No sync. Sync is below where no updates lock and we do FlushAction.COMMIT_FLUSH</span> |
| <span class="source-line-no">2852</span><span id="line-2852"> WALUtil.writeFlushMarker(wal, this.getReplicationScope(), getRegionInfo(), desc, false,</span> |
| <span class="source-line-no">2853</span><span id="line-2853"> mvcc, regionReplicationSink.orElse(null));</span> |
| <span class="source-line-no">2854</span><span id="line-2854"> }</span> |
| <span class="source-line-no">2855</span><span id="line-2855"></span> |
| <span class="source-line-no">2856</span><span id="line-2856"> // Prepare flush (take a snapshot)</span> |
| <span class="source-line-no">2857</span><span id="line-2857"> storeFlushCtxs.forEach((name, flush) -> {</span> |
| <span class="source-line-no">2858</span><span id="line-2858"> MemStoreSize snapshotSize = flush.prepare();</span> |
| <span class="source-line-no">2859</span><span id="line-2859"> totalSizeOfFlushableStores.incMemStoreSize(snapshotSize);</span> |
| <span class="source-line-no">2860</span><span id="line-2860"> storeFlushableSize.put(name, snapshotSize);</span> |
| <span class="source-line-no">2861</span><span id="line-2861"> });</span> |
| <span class="source-line-no">2862</span><span id="line-2862"> } catch (IOException ex) {</span> |
| <span class="source-line-no">2863</span><span id="line-2863"> doAbortFlushToWAL(wal, flushOpSeqId, committedFiles);</span> |
| <span class="source-line-no">2864</span><span id="line-2864"> throw ex;</span> |
| <span class="source-line-no">2865</span><span id="line-2865"> } finally {</span> |
| <span class="source-line-no">2866</span><span id="line-2866"> this.updatesLock.writeLock().unlock();</span> |
| <span class="source-line-no">2867</span><span id="line-2867"> }</span> |
| <span class="source-line-no">2868</span><span id="line-2868"> String s = "Finished memstore snapshotting " + this + ", syncing WAL and waiting on mvcc, "</span> |
| <span class="source-line-no">2869</span><span id="line-2869"> + "flushsize=" + totalSizeOfFlushableStores;</span> |
| <span class="source-line-no">2870</span><span id="line-2870"> status.setStatus(s);</span> |
| <span class="source-line-no">2871</span><span id="line-2871"> doSyncOfUnflushedWALChanges(wal, getRegionInfo());</span> |
| <span class="source-line-no">2872</span><span id="line-2872"> return new PrepareFlushResult(storeFlushCtxs, committedFiles, storeFlushableSize, startTime,</span> |
| <span class="source-line-no">2873</span><span id="line-2873"> flushOpSeqId, flushedSeqId, totalSizeOfFlushableStores);</span> |
| <span class="source-line-no">2874</span><span id="line-2874"> }</span> |
| <span class="source-line-no">2875</span><span id="line-2875"></span> |
| <span class="source-line-no">2876</span><span id="line-2876"> /**</span> |
| <span class="source-line-no">2877</span><span id="line-2877"> * Utility method broken out of internalPrepareFlushCache so that method is smaller.</span> |
| <span class="source-line-no">2878</span><span id="line-2878"> */</span> |
| <span class="source-line-no">2879</span><span id="line-2879"> private void logFatLineOnFlush(Collection<HStore> storesToFlush, long sequenceId) {</span> |
| <span class="source-line-no">2880</span><span id="line-2880"> if (!LOG.isInfoEnabled()) {</span> |
| <span class="source-line-no">2881</span><span id="line-2881"> return;</span> |
| <span class="source-line-no">2882</span><span id="line-2882"> }</span> |
| <span class="source-line-no">2883</span><span id="line-2883"> // Log a fat line detailing what is being flushed.</span> |
| <span class="source-line-no">2884</span><span id="line-2884"> StringBuilder perCfExtras = null;</span> |
| <span class="source-line-no">2885</span><span id="line-2885"> if (!isAllFamilies(storesToFlush)) {</span> |
| <span class="source-line-no">2886</span><span id="line-2886"> perCfExtras = new StringBuilder();</span> |
| <span class="source-line-no">2887</span><span id="line-2887"> for (HStore store : storesToFlush) {</span> |
| <span class="source-line-no">2888</span><span id="line-2888"> MemStoreSize mss = store.getFlushableSize();</span> |
| <span class="source-line-no">2889</span><span id="line-2889"> perCfExtras.append("; ").append(store.getColumnFamilyName());</span> |
| <span class="source-line-no">2890</span><span id="line-2890"> perCfExtras.append("={dataSize=").append(StringUtils.byteDesc(mss.getDataSize()));</span> |
| <span class="source-line-no">2891</span><span id="line-2891"> perCfExtras.append(", heapSize=").append(StringUtils.byteDesc(mss.getHeapSize()));</span> |
| <span class="source-line-no">2892</span><span id="line-2892"> perCfExtras.append(", offHeapSize=").append(StringUtils.byteDesc(mss.getOffHeapSize()));</span> |
| <span class="source-line-no">2893</span><span id="line-2893"> perCfExtras.append("}");</span> |
| <span class="source-line-no">2894</span><span id="line-2894"> }</span> |
| <span class="source-line-no">2895</span><span id="line-2895"> }</span> |
| <span class="source-line-no">2896</span><span id="line-2896"> MemStoreSize mss = this.memStoreSizing.getMemStoreSize();</span> |
| <span class="source-line-no">2897</span><span id="line-2897"> LOG.info("Flushing " + this.getRegionInfo().getEncodedName() + " " + storesToFlush.size() + "/"</span> |
| <span class="source-line-no">2898</span><span id="line-2898"> + stores.size() + " column families," + " dataSize=" + StringUtils.byteDesc(mss.getDataSize())</span> |
| <span class="source-line-no">2899</span><span id="line-2899"> + " heapSize=" + StringUtils.byteDesc(mss.getHeapSize())</span> |
| <span class="source-line-no">2900</span><span id="line-2900"> + ((perCfExtras != null && perCfExtras.length() > 0) ? perCfExtras.toString() : "")</span> |
| <span class="source-line-no">2901</span><span id="line-2901"> + ((wal != null) ? "" : "; WAL is null, using passed sequenceid=" + sequenceId));</span> |
| <span class="source-line-no">2902</span><span id="line-2902"> }</span> |
| <span class="source-line-no">2903</span><span id="line-2903"></span> |
| <span class="source-line-no">2904</span><span id="line-2904"> private void doAbortFlushToWAL(final WAL wal, final long flushOpSeqId,</span> |
| <span class="source-line-no">2905</span><span id="line-2905"> final Map<byte[], List<Path>> committedFiles) {</span> |
| <span class="source-line-no">2906</span><span id="line-2906"> if (wal == null) return;</span> |
| <span class="source-line-no">2907</span><span id="line-2907"> try {</span> |
| <span class="source-line-no">2908</span><span id="line-2908"> FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.ABORT_FLUSH,</span> |
| <span class="source-line-no">2909</span><span id="line-2909"> getRegionInfo(), flushOpSeqId, committedFiles);</span> |
| <span class="source-line-no">2910</span><span id="line-2910"> WALUtil.writeFlushMarker(wal, this.getReplicationScope(), getRegionInfo(), desc, false, mvcc,</span> |
| <span class="source-line-no">2911</span><span id="line-2911"> null);</span> |
| <span class="source-line-no">2912</span><span id="line-2912"> } catch (Throwable t) {</span> |
| <span class="source-line-no">2913</span><span id="line-2913"> LOG.warn("Received unexpected exception trying to write ABORT_FLUSH marker to WAL: {} in "</span> |
| <span class="source-line-no">2914</span><span id="line-2914"> + " region {}", StringUtils.stringifyException(t), this);</span> |
| <span class="source-line-no">2915</span><span id="line-2915"> // ignore this since we will be aborting the RS with DSE.</span> |
| <span class="source-line-no">2916</span><span id="line-2916"> }</span> |
| <span class="source-line-no">2917</span><span id="line-2917"> // we have called wal.startCacheFlush(), now we have to abort it</span> |
| <span class="source-line-no">2918</span><span id="line-2918"> wal.abortCacheFlush(this.getRegionInfo().getEncodedNameAsBytes());</span> |
| <span class="source-line-no">2919</span><span id="line-2919"> }</span> |
| <span class="source-line-no">2920</span><span id="line-2920"></span> |
| <span class="source-line-no">2921</span><span id="line-2921"> /**</span> |
| <span class="source-line-no">2922</span><span id="line-2922"> * Sync unflushed WAL changes. See HBASE-8208 for details</span> |
| <span class="source-line-no">2923</span><span id="line-2923"> */</span> |
| <span class="source-line-no">2924</span><span id="line-2924"> private static void doSyncOfUnflushedWALChanges(final WAL wal, final RegionInfo hri)</span> |
| <span class="source-line-no">2925</span><span id="line-2925"> throws IOException {</span> |
| <span class="source-line-no">2926</span><span id="line-2926"> if (wal == null) {</span> |
| <span class="source-line-no">2927</span><span id="line-2927"> return;</span> |
| <span class="source-line-no">2928</span><span id="line-2928"> }</span> |
| <span class="source-line-no">2929</span><span id="line-2929"> try {</span> |
| <span class="source-line-no">2930</span><span id="line-2930"> wal.sync(); // ensure that flush marker is sync'ed</span> |
| <span class="source-line-no">2931</span><span id="line-2931"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">2932</span><span id="line-2932"> wal.abortCacheFlush(hri.getEncodedNameAsBytes());</span> |
| <span class="source-line-no">2933</span><span id="line-2933"> throw ioe;</span> |
| <span class="source-line-no">2934</span><span id="line-2934"> }</span> |
| <span class="source-line-no">2935</span><span id="line-2935"> }</span> |
| <span class="source-line-no">2936</span><span id="line-2936"></span> |
| <span class="source-line-no">2937</span><span id="line-2937"> /** Returns True if passed Set is all families in the region. */</span> |
| <span class="source-line-no">2938</span><span id="line-2938"> private boolean isAllFamilies(Collection<HStore> families) {</span> |
| <span class="source-line-no">2939</span><span id="line-2939"> return families == null || this.stores.size() == families.size();</span> |
| <span class="source-line-no">2940</span><span id="line-2940"> }</span> |
| <span class="source-line-no">2941</span><span id="line-2941"></span> |
| <span class="source-line-no">2942</span><span id="line-2942"> /**</span> |
| <span class="source-line-no">2943</span><span id="line-2943"> * This method is only used when we flush but the memstore is empty,if writeFlushWalMarker is</span> |
| <span class="source-line-no">2944</span><span id="line-2944"> * true,we write the {@link FlushAction#CANNOT_FLUSH} flush marker to WAL when the memstore is</span> |
| <span class="source-line-no">2945</span><span id="line-2945"> * empty. Ignores exceptions from WAL. Returns whether the write succeeded.</span> |
| <span class="source-line-no">2946</span><span id="line-2946"> * @return whether WAL write was successful</span> |
| <span class="source-line-no">2947</span><span id="line-2947"> */</span> |
| <span class="source-line-no">2948</span><span id="line-2948"> private boolean writeCanNotFlushMarkerToWAL(WriteEntry flushOpSeqIdMVCCEntry, WAL wal,</span> |
| <span class="source-line-no">2949</span><span id="line-2949"> boolean writeFlushWalMarker) {</span> |
| <span class="source-line-no">2950</span><span id="line-2950"> FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.CANNOT_FLUSH, getRegionInfo(),</span> |
| <span class="source-line-no">2951</span><span id="line-2951"> -1, new TreeMap<>(Bytes.BYTES_COMPARATOR));</span> |
| <span class="source-line-no">2952</span><span id="line-2952"> RegionReplicationSink sink = regionReplicationSink.orElse(null);</span> |
| <span class="source-line-no">2953</span><span id="line-2953"></span> |
| <span class="source-line-no">2954</span><span id="line-2954"> if (sink != null && !writeFlushWalMarker) {</span> |
| <span class="source-line-no">2955</span><span id="line-2955"> /**</span> |
| <span class="source-line-no">2956</span><span id="line-2956"> * Here for replication to secondary region replica could use {@link FlushAction#CANNOT_FLUSH}</span> |
| <span class="source-line-no">2957</span><span id="line-2957"> * to recover when writeFlushWalMarker is false, we create {@link WALEdit} for</span> |
| <span class="source-line-no">2958</span><span id="line-2958"> * {@link FlushDescriptor} and attach the {@link RegionReplicationSink#add} to the</span> |
| <span class="source-line-no">2959</span><span id="line-2959"> * flushOpSeqIdMVCCEntry,see HBASE-26960 for more details.</span> |
| <span class="source-line-no">2960</span><span id="line-2960"> */</span> |
| <span class="source-line-no">2961</span><span id="line-2961"> this.attachRegionReplicationToFlushOpSeqIdMVCCEntry(flushOpSeqIdMVCCEntry, desc, sink);</span> |
| <span class="source-line-no">2962</span><span id="line-2962"> return false;</span> |
| <span class="source-line-no">2963</span><span id="line-2963"> }</span> |
| <span class="source-line-no">2964</span><span id="line-2964"></span> |
| <span class="source-line-no">2965</span><span id="line-2965"> if (writeFlushWalMarker && wal != null && !writestate.readOnly) {</span> |
| <span class="source-line-no">2966</span><span id="line-2966"> try {</span> |
| <span class="source-line-no">2967</span><span id="line-2967"> WALUtil.writeFlushMarker(wal, this.getReplicationScope(), getRegionInfo(), desc, true, mvcc,</span> |
| <span class="source-line-no">2968</span><span id="line-2968"> sink);</span> |
| <span class="source-line-no">2969</span><span id="line-2969"> return true;</span> |
| <span class="source-line-no">2970</span><span id="line-2970"> } catch (IOException e) {</span> |
| <span class="source-line-no">2971</span><span id="line-2971"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">2972</span><span id="line-2972"> + "Received exception while trying to write the flush request to wal", e);</span> |
| <span class="source-line-no">2973</span><span id="line-2973"> }</span> |
| <span class="source-line-no">2974</span><span id="line-2974"> }</span> |
| <span class="source-line-no">2975</span><span id="line-2975"> return false;</span> |
| <span class="source-line-no">2976</span><span id="line-2976"> }</span> |
| <span class="source-line-no">2977</span><span id="line-2977"></span> |
| <span class="source-line-no">2978</span><span id="line-2978"> /**</span> |
| <span class="source-line-no">2979</span><span id="line-2979"> * Create {@link WALEdit} for {@link FlushDescriptor} and attach {@link RegionReplicationSink#add}</span> |
| <span class="source-line-no">2980</span><span id="line-2980"> * to the flushOpSeqIdMVCCEntry.</span> |
| <span class="source-line-no">2981</span><span id="line-2981"> */</span> |
| <span class="source-line-no">2982</span><span id="line-2982"> private void attachRegionReplicationToFlushOpSeqIdMVCCEntry(WriteEntry flushOpSeqIdMVCCEntry,</span> |
| <span class="source-line-no">2983</span><span id="line-2983"> FlushDescriptor desc, RegionReplicationSink sink) {</span> |
| <span class="source-line-no">2984</span><span id="line-2984"> assert !flushOpSeqIdMVCCEntry.getCompletionAction().isPresent();</span> |
| <span class="source-line-no">2985</span><span id="line-2985"> WALEdit flushMarkerWALEdit = WALEdit.createFlushWALEdit(getRegionInfo(), desc);</span> |
| <span class="source-line-no">2986</span><span id="line-2986"> WALKeyImpl walKey =</span> |
| <span class="source-line-no">2987</span><span id="line-2987"> WALUtil.createWALKey(getRegionInfo(), mvcc, this.getReplicationScope(), null);</span> |
| <span class="source-line-no">2988</span><span id="line-2988"> walKey.setWriteEntry(flushOpSeqIdMVCCEntry);</span> |
| <span class="source-line-no">2989</span><span id="line-2989"> /**</span> |
| <span class="source-line-no">2990</span><span id="line-2990"> * Here the {@link ServerCall} is null for {@link RegionReplicationSink#add} because the</span> |
| <span class="source-line-no">2991</span><span id="line-2991"> * flushMarkerWALEdit is created by ourselves, not from rpc.</span> |
| <span class="source-line-no">2992</span><span id="line-2992"> */</span> |
| <span class="source-line-no">2993</span><span id="line-2993"> flushOpSeqIdMVCCEntry.attachCompletionAction(() -> sink.add(walKey, flushMarkerWALEdit, null));</span> |
| <span class="source-line-no">2994</span><span id="line-2994"> }</span> |
| <span class="source-line-no">2995</span><span id="line-2995"></span> |
| <span class="source-line-no">2996</span><span id="line-2996"> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NN_NAKED_NOTIFY",</span> |
| <span class="source-line-no">2997</span><span id="line-2997"> justification = "Intentional; notify is about completed flush")</span> |
| <span class="source-line-no">2998</span><span id="line-2998"> FlushResultImpl internalFlushCacheAndCommit(WAL wal, MonitoredTask status,</span> |
| <span class="source-line-no">2999</span><span id="line-2999"> PrepareFlushResult prepareResult, Collection<HStore> storesToFlush) throws IOException {</span> |
| <span class="source-line-no">3000</span><span id="line-3000"> // prepare flush context is carried via PrepareFlushResult</span> |
| <span class="source-line-no">3001</span><span id="line-3001"> TreeMap<byte[], StoreFlushContext> storeFlushCtxs = prepareResult.storeFlushCtxs;</span> |
| <span class="source-line-no">3002</span><span id="line-3002"> TreeMap<byte[], List<Path>> committedFiles = prepareResult.committedFiles;</span> |
| <span class="source-line-no">3003</span><span id="line-3003"> long startTime = prepareResult.startTime;</span> |
| <span class="source-line-no">3004</span><span id="line-3004"> long flushOpSeqId = prepareResult.flushOpSeqId;</span> |
| <span class="source-line-no">3005</span><span id="line-3005"> long flushedSeqId = prepareResult.flushedSeqId;</span> |
| <span class="source-line-no">3006</span><span id="line-3006"></span> |
| <span class="source-line-no">3007</span><span id="line-3007"> String s = "Flushing stores of " + this;</span> |
| <span class="source-line-no">3008</span><span id="line-3008"> status.setStatus(s);</span> |
| <span class="source-line-no">3009</span><span id="line-3009"> if (LOG.isTraceEnabled()) LOG.trace(s);</span> |
| <span class="source-line-no">3010</span><span id="line-3010"></span> |
| <span class="source-line-no">3011</span><span id="line-3011"> // Any failure from here on out will be catastrophic requiring server</span> |
| <span class="source-line-no">3012</span><span id="line-3012"> // restart so wal content can be replayed and put back into the memstore.</span> |
| <span class="source-line-no">3013</span><span id="line-3013"> // Otherwise, the snapshot content while backed up in the wal, it will not</span> |
| <span class="source-line-no">3014</span><span id="line-3014"> // be part of the current running servers state.</span> |
| <span class="source-line-no">3015</span><span id="line-3015"> boolean compactionRequested = false;</span> |
| <span class="source-line-no">3016</span><span id="line-3016"> long flushedOutputFileSize = 0;</span> |
| <span class="source-line-no">3017</span><span id="line-3017"> try {</span> |
| <span class="source-line-no">3018</span><span id="line-3018"> // A. Flush memstore to all the HStores.</span> |
| <span class="source-line-no">3019</span><span id="line-3019"> // Keep running vector of all store files that includes both old and the</span> |
| <span class="source-line-no">3020</span><span id="line-3020"> // just-made new flush store file. The new flushed file is still in the</span> |
| <span class="source-line-no">3021</span><span id="line-3021"> // tmp directory.</span> |
| <span class="source-line-no">3022</span><span id="line-3022"></span> |
| <span class="source-line-no">3023</span><span id="line-3023"> for (StoreFlushContext flush : storeFlushCtxs.values()) {</span> |
| <span class="source-line-no">3024</span><span id="line-3024"> flush.flushCache(status);</span> |
| <span class="source-line-no">3025</span><span id="line-3025"> }</span> |
| <span class="source-line-no">3026</span><span id="line-3026"></span> |
| <span class="source-line-no">3027</span><span id="line-3027"> // Switch snapshot (in memstore) -> new hfile (thus causing</span> |
| <span class="source-line-no">3028</span><span id="line-3028"> // all the store scanners to reset/reseek).</span> |
| <span class="source-line-no">3029</span><span id="line-3029"> for (Map.Entry<byte[], StoreFlushContext> flushEntry : storeFlushCtxs.entrySet()) {</span> |
| <span class="source-line-no">3030</span><span id="line-3030"> StoreFlushContext sfc = flushEntry.getValue();</span> |
| <span class="source-line-no">3031</span><span id="line-3031"> boolean needsCompaction = sfc.commit(status);</span> |
| <span class="source-line-no">3032</span><span id="line-3032"> if (needsCompaction) {</span> |
| <span class="source-line-no">3033</span><span id="line-3033"> compactionRequested = true;</span> |
| <span class="source-line-no">3034</span><span id="line-3034"> }</span> |
| <span class="source-line-no">3035</span><span id="line-3035"> byte[] storeName = flushEntry.getKey();</span> |
| <span class="source-line-no">3036</span><span id="line-3036"> List<Path> storeCommittedFiles = sfc.getCommittedFiles();</span> |
| <span class="source-line-no">3037</span><span id="line-3037"> committedFiles.put(storeName, storeCommittedFiles);</span> |
| <span class="source-line-no">3038</span><span id="line-3038"> // Flush committed no files, indicating flush is empty or flush was canceled</span> |
| <span class="source-line-no">3039</span><span id="line-3039"> if (storeCommittedFiles == null || storeCommittedFiles.isEmpty()) {</span> |
| <span class="source-line-no">3040</span><span id="line-3040"> MemStoreSize storeFlushableSize = prepareResult.storeFlushableSize.get(storeName);</span> |
| <span class="source-line-no">3041</span><span id="line-3041"> prepareResult.totalFlushableSize.decMemStoreSize(storeFlushableSize);</span> |
| <span class="source-line-no">3042</span><span id="line-3042"> }</span> |
| <span class="source-line-no">3043</span><span id="line-3043"> flushedOutputFileSize += sfc.getOutputFileSize();</span> |
| <span class="source-line-no">3044</span><span id="line-3044"> }</span> |
| <span class="source-line-no">3045</span><span id="line-3045"> storeFlushCtxs.clear();</span> |
| <span class="source-line-no">3046</span><span id="line-3046"></span> |
| <span class="source-line-no">3047</span><span id="line-3047"> // Set down the memstore size by amount of flush.</span> |
| <span class="source-line-no">3048</span><span id="line-3048"> MemStoreSize mss = prepareResult.totalFlushableSize.getMemStoreSize();</span> |
| <span class="source-line-no">3049</span><span id="line-3049"> this.decrMemStoreSize(mss);</span> |
| <span class="source-line-no">3050</span><span id="line-3050"></span> |
| <span class="source-line-no">3051</span><span id="line-3051"> // Increase the size of this Region for the purposes of quota. Noop if quotas are disabled.</span> |
| <span class="source-line-no">3052</span><span id="line-3052"> // During startup, quota manager may not be initialized yet.</span> |
| <span class="source-line-no">3053</span><span id="line-3053"> if (rsServices != null) {</span> |
| <span class="source-line-no">3054</span><span id="line-3054"> RegionServerSpaceQuotaManager quotaManager = rsServices.getRegionServerSpaceQuotaManager();</span> |
| <span class="source-line-no">3055</span><span id="line-3055"> if (quotaManager != null) {</span> |
| <span class="source-line-no">3056</span><span id="line-3056"> quotaManager.getRegionSizeStore().incrementRegionSize(this.getRegionInfo(),</span> |
| <span class="source-line-no">3057</span><span id="line-3057"> flushedOutputFileSize);</span> |
| <span class="source-line-no">3058</span><span id="line-3058"> }</span> |
| <span class="source-line-no">3059</span><span id="line-3059"> }</span> |
| <span class="source-line-no">3060</span><span id="line-3060"></span> |
| <span class="source-line-no">3061</span><span id="line-3061"> if (wal != null) {</span> |
| <span class="source-line-no">3062</span><span id="line-3062"> // write flush marker to WAL. If fail, we should throw DroppedSnapshotException</span> |
| <span class="source-line-no">3063</span><span id="line-3063"> FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.COMMIT_FLUSH,</span> |
| <span class="source-line-no">3064</span><span id="line-3064"> getRegionInfo(), flushOpSeqId, committedFiles);</span> |
| <span class="source-line-no">3065</span><span id="line-3065"> WALUtil.writeFlushMarker(wal, this.getReplicationScope(), getRegionInfo(), desc, true, mvcc,</span> |
| <span class="source-line-no">3066</span><span id="line-3066"> regionReplicationSink.orElse(null));</span> |
| <span class="source-line-no">3067</span><span id="line-3067"> }</span> |
| <span class="source-line-no">3068</span><span id="line-3068"> } catch (Throwable t) {</span> |
| <span class="source-line-no">3069</span><span id="line-3069"> // An exception here means that the snapshot was not persisted.</span> |
| <span class="source-line-no">3070</span><span id="line-3070"> // The wal needs to be replayed so its content is restored to memstore.</span> |
| <span class="source-line-no">3071</span><span id="line-3071"> // Currently, only a server restart will do this.</span> |
| <span class="source-line-no">3072</span><span id="line-3072"> // We used to only catch IOEs but its possible that we'd get other</span> |
| <span class="source-line-no">3073</span><span id="line-3073"> // exceptions -- e.g. HBASE-659 was about an NPE -- so now we catch</span> |
| <span class="source-line-no">3074</span><span id="line-3074"> // all and sundry.</span> |
| <span class="source-line-no">3075</span><span id="line-3075"> if (wal != null) {</span> |
| <span class="source-line-no">3076</span><span id="line-3076"> try {</span> |
| <span class="source-line-no">3077</span><span id="line-3077"> FlushDescriptor desc = ProtobufUtil.toFlushDescriptor(FlushAction.ABORT_FLUSH,</span> |
| <span class="source-line-no">3078</span><span id="line-3078"> getRegionInfo(), flushOpSeqId, committedFiles);</span> |
| <span class="source-line-no">3079</span><span id="line-3079"> WALUtil.writeFlushMarker(wal, this.replicationScope, getRegionInfo(), desc, false, mvcc,</span> |
| <span class="source-line-no">3080</span><span id="line-3080"> null);</span> |
| <span class="source-line-no">3081</span><span id="line-3081"> } catch (Throwable ex) {</span> |
| <span class="source-line-no">3082</span><span id="line-3082"> LOG.warn(</span> |
| <span class="source-line-no">3083</span><span id="line-3083"> getRegionInfo().getEncodedName() + " : " + "failed writing ABORT_FLUSH marker to WAL",</span> |
| <span class="source-line-no">3084</span><span id="line-3084"> ex);</span> |
| <span class="source-line-no">3085</span><span id="line-3085"> // ignore this since we will be aborting the RS with DSE.</span> |
| <span class="source-line-no">3086</span><span id="line-3086"> }</span> |
| <span class="source-line-no">3087</span><span id="line-3087"> wal.abortCacheFlush(this.getRegionInfo().getEncodedNameAsBytes());</span> |
| <span class="source-line-no">3088</span><span id="line-3088"> }</span> |
| <span class="source-line-no">3089</span><span id="line-3089"> DroppedSnapshotException dse = new DroppedSnapshotException(</span> |
| <span class="source-line-no">3090</span><span id="line-3090"> "region: " + Bytes.toStringBinary(getRegionInfo().getRegionName()), t);</span> |
| <span class="source-line-no">3091</span><span id="line-3091"> status.abort("Flush failed: " + StringUtils.stringifyException(t));</span> |
| <span class="source-line-no">3092</span><span id="line-3092"></span> |
| <span class="source-line-no">3093</span><span id="line-3093"> // Callers for flushcache() should catch DroppedSnapshotException and abort the region server.</span> |
| <span class="source-line-no">3094</span><span id="line-3094"> // However, since we may have the region read lock, we cannot call close(true) here since</span> |
| <span class="source-line-no">3095</span><span id="line-3095"> // we cannot promote to a write lock. Instead we are setting closing so that all other region</span> |
| <span class="source-line-no">3096</span><span id="line-3096"> // operations except for close will be rejected.</span> |
| <span class="source-line-no">3097</span><span id="line-3097"> this.closing.set(true);</span> |
| <span class="source-line-no">3098</span><span id="line-3098"></span> |
| <span class="source-line-no">3099</span><span id="line-3099"> if (rsServices != null) {</span> |
| <span class="source-line-no">3100</span><span id="line-3100"> // This is a safeguard against the case where the caller fails to explicitly handle aborting</span> |
| <span class="source-line-no">3101</span><span id="line-3101"> rsServices.abort("Replay of WAL required. Forcing server shutdown", dse);</span> |
| <span class="source-line-no">3102</span><span id="line-3102"> }</span> |
| <span class="source-line-no">3103</span><span id="line-3103"></span> |
| <span class="source-line-no">3104</span><span id="line-3104"> throw dse;</span> |
| <span class="source-line-no">3105</span><span id="line-3105"> }</span> |
| <span class="source-line-no">3106</span><span id="line-3106"></span> |
| <span class="source-line-no">3107</span><span id="line-3107"> // If we get to here, the HStores have been written.</span> |
| <span class="source-line-no">3108</span><span id="line-3108"> if (wal != null) {</span> |
| <span class="source-line-no">3109</span><span id="line-3109"> wal.completeCacheFlush(this.getRegionInfo().getEncodedNameAsBytes(), flushedSeqId);</span> |
| <span class="source-line-no">3110</span><span id="line-3110"> }</span> |
| <span class="source-line-no">3111</span><span id="line-3111"></span> |
| <span class="source-line-no">3112</span><span id="line-3112"> // Record latest flush time</span> |
| <span class="source-line-no">3113</span><span id="line-3113"> for (HStore store : storesToFlush) {</span> |
| <span class="source-line-no">3114</span><span id="line-3114"> this.lastStoreFlushTimeMap.put(store, startTime);</span> |
| <span class="source-line-no">3115</span><span id="line-3115"> }</span> |
| <span class="source-line-no">3116</span><span id="line-3116"></span> |
| <span class="source-line-no">3117</span><span id="line-3117"> this.maxFlushedSeqId = flushedSeqId;</span> |
| <span class="source-line-no">3118</span><span id="line-3118"> this.lastFlushOpSeqId = flushOpSeqId;</span> |
| <span class="source-line-no">3119</span><span id="line-3119"></span> |
| <span class="source-line-no">3120</span><span id="line-3120"> // C. Finally notify anyone waiting on memstore to clear:</span> |
| <span class="source-line-no">3121</span><span id="line-3121"> // e.g. checkResources().</span> |
| <span class="source-line-no">3122</span><span id="line-3122"> synchronized (this) {</span> |
| <span class="source-line-no">3123</span><span id="line-3123"> notifyAll(); // FindBugs NN_NAKED_NOTIFY</span> |
| <span class="source-line-no">3124</span><span id="line-3124"> }</span> |
| <span class="source-line-no">3125</span><span id="line-3125"></span> |
| <span class="source-line-no">3126</span><span id="line-3126"> long time = EnvironmentEdgeManager.currentTime() - startTime;</span> |
| <span class="source-line-no">3127</span><span id="line-3127"> MemStoreSize mss = prepareResult.totalFlushableSize.getMemStoreSize();</span> |
| <span class="source-line-no">3128</span><span id="line-3128"> long memstoresize = this.memStoreSizing.getMemStoreSize().getDataSize();</span> |
| <span class="source-line-no">3129</span><span id="line-3129"> String msg = "Finished flush of" + " dataSize ~" + StringUtils.byteDesc(mss.getDataSize()) + "/"</span> |
| <span class="source-line-no">3130</span><span id="line-3130"> + mss.getDataSize() + ", heapSize ~" + StringUtils.byteDesc(mss.getHeapSize()) + "/"</span> |
| <span class="source-line-no">3131</span><span id="line-3131"> + mss.getHeapSize() + ", currentSize=" + StringUtils.byteDesc(memstoresize) + "/"</span> |
| <span class="source-line-no">3132</span><span id="line-3132"> + memstoresize + " for " + this.getRegionInfo().getEncodedName() + " in " + time</span> |
| <span class="source-line-no">3133</span><span id="line-3133"> + "ms, sequenceid=" + flushOpSeqId + ", compaction requested=" + compactionRequested</span> |
| <span class="source-line-no">3134</span><span id="line-3134"> + ((wal == null) ? "; wal=null" : "");</span> |
| <span class="source-line-no">3135</span><span id="line-3135"> LOG.info(msg);</span> |
| <span class="source-line-no">3136</span><span id="line-3136"> status.setStatus(msg);</span> |
| <span class="source-line-no">3137</span><span id="line-3137"></span> |
| <span class="source-line-no">3138</span><span id="line-3138"> if (rsServices != null && rsServices.getMetrics() != null) {</span> |
| <span class="source-line-no">3139</span><span id="line-3139"> rsServices.getMetrics().updateFlush(getTableDescriptor().getTableName().getNameAsString(),</span> |
| <span class="source-line-no">3140</span><span id="line-3140"> time, mss.getDataSize(), flushedOutputFileSize);</span> |
| <span class="source-line-no">3141</span><span id="line-3141"> }</span> |
| <span class="source-line-no">3142</span><span id="line-3142"></span> |
| <span class="source-line-no">3143</span><span id="line-3143"> return new FlushResultImpl(compactionRequested</span> |
| <span class="source-line-no">3144</span><span id="line-3144"> ? FlushResult.Result.FLUSHED_COMPACTION_NEEDED</span> |
| <span class="source-line-no">3145</span><span id="line-3145"> : FlushResult.Result.FLUSHED_NO_COMPACTION_NEEDED, flushOpSeqId);</span> |
| <span class="source-line-no">3146</span><span id="line-3146"> }</span> |
| <span class="source-line-no">3147</span><span id="line-3147"></span> |
| <span class="source-line-no">3148</span><span id="line-3148"> /**</span> |
| <span class="source-line-no">3149</span><span id="line-3149"> * Method to safely get the next sequence number.</span> |
| <span class="source-line-no">3150</span><span id="line-3150"> * @return Next sequence number unassociated with any actual edit.</span> |
| <span class="source-line-no">3151</span><span id="line-3151"> */</span> |
| <span class="source-line-no">3152</span><span id="line-3152"> protected long getNextSequenceId(final WAL wal) throws IOException {</span> |
| <span class="source-line-no">3153</span><span id="line-3153"> WriteEntry we = mvcc.begin();</span> |
| <span class="source-line-no">3154</span><span id="line-3154"> mvcc.completeAndWait(we);</span> |
| <span class="source-line-no">3155</span><span id="line-3155"> return we.getWriteNumber();</span> |
| <span class="source-line-no">3156</span><span id="line-3156"> }</span> |
| <span class="source-line-no">3157</span><span id="line-3157"></span> |
| <span class="source-line-no">3158</span><span id="line-3158"> //////////////////////////////////////////////////////////////////////////////</span> |
| <span class="source-line-no">3159</span><span id="line-3159"> // get() methods for client use.</span> |
| <span class="source-line-no">3160</span><span id="line-3160"> //////////////////////////////////////////////////////////////////////////////</span> |
| <span class="source-line-no">3161</span><span id="line-3161"></span> |
| <span class="source-line-no">3162</span><span id="line-3162"> @Override</span> |
| <span class="source-line-no">3163</span><span id="line-3163"> public RegionScannerImpl getScanner(Scan scan) throws IOException {</span> |
| <span class="source-line-no">3164</span><span id="line-3164"> return getScanner(scan, null);</span> |
| <span class="source-line-no">3165</span><span id="line-3165"> }</span> |
| <span class="source-line-no">3166</span><span id="line-3166"></span> |
| <span class="source-line-no">3167</span><span id="line-3167"> @Override</span> |
| <span class="source-line-no">3168</span><span id="line-3168"> public RegionScannerImpl getScanner(Scan scan, List<KeyValueScanner> additionalScanners)</span> |
| <span class="source-line-no">3169</span><span id="line-3169"> throws IOException {</span> |
| <span class="source-line-no">3170</span><span id="line-3170"> return getScanner(scan, additionalScanners, HConstants.NO_NONCE, HConstants.NO_NONCE);</span> |
| <span class="source-line-no">3171</span><span id="line-3171"> }</span> |
| <span class="source-line-no">3172</span><span id="line-3172"></span> |
| <span class="source-line-no">3173</span><span id="line-3173"> private RegionScannerImpl getScanner(Scan scan, List<KeyValueScanner> additionalScanners,</span> |
| <span class="source-line-no">3174</span><span id="line-3174"> long nonceGroup, long nonce) throws IOException {</span> |
| <span class="source-line-no">3175</span><span id="line-3175"> return TraceUtil.trace(() -> {</span> |
| <span class="source-line-no">3176</span><span id="line-3176"> startRegionOperation(Operation.SCAN);</span> |
| <span class="source-line-no">3177</span><span id="line-3177"> try {</span> |
| <span class="source-line-no">3178</span><span id="line-3178"> // Verify families are all valid</span> |
| <span class="source-line-no">3179</span><span id="line-3179"> if (!scan.hasFamilies()) {</span> |
| <span class="source-line-no">3180</span><span id="line-3180"> // Adding all families to scanner</span> |
| <span class="source-line-no">3181</span><span id="line-3181"> for (byte[] family : this.htableDescriptor.getColumnFamilyNames()) {</span> |
| <span class="source-line-no">3182</span><span id="line-3182"> scan.addFamily(family);</span> |
| <span class="source-line-no">3183</span><span id="line-3183"> }</span> |
| <span class="source-line-no">3184</span><span id="line-3184"> } else {</span> |
| <span class="source-line-no">3185</span><span id="line-3185"> for (byte[] family : scan.getFamilyMap().keySet()) {</span> |
| <span class="source-line-no">3186</span><span id="line-3186"> checkFamily(family);</span> |
| <span class="source-line-no">3187</span><span id="line-3187"> }</span> |
| <span class="source-line-no">3188</span><span id="line-3188"> }</span> |
| <span class="source-line-no">3189</span><span id="line-3189"> return instantiateRegionScanner(scan, additionalScanners, nonceGroup, nonce);</span> |
| <span class="source-line-no">3190</span><span id="line-3190"> } finally {</span> |
| <span class="source-line-no">3191</span><span id="line-3191"> closeRegionOperation(Operation.SCAN);</span> |
| <span class="source-line-no">3192</span><span id="line-3192"> }</span> |
| <span class="source-line-no">3193</span><span id="line-3193"> }, () -> createRegionSpan("Region.getScanner"));</span> |
| <span class="source-line-no">3194</span><span id="line-3194"> }</span> |
| <span class="source-line-no">3195</span><span id="line-3195"></span> |
| <span class="source-line-no">3196</span><span id="line-3196"> protected RegionScannerImpl instantiateRegionScanner(Scan scan,</span> |
| <span class="source-line-no">3197</span><span id="line-3197"> List<KeyValueScanner> additionalScanners, long nonceGroup, long nonce) throws IOException {</span> |
| <span class="source-line-no">3198</span><span id="line-3198"> if (scan.isReversed()) {</span> |
| <span class="source-line-no">3199</span><span id="line-3199"> if (scan.getFilter() != null) {</span> |
| <span class="source-line-no">3200</span><span id="line-3200"> scan.getFilter().setReversed(true);</span> |
| <span class="source-line-no">3201</span><span id="line-3201"> }</span> |
| <span class="source-line-no">3202</span><span id="line-3202"> return new ReversedRegionScannerImpl(scan, additionalScanners, this, nonceGroup, nonce);</span> |
| <span class="source-line-no">3203</span><span id="line-3203"> }</span> |
| <span class="source-line-no">3204</span><span id="line-3204"> return new RegionScannerImpl(scan, additionalScanners, this, nonceGroup, nonce);</span> |
| <span class="source-line-no">3205</span><span id="line-3205"> }</span> |
| <span class="source-line-no">3206</span><span id="line-3206"></span> |
| <span class="source-line-no">3207</span><span id="line-3207"> /**</span> |
| <span class="source-line-no">3208</span><span id="line-3208"> * Prepare a delete for a row mutation processor</span> |
| <span class="source-line-no">3209</span><span id="line-3209"> * @param delete The passed delete is modified by this method. WARNING!</span> |
| <span class="source-line-no">3210</span><span id="line-3210"> */</span> |
| <span class="source-line-no">3211</span><span id="line-3211"> private void prepareDelete(Delete delete) throws IOException {</span> |
| <span class="source-line-no">3212</span><span id="line-3212"> // Check to see if this is a deleteRow insert</span> |
| <span class="source-line-no">3213</span><span id="line-3213"> if (delete.getFamilyCellMap().isEmpty()) {</span> |
| <span class="source-line-no">3214</span><span id="line-3214"> for (byte[] family : this.htableDescriptor.getColumnFamilyNames()) {</span> |
| <span class="source-line-no">3215</span><span id="line-3215"> // Don't eat the timestamp</span> |
| <span class="source-line-no">3216</span><span id="line-3216"> delete.addFamily(family, delete.getTimestamp());</span> |
| <span class="source-line-no">3217</span><span id="line-3217"> }</span> |
| <span class="source-line-no">3218</span><span id="line-3218"> } else {</span> |
| <span class="source-line-no">3219</span><span id="line-3219"> for (byte[] family : delete.getFamilyCellMap().keySet()) {</span> |
| <span class="source-line-no">3220</span><span id="line-3220"> if (family == null) {</span> |
| <span class="source-line-no">3221</span><span id="line-3221"> throw new NoSuchColumnFamilyException("Empty family is invalid");</span> |
| <span class="source-line-no">3222</span><span id="line-3222"> }</span> |
| <span class="source-line-no">3223</span><span id="line-3223"> checkFamily(family, delete.getDurability());</span> |
| <span class="source-line-no">3224</span><span id="line-3224"> }</span> |
| <span class="source-line-no">3225</span><span id="line-3225"> }</span> |
| <span class="source-line-no">3226</span><span id="line-3226"> }</span> |
| <span class="source-line-no">3227</span><span id="line-3227"></span> |
| <span class="source-line-no">3228</span><span id="line-3228"> @Override</span> |
| <span class="source-line-no">3229</span><span id="line-3229"> public void delete(Delete delete) throws IOException {</span> |
| <span class="source-line-no">3230</span><span id="line-3230"> TraceUtil.trace(() -> {</span> |
| <span class="source-line-no">3231</span><span id="line-3231"> checkReadOnly();</span> |
| <span class="source-line-no">3232</span><span id="line-3232"> checkResources();</span> |
| <span class="source-line-no">3233</span><span id="line-3233"> startRegionOperation(Operation.DELETE);</span> |
| <span class="source-line-no">3234</span><span id="line-3234"> try {</span> |
| <span class="source-line-no">3235</span><span id="line-3235"> // All edits for the given row (across all column families) must happen atomically.</span> |
| <span class="source-line-no">3236</span><span id="line-3236"> return mutate(delete);</span> |
| <span class="source-line-no">3237</span><span id="line-3237"> } finally {</span> |
| <span class="source-line-no">3238</span><span id="line-3238"> closeRegionOperation(Operation.DELETE);</span> |
| <span class="source-line-no">3239</span><span id="line-3239"> }</span> |
| <span class="source-line-no">3240</span><span id="line-3240"> }, () -> createRegionSpan("Region.delete"));</span> |
| <span class="source-line-no">3241</span><span id="line-3241"> }</span> |
| <span class="source-line-no">3242</span><span id="line-3242"></span> |
| <span class="source-line-no">3243</span><span id="line-3243"> /**</span> |
| <span class="source-line-no">3244</span><span id="line-3244"> * Set up correct timestamps in the KVs in Delete object.</span> |
| <span class="source-line-no">3245</span><span id="line-3245"> * <p/></span> |
| <span class="source-line-no">3246</span><span id="line-3246"> * Caller should have the row and region locks.</span> |
| <span class="source-line-no">3247</span><span id="line-3247"> */</span> |
| <span class="source-line-no">3248</span><span id="line-3248"> private void prepareDeleteTimestamps(Mutation mutation, Map<byte[], List<ExtendedCell>> familyMap,</span> |
| <span class="source-line-no">3249</span><span id="line-3249"> byte[] byteNow) throws IOException {</span> |
| <span class="source-line-no">3250</span><span id="line-3250"> for (Map.Entry<byte[], List<ExtendedCell>> e : familyMap.entrySet()) {</span> |
| <span class="source-line-no">3251</span><span id="line-3251"></span> |
| <span class="source-line-no">3252</span><span id="line-3252"> byte[] family = e.getKey();</span> |
| <span class="source-line-no">3253</span><span id="line-3253"> List<ExtendedCell> cells = e.getValue();</span> |
| <span class="source-line-no">3254</span><span id="line-3254"> assert cells instanceof RandomAccess;</span> |
| <span class="source-line-no">3255</span><span id="line-3255"></span> |
| <span class="source-line-no">3256</span><span id="line-3256"> Map<byte[], Integer> kvCount = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">3257</span><span id="line-3257"> int listSize = cells.size();</span> |
| <span class="source-line-no">3258</span><span id="line-3258"> for (int i = 0; i < listSize; i++) {</span> |
| <span class="source-line-no">3259</span><span id="line-3259"> ExtendedCell cell = cells.get(i);</span> |
| <span class="source-line-no">3260</span><span id="line-3260"> // Check if time is LATEST, change to time of most recent addition if so</span> |
| <span class="source-line-no">3261</span><span id="line-3261"> // This is expensive.</span> |
| <span class="source-line-no">3262</span><span id="line-3262"> if (</span> |
| <span class="source-line-no">3263</span><span id="line-3263"> cell.getTimestamp() == HConstants.LATEST_TIMESTAMP && PrivateCellUtil.isDeleteType(cell)</span> |
| <span class="source-line-no">3264</span><span id="line-3264"> ) {</span> |
| <span class="source-line-no">3265</span><span id="line-3265"> byte[] qual = CellUtil.cloneQualifier(cell);</span> |
| <span class="source-line-no">3266</span><span id="line-3266"></span> |
| <span class="source-line-no">3267</span><span id="line-3267"> Integer count = kvCount.get(qual);</span> |
| <span class="source-line-no">3268</span><span id="line-3268"> if (count == null) {</span> |
| <span class="source-line-no">3269</span><span id="line-3269"> kvCount.put(qual, 1);</span> |
| <span class="source-line-no">3270</span><span id="line-3270"> } else {</span> |
| <span class="source-line-no">3271</span><span id="line-3271"> kvCount.put(qual, count + 1);</span> |
| <span class="source-line-no">3272</span><span id="line-3272"> }</span> |
| <span class="source-line-no">3273</span><span id="line-3273"> count = kvCount.get(qual);</span> |
| <span class="source-line-no">3274</span><span id="line-3274"></span> |
| <span class="source-line-no">3275</span><span id="line-3275"> Get get = new Get(CellUtil.cloneRow(cell));</span> |
| <span class="source-line-no">3276</span><span id="line-3276"> get.readVersions(count);</span> |
| <span class="source-line-no">3277</span><span id="line-3277"> get.addColumn(family, qual);</span> |
| <span class="source-line-no">3278</span><span id="line-3278"> if (coprocessorHost != null) {</span> |
| <span class="source-line-no">3279</span><span id="line-3279"> if (</span> |
| <span class="source-line-no">3280</span><span id="line-3280"> !coprocessorHost.prePrepareTimeStampForDeleteVersion(mutation, cell, byteNow, get)</span> |
| <span class="source-line-no">3281</span><span id="line-3281"> ) {</span> |
| <span class="source-line-no">3282</span><span id="line-3282"> updateDeleteLatestVersionTimestamp(cell, get, count, byteNow);</span> |
| <span class="source-line-no">3283</span><span id="line-3283"> }</span> |
| <span class="source-line-no">3284</span><span id="line-3284"> } else {</span> |
| <span class="source-line-no">3285</span><span id="line-3285"> updateDeleteLatestVersionTimestamp(cell, get, count, byteNow);</span> |
| <span class="source-line-no">3286</span><span id="line-3286"> }</span> |
| <span class="source-line-no">3287</span><span id="line-3287"> } else {</span> |
| <span class="source-line-no">3288</span><span id="line-3288"> PrivateCellUtil.updateLatestStamp(cell, byteNow);</span> |
| <span class="source-line-no">3289</span><span id="line-3289"> }</span> |
| <span class="source-line-no">3290</span><span id="line-3290"> }</span> |
| <span class="source-line-no">3291</span><span id="line-3291"> }</span> |
| <span class="source-line-no">3292</span><span id="line-3292"> }</span> |
| <span class="source-line-no">3293</span><span id="line-3293"></span> |
| <span class="source-line-no">3294</span><span id="line-3294"> private void updateDeleteLatestVersionTimestamp(Cell cell, Get get, int count, byte[] byteNow)</span> |
| <span class="source-line-no">3295</span><span id="line-3295"> throws IOException {</span> |
| <span class="source-line-no">3296</span><span id="line-3296"> try (RegionScanner scanner = getScanner(new Scan(get))) {</span> |
| <span class="source-line-no">3297</span><span id="line-3297"> // NOTE: Please don't use HRegion.get() instead,</span> |
| <span class="source-line-no">3298</span><span id="line-3298"> // because it will copy cells to heap. See HBASE-26036</span> |
| <span class="source-line-no">3299</span><span id="line-3299"> List<Cell> result = new ArrayList<>();</span> |
| <span class="source-line-no">3300</span><span id="line-3300"> scanner.next(result);</span> |
| <span class="source-line-no">3301</span><span id="line-3301"></span> |
| <span class="source-line-no">3302</span><span id="line-3302"> if (result.size() < count) {</span> |
| <span class="source-line-no">3303</span><span id="line-3303"> // Nothing to delete</span> |
| <span class="source-line-no">3304</span><span id="line-3304"> PrivateCellUtil.updateLatestStamp(cell, byteNow);</span> |
| <span class="source-line-no">3305</span><span id="line-3305"> return;</span> |
| <span class="source-line-no">3306</span><span id="line-3306"> }</span> |
| <span class="source-line-no">3307</span><span id="line-3307"> if (result.size() > count) {</span> |
| <span class="source-line-no">3308</span><span id="line-3308"> throw new RuntimeException("Unexpected size: " + result.size());</span> |
| <span class="source-line-no">3309</span><span id="line-3309"> }</span> |
| <span class="source-line-no">3310</span><span id="line-3310"> Cell getCell = result.get(count - 1);</span> |
| <span class="source-line-no">3311</span><span id="line-3311"> PrivateCellUtil.setTimestamp(cell, getCell.getTimestamp());</span> |
| <span class="source-line-no">3312</span><span id="line-3312"> }</span> |
| <span class="source-line-no">3313</span><span id="line-3313"> }</span> |
| <span class="source-line-no">3314</span><span id="line-3314"></span> |
| <span class="source-line-no">3315</span><span id="line-3315"> @Override</span> |
| <span class="source-line-no">3316</span><span id="line-3316"> public void put(Put put) throws IOException {</span> |
| <span class="source-line-no">3317</span><span id="line-3317"> TraceUtil.trace(() -> {</span> |
| <span class="source-line-no">3318</span><span id="line-3318"> checkReadOnly();</span> |
| <span class="source-line-no">3319</span><span id="line-3319"></span> |
| <span class="source-line-no">3320</span><span id="line-3320"> // Do a rough check that we have resources to accept a write. The check is</span> |
| <span class="source-line-no">3321</span><span id="line-3321"> // 'rough' in that between the resource check and the call to obtain a</span> |
| <span class="source-line-no">3322</span><span id="line-3322"> // read lock, resources may run out. For now, the thought is that this</span> |
| <span class="source-line-no">3323</span><span id="line-3323"> // will be extremely rare; we'll deal with it when it happens.</span> |
| <span class="source-line-no">3324</span><span id="line-3324"> checkResources();</span> |
| <span class="source-line-no">3325</span><span id="line-3325"> startRegionOperation(Operation.PUT);</span> |
| <span class="source-line-no">3326</span><span id="line-3326"> try {</span> |
| <span class="source-line-no">3327</span><span id="line-3327"> // All edits for the given row (across all column families) must happen atomically.</span> |
| <span class="source-line-no">3328</span><span id="line-3328"> return mutate(put);</span> |
| <span class="source-line-no">3329</span><span id="line-3329"> } finally {</span> |
| <span class="source-line-no">3330</span><span id="line-3330"> closeRegionOperation(Operation.PUT);</span> |
| <span class="source-line-no">3331</span><span id="line-3331"> }</span> |
| <span class="source-line-no">3332</span><span id="line-3332"> }, () -> createRegionSpan("Region.put"));</span> |
| <span class="source-line-no">3333</span><span id="line-3333"> }</span> |
| <span class="source-line-no">3334</span><span id="line-3334"></span> |
| <span class="source-line-no">3335</span><span id="line-3335"> /**</span> |
| <span class="source-line-no">3336</span><span id="line-3336"> * Class that tracks the progress of a batch operations, accumulating status codes and tracking</span> |
| <span class="source-line-no">3337</span><span id="line-3337"> * the index at which processing is proceeding. These batch operations may get split into</span> |
| <span class="source-line-no">3338</span><span id="line-3338"> * mini-batches for processing.</span> |
| <span class="source-line-no">3339</span><span id="line-3339"> */</span> |
| <span class="source-line-no">3340</span><span id="line-3340"> private abstract static class BatchOperation<T> {</span> |
| <span class="source-line-no">3341</span><span id="line-3341"> protected final T[] operations;</span> |
| <span class="source-line-no">3342</span><span id="line-3342"> protected final OperationStatus[] retCodeDetails;</span> |
| <span class="source-line-no">3343</span><span id="line-3343"> protected final WALEdit[] walEditsFromCoprocessors;</span> |
| <span class="source-line-no">3344</span><span id="line-3344"> // reference family cell maps directly so coprocessors can mutate them if desired</span> |
| <span class="source-line-no">3345</span><span id="line-3345"> protected final Map<byte[], List<ExtendedCell>>[] familyCellMaps;</span> |
| <span class="source-line-no">3346</span><span id="line-3346"> // For Increment/Append operations</span> |
| <span class="source-line-no">3347</span><span id="line-3347"> protected final Result[] results;</span> |
| <span class="source-line-no">3348</span><span id="line-3348"></span> |
| <span class="source-line-no">3349</span><span id="line-3349"> protected final HRegion region;</span> |
| <span class="source-line-no">3350</span><span id="line-3350"> protected int nextIndexToProcess = 0;</span> |
| <span class="source-line-no">3351</span><span id="line-3351"> protected final ObservedExceptionsInBatch observedExceptions;</span> |
| <span class="source-line-no">3352</span><span id="line-3352"> // Durability of the batch (highest durability of all operations)</span> |
| <span class="source-line-no">3353</span><span id="line-3353"> protected Durability durability;</span> |
| <span class="source-line-no">3354</span><span id="line-3354"> protected boolean atomic = false;</span> |
| <span class="source-line-no">3355</span><span id="line-3355"></span> |
| <span class="source-line-no">3356</span><span id="line-3356"> public BatchOperation(final HRegion region, T[] operations) {</span> |
| <span class="source-line-no">3357</span><span id="line-3357"> this.operations = operations;</span> |
| <span class="source-line-no">3358</span><span id="line-3358"> this.retCodeDetails = new OperationStatus[operations.length];</span> |
| <span class="source-line-no">3359</span><span id="line-3359"> Arrays.fill(this.retCodeDetails, OperationStatus.NOT_RUN);</span> |
| <span class="source-line-no">3360</span><span id="line-3360"> this.walEditsFromCoprocessors = new WALEdit[operations.length];</span> |
| <span class="source-line-no">3361</span><span id="line-3361"> familyCellMaps = new Map[operations.length];</span> |
| <span class="source-line-no">3362</span><span id="line-3362"> this.results = new Result[operations.length];</span> |
| <span class="source-line-no">3363</span><span id="line-3363"></span> |
| <span class="source-line-no">3364</span><span id="line-3364"> this.region = region;</span> |
| <span class="source-line-no">3365</span><span id="line-3365"> observedExceptions = new ObservedExceptionsInBatch();</span> |
| <span class="source-line-no">3366</span><span id="line-3366"> durability = Durability.USE_DEFAULT;</span> |
| <span class="source-line-no">3367</span><span id="line-3367"> }</span> |
| <span class="source-line-no">3368</span><span id="line-3368"></span> |
| <span class="source-line-no">3369</span><span id="line-3369"> /**</span> |
| <span class="source-line-no">3370</span><span id="line-3370"> * Visitor interface for batch operations</span> |
| <span class="source-line-no">3371</span><span id="line-3371"> */</span> |
| <span class="source-line-no">3372</span><span id="line-3372"> @FunctionalInterface</span> |
| <span class="source-line-no">3373</span><span id="line-3373"> interface Visitor {</span> |
| <span class="source-line-no">3374</span><span id="line-3374"> /**</span> |
| <span class="source-line-no">3375</span><span id="line-3375"> * @param index operation index</span> |
| <span class="source-line-no">3376</span><span id="line-3376"> * @return If true continue visiting remaining entries, break otherwise</span> |
| <span class="source-line-no">3377</span><span id="line-3377"> */</span> |
| <span class="source-line-no">3378</span><span id="line-3378"> boolean visit(int index) throws IOException;</span> |
| <span class="source-line-no">3379</span><span id="line-3379"> }</span> |
| <span class="source-line-no">3380</span><span id="line-3380"></span> |
| <span class="source-line-no">3381</span><span id="line-3381"> /**</span> |
| <span class="source-line-no">3382</span><span id="line-3382"> * Helper method for visiting pending/ all batch operations</span> |
| <span class="source-line-no">3383</span><span id="line-3383"> */</span> |
| <span class="source-line-no">3384</span><span id="line-3384"> public void visitBatchOperations(boolean pendingOnly, int lastIndexExclusive, Visitor visitor)</span> |
| <span class="source-line-no">3385</span><span id="line-3385"> throws IOException {</span> |
| <span class="source-line-no">3386</span><span id="line-3386"> assert lastIndexExclusive <= this.size();</span> |
| <span class="source-line-no">3387</span><span id="line-3387"> for (int i = nextIndexToProcess; i < lastIndexExclusive; i++) {</span> |
| <span class="source-line-no">3388</span><span id="line-3388"> if (!pendingOnly || isOperationPending(i)) {</span> |
| <span class="source-line-no">3389</span><span id="line-3389"> if (!visitor.visit(i)) {</span> |
| <span class="source-line-no">3390</span><span id="line-3390"> break;</span> |
| <span class="source-line-no">3391</span><span id="line-3391"> }</span> |
| <span class="source-line-no">3392</span><span id="line-3392"> }</span> |
| <span class="source-line-no">3393</span><span id="line-3393"> }</span> |
| <span class="source-line-no">3394</span><span id="line-3394"> }</span> |
| <span class="source-line-no">3395</span><span id="line-3395"></span> |
| <span class="source-line-no">3396</span><span id="line-3396"> public abstract Mutation getMutation(int index);</span> |
| <span class="source-line-no">3397</span><span id="line-3397"></span> |
| <span class="source-line-no">3398</span><span id="line-3398"> public abstract long getNonceGroup(int index);</span> |
| <span class="source-line-no">3399</span><span id="line-3399"></span> |
| <span class="source-line-no">3400</span><span id="line-3400"> public abstract long getNonce(int index);</span> |
| <span class="source-line-no">3401</span><span id="line-3401"></span> |
| <span class="source-line-no">3402</span><span id="line-3402"> /**</span> |
| <span class="source-line-no">3403</span><span id="line-3403"> * This method is potentially expensive and useful mostly for non-replay CP path.</span> |
| <span class="source-line-no">3404</span><span id="line-3404"> */</span> |
| <span class="source-line-no">3405</span><span id="line-3405"> public abstract Mutation[] getMutationsForCoprocs();</span> |
| <span class="source-line-no">3406</span><span id="line-3406"></span> |
| <span class="source-line-no">3407</span><span id="line-3407"> public abstract boolean isInReplay();</span> |
| <span class="source-line-no">3408</span><span id="line-3408"></span> |
| <span class="source-line-no">3409</span><span id="line-3409"> public abstract long getOrigLogSeqNum();</span> |
| <span class="source-line-no">3410</span><span id="line-3410"></span> |
| <span class="source-line-no">3411</span><span id="line-3411"> public abstract void startRegionOperation() throws IOException;</span> |
| <span class="source-line-no">3412</span><span id="line-3412"></span> |
| <span class="source-line-no">3413</span><span id="line-3413"> public abstract void closeRegionOperation() throws IOException;</span> |
| <span class="source-line-no">3414</span><span id="line-3414"></span> |
| <span class="source-line-no">3415</span><span id="line-3415"> /**</span> |
| <span class="source-line-no">3416</span><span id="line-3416"> * Validates each mutation and prepares a batch for write. If necessary (non-replay case), runs</span> |
| <span class="source-line-no">3417</span><span id="line-3417"> * CP prePut()/preDelete()/preIncrement()/preAppend() hooks for all mutations in a batch. This</span> |
| <span class="source-line-no">3418</span><span id="line-3418"> * is intended to operate on entire batch and will be called from outside of class to check and</span> |
| <span class="source-line-no">3419</span><span id="line-3419"> * prepare batch. This can be implemented by calling helper method</span> |
| <span class="source-line-no">3420</span><span id="line-3420"> * {@link #checkAndPrepareMutation(int, long)} in a 'for' loop over mutations.</span> |
| <span class="source-line-no">3421</span><span id="line-3421"> */</span> |
| <span class="source-line-no">3422</span><span id="line-3422"> public abstract void checkAndPrepare() throws IOException;</span> |
| <span class="source-line-no">3423</span><span id="line-3423"></span> |
| <span class="source-line-no">3424</span><span id="line-3424"> /**</span> |
| <span class="source-line-no">3425</span><span id="line-3425"> * Implement any Put request specific check and prepare logic here. Please refer to</span> |
| <span class="source-line-no">3426</span><span id="line-3426"> * {@link #checkAndPrepareMutation(Mutation, long)} for how its used.</span> |
| <span class="source-line-no">3427</span><span id="line-3427"> */</span> |
| <span class="source-line-no">3428</span><span id="line-3428"> protected abstract void checkAndPreparePut(final Put p) throws IOException;</span> |
| <span class="source-line-no">3429</span><span id="line-3429"></span> |
| <span class="source-line-no">3430</span><span id="line-3430"> /**</span> |
| <span class="source-line-no">3431</span><span id="line-3431"> * If necessary, calls preBatchMutate() CP hook for a mini-batch and updates metrics, cell</span> |
| <span class="source-line-no">3432</span><span id="line-3432"> * count, tags and timestamp for all cells of all operations in a mini-batch.</span> |
| <span class="source-line-no">3433</span><span id="line-3433"> */</span> |
| <span class="source-line-no">3434</span><span id="line-3434"> public abstract void prepareMiniBatchOperations(</span> |
| <span class="source-line-no">3435</span><span id="line-3435"> MiniBatchOperationInProgress<Mutation> miniBatchOp, long timestamp,</span> |
| <span class="source-line-no">3436</span><span id="line-3436"> final List<RowLock> acquiredRowLocks) throws IOException;</span> |
| <span class="source-line-no">3437</span><span id="line-3437"></span> |
| <span class="source-line-no">3438</span><span id="line-3438"> /**</span> |
| <span class="source-line-no">3439</span><span id="line-3439"> * Write mini-batch operations to MemStore</span> |
| <span class="source-line-no">3440</span><span id="line-3440"> */</span> |
| <span class="source-line-no">3441</span><span id="line-3441"> public abstract WriteEntry writeMiniBatchOperationsToMemStore(</span> |
| <span class="source-line-no">3442</span><span id="line-3442"> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final WriteEntry writeEntry,</span> |
| <span class="source-line-no">3443</span><span id="line-3443"> long now) throws IOException;</span> |
| <span class="source-line-no">3444</span><span id="line-3444"></span> |
| <span class="source-line-no">3445</span><span id="line-3445"> protected void writeMiniBatchOperationsToMemStore(</span> |
| <span class="source-line-no">3446</span><span id="line-3446"> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final long writeNumber)</span> |
| <span class="source-line-no">3447</span><span id="line-3447"> throws IOException {</span> |
| <span class="source-line-no">3448</span><span id="line-3448"> MemStoreSizing memStoreAccounting = new NonThreadSafeMemStoreSizing();</span> |
| <span class="source-line-no">3449</span><span id="line-3449"> visitBatchOperations(true, miniBatchOp.getLastIndexExclusive(), (int index) -> {</span> |
| <span class="source-line-no">3450</span><span id="line-3450"> // We need to update the sequence id for following reasons.</span> |
| <span class="source-line-no">3451</span><span id="line-3451"> // 1) If the op is in replay mode, FSWALEntry#stampRegionSequenceId won't stamp sequence id.</span> |
| <span class="source-line-no">3452</span><span id="line-3452"> // 2) If no WAL, FSWALEntry won't be used</span> |
| <span class="source-line-no">3453</span><span id="line-3453"> // we use durability of the original mutation for the mutation passed by CP.</span> |
| <span class="source-line-no">3454</span><span id="line-3454"> if (isInReplay() || getMutation(index).getDurability() == Durability.SKIP_WAL) {</span> |
| <span class="source-line-no">3455</span><span id="line-3455"> region.updateSequenceId(familyCellMaps[index].values(), writeNumber);</span> |
| <span class="source-line-no">3456</span><span id="line-3456"> }</span> |
| <span class="source-line-no">3457</span><span id="line-3457"> applyFamilyMapToMemStore(familyCellMaps[index], memStoreAccounting);</span> |
| <span class="source-line-no">3458</span><span id="line-3458"> return true;</span> |
| <span class="source-line-no">3459</span><span id="line-3459"> });</span> |
| <span class="source-line-no">3460</span><span id="line-3460"> // update memStore size</span> |
| <span class="source-line-no">3461</span><span id="line-3461"> region.incMemStoreSize(memStoreAccounting.getDataSize(), memStoreAccounting.getHeapSize(),</span> |
| <span class="source-line-no">3462</span><span id="line-3462"> memStoreAccounting.getOffHeapSize(), memStoreAccounting.getCellsCount());</span> |
| <span class="source-line-no">3463</span><span id="line-3463"> }</span> |
| <span class="source-line-no">3464</span><span id="line-3464"></span> |
| <span class="source-line-no">3465</span><span id="line-3465"> public boolean isDone() {</span> |
| <span class="source-line-no">3466</span><span id="line-3466"> return nextIndexToProcess == operations.length;</span> |
| <span class="source-line-no">3467</span><span id="line-3467"> }</span> |
| <span class="source-line-no">3468</span><span id="line-3468"></span> |
| <span class="source-line-no">3469</span><span id="line-3469"> public int size() {</span> |
| <span class="source-line-no">3470</span><span id="line-3470"> return operations.length;</span> |
| <span class="source-line-no">3471</span><span id="line-3471"> }</span> |
| <span class="source-line-no">3472</span><span id="line-3472"></span> |
| <span class="source-line-no">3473</span><span id="line-3473"> public boolean isOperationPending(int index) {</span> |
| <span class="source-line-no">3474</span><span id="line-3474"> return retCodeDetails[index].getOperationStatusCode() == OperationStatusCode.NOT_RUN;</span> |
| <span class="source-line-no">3475</span><span id="line-3475"> }</span> |
| <span class="source-line-no">3476</span><span id="line-3476"></span> |
| <span class="source-line-no">3477</span><span id="line-3477"> public List<UUID> getClusterIds() {</span> |
| <span class="source-line-no">3478</span><span id="line-3478"> assert size() != 0;</span> |
| <span class="source-line-no">3479</span><span id="line-3479"> return getMutation(0).getClusterIds();</span> |
| <span class="source-line-no">3480</span><span id="line-3480"> }</span> |
| <span class="source-line-no">3481</span><span id="line-3481"></span> |
| <span class="source-line-no">3482</span><span id="line-3482"> boolean isAtomic() {</span> |
| <span class="source-line-no">3483</span><span id="line-3483"> return atomic;</span> |
| <span class="source-line-no">3484</span><span id="line-3484"> }</span> |
| <span class="source-line-no">3485</span><span id="line-3485"></span> |
| <span class="source-line-no">3486</span><span id="line-3486"> /**</span> |
| <span class="source-line-no">3487</span><span id="line-3487"> * Helper method that checks and prepares only one mutation. This can be used to implement</span> |
| <span class="source-line-no">3488</span><span id="line-3488"> * {@link #checkAndPrepare()} for entire Batch. NOTE: As CP</span> |
| <span class="source-line-no">3489</span><span id="line-3489"> * prePut()/preDelete()/preIncrement()/preAppend() hooks may modify mutations, this method</span> |
| <span class="source-line-no">3490</span><span id="line-3490"> * should be called after prePut()/preDelete()/preIncrement()/preAppend() CP hooks are run for</span> |
| <span class="source-line-no">3491</span><span id="line-3491"> * the mutation</span> |
| <span class="source-line-no">3492</span><span id="line-3492"> */</span> |
| <span class="source-line-no">3493</span><span id="line-3493"> protected void checkAndPrepareMutation(Mutation mutation, final long timestamp)</span> |
| <span class="source-line-no">3494</span><span id="line-3494"> throws IOException {</span> |
| <span class="source-line-no">3495</span><span id="line-3495"> region.checkRow(mutation.getRow(), "batchMutate");</span> |
| <span class="source-line-no">3496</span><span id="line-3496"> if (mutation instanceof Put) {</span> |
| <span class="source-line-no">3497</span><span id="line-3497"> // Check the families in the put. If bad, skip this one.</span> |
| <span class="source-line-no">3498</span><span id="line-3498"> checkAndPreparePut((Put) mutation);</span> |
| <span class="source-line-no">3499</span><span id="line-3499"> region.checkTimestamps(mutation.getFamilyCellMap(), timestamp);</span> |
| <span class="source-line-no">3500</span><span id="line-3500"> } else if (mutation instanceof Delete) {</span> |
| <span class="source-line-no">3501</span><span id="line-3501"> region.prepareDelete((Delete) mutation);</span> |
| <span class="source-line-no">3502</span><span id="line-3502"> } else if (mutation instanceof Increment || mutation instanceof Append) {</span> |
| <span class="source-line-no">3503</span><span id="line-3503"> region.checkFamilies(mutation.getFamilyCellMap().keySet(), mutation.getDurability());</span> |
| <span class="source-line-no">3504</span><span id="line-3504"> }</span> |
| <span class="source-line-no">3505</span><span id="line-3505"> }</span> |
| <span class="source-line-no">3506</span><span id="line-3506"></span> |
| <span class="source-line-no">3507</span><span id="line-3507"> protected void checkAndPrepareMutation(int index, long timestamp) throws IOException {</span> |
| <span class="source-line-no">3508</span><span id="line-3508"> Mutation mutation = getMutation(index);</span> |
| <span class="source-line-no">3509</span><span id="line-3509"> try {</span> |
| <span class="source-line-no">3510</span><span id="line-3510"> this.checkAndPrepareMutation(mutation, timestamp);</span> |
| <span class="source-line-no">3511</span><span id="line-3511"></span> |
| <span class="source-line-no">3512</span><span id="line-3512"> if (mutation instanceof Put || mutation instanceof Delete) {</span> |
| <span class="source-line-no">3513</span><span id="line-3513"> // store the family map reference to allow for mutations</span> |
| <span class="source-line-no">3514</span><span id="line-3514"> // we know that in mutation, only ExtendedCells are allow so here we do a fake cast, to</span> |
| <span class="source-line-no">3515</span><span id="line-3515"> // simplify later logic</span> |
| <span class="source-line-no">3516</span><span id="line-3516"> familyCellMaps[index] = PackagePrivateFieldAccessor.getExtendedFamilyCellMap(mutation);</span> |
| <span class="source-line-no">3517</span><span id="line-3517"> }</span> |
| <span class="source-line-no">3518</span><span id="line-3518"></span> |
| <span class="source-line-no">3519</span><span id="line-3519"> // store durability for the batch (highest durability of all operations in the batch)</span> |
| <span class="source-line-no">3520</span><span id="line-3520"> Durability tmpDur = region.getEffectiveDurability(mutation.getDurability());</span> |
| <span class="source-line-no">3521</span><span id="line-3521"> if (tmpDur.ordinal() > durability.ordinal()) {</span> |
| <span class="source-line-no">3522</span><span id="line-3522"> durability = tmpDur;</span> |
| <span class="source-line-no">3523</span><span id="line-3523"> }</span> |
| <span class="source-line-no">3524</span><span id="line-3524"> } catch (NoSuchColumnFamilyException nscfe) {</span> |
| <span class="source-line-no">3525</span><span id="line-3525"> final String msg = "No such column family in batch mutation in region " + this;</span> |
| <span class="source-line-no">3526</span><span id="line-3526"> if (observedExceptions.hasSeenNoSuchFamily()) {</span> |
| <span class="source-line-no">3527</span><span id="line-3527"> LOG.warn(msg + nscfe.getMessage());</span> |
| <span class="source-line-no">3528</span><span id="line-3528"> } else {</span> |
| <span class="source-line-no">3529</span><span id="line-3529"> LOG.warn(msg, nscfe);</span> |
| <span class="source-line-no">3530</span><span id="line-3530"> observedExceptions.sawNoSuchFamily();</span> |
| <span class="source-line-no">3531</span><span id="line-3531"> }</span> |
| <span class="source-line-no">3532</span><span id="line-3532"> retCodeDetails[index] =</span> |
| <span class="source-line-no">3533</span><span id="line-3533"> new OperationStatus(OperationStatusCode.BAD_FAMILY, nscfe.getMessage());</span> |
| <span class="source-line-no">3534</span><span id="line-3534"> if (isAtomic()) { // fail, atomic means all or none</span> |
| <span class="source-line-no">3535</span><span id="line-3535"> throw nscfe;</span> |
| <span class="source-line-no">3536</span><span id="line-3536"> }</span> |
| <span class="source-line-no">3537</span><span id="line-3537"> } catch (FailedSanityCheckException fsce) {</span> |
| <span class="source-line-no">3538</span><span id="line-3538"> final String msg = "Batch Mutation did not pass sanity check in region " + this;</span> |
| <span class="source-line-no">3539</span><span id="line-3539"> if (observedExceptions.hasSeenFailedSanityCheck()) {</span> |
| <span class="source-line-no">3540</span><span id="line-3540"> LOG.warn(msg + fsce.getMessage());</span> |
| <span class="source-line-no">3541</span><span id="line-3541"> } else {</span> |
| <span class="source-line-no">3542</span><span id="line-3542"> LOG.warn(msg, fsce);</span> |
| <span class="source-line-no">3543</span><span id="line-3543"> observedExceptions.sawFailedSanityCheck();</span> |
| <span class="source-line-no">3544</span><span id="line-3544"> }</span> |
| <span class="source-line-no">3545</span><span id="line-3545"> retCodeDetails[index] =</span> |
| <span class="source-line-no">3546</span><span id="line-3546"> new OperationStatus(OperationStatusCode.SANITY_CHECK_FAILURE, fsce.getMessage());</span> |
| <span class="source-line-no">3547</span><span id="line-3547"> if (isAtomic()) {</span> |
| <span class="source-line-no">3548</span><span id="line-3548"> throw fsce;</span> |
| <span class="source-line-no">3549</span><span id="line-3549"> }</span> |
| <span class="source-line-no">3550</span><span id="line-3550"> } catch (WrongRegionException we) {</span> |
| <span class="source-line-no">3551</span><span id="line-3551"> final String msg = "Batch mutation had a row that does not belong to this region " + this;</span> |
| <span class="source-line-no">3552</span><span id="line-3552"> if (observedExceptions.hasSeenWrongRegion()) {</span> |
| <span class="source-line-no">3553</span><span id="line-3553"> LOG.warn(msg + we.getMessage());</span> |
| <span class="source-line-no">3554</span><span id="line-3554"> } else {</span> |
| <span class="source-line-no">3555</span><span id="line-3555"> LOG.warn(msg, we);</span> |
| <span class="source-line-no">3556</span><span id="line-3556"> observedExceptions.sawWrongRegion();</span> |
| <span class="source-line-no">3557</span><span id="line-3557"> }</span> |
| <span class="source-line-no">3558</span><span id="line-3558"> retCodeDetails[index] =</span> |
| <span class="source-line-no">3559</span><span id="line-3559"> new OperationStatus(OperationStatusCode.SANITY_CHECK_FAILURE, we.getMessage());</span> |
| <span class="source-line-no">3560</span><span id="line-3560"> if (isAtomic()) {</span> |
| <span class="source-line-no">3561</span><span id="line-3561"> throw we;</span> |
| <span class="source-line-no">3562</span><span id="line-3562"> }</span> |
| <span class="source-line-no">3563</span><span id="line-3563"> }</span> |
| <span class="source-line-no">3564</span><span id="line-3564"> }</span> |
| <span class="source-line-no">3565</span><span id="line-3565"></span> |
| <span class="source-line-no">3566</span><span id="line-3566"> /**</span> |
| <span class="source-line-no">3567</span><span id="line-3567"> * Creates Mini-batch of all operations [nextIndexToProcess, lastIndexExclusive) for which a row</span> |
| <span class="source-line-no">3568</span><span id="line-3568"> * lock can be acquired. All mutations with locked rows are considered to be In-progress</span> |
| <span class="source-line-no">3569</span><span id="line-3569"> * operations and hence the name {@link MiniBatchOperationInProgress}. Mini batch is window over</span> |
| <span class="source-line-no">3570</span><span id="line-3570"> * {@link BatchOperation} and contains contiguous pending operations.</span> |
| <span class="source-line-no">3571</span><span id="line-3571"> * @param acquiredRowLocks keeps track of rowLocks acquired.</span> |
| <span class="source-line-no">3572</span><span id="line-3572"> */</span> |
| <span class="source-line-no">3573</span><span id="line-3573"> public MiniBatchOperationInProgress<Mutation></span> |
| <span class="source-line-no">3574</span><span id="line-3574"> lockRowsAndBuildMiniBatch(List<RowLock> acquiredRowLocks) throws IOException {</span> |
| <span class="source-line-no">3575</span><span id="line-3575"> int readyToWriteCount = 0;</span> |
| <span class="source-line-no">3576</span><span id="line-3576"> int lastIndexExclusive = 0;</span> |
| <span class="source-line-no">3577</span><span id="line-3577"> RowLock prevRowLock = null;</span> |
| <span class="source-line-no">3578</span><span id="line-3578"> for (; lastIndexExclusive < size(); lastIndexExclusive++) {</span> |
| <span class="source-line-no">3579</span><span id="line-3579"> // It reaches the miniBatchSize, stop here and process the miniBatch</span> |
| <span class="source-line-no">3580</span><span id="line-3580"> // This only applies to non-atomic batch operations.</span> |
| <span class="source-line-no">3581</span><span id="line-3581"> if (!isAtomic() && (readyToWriteCount == region.miniBatchSize)) {</span> |
| <span class="source-line-no">3582</span><span id="line-3582"> break;</span> |
| <span class="source-line-no">3583</span><span id="line-3583"> }</span> |
| <span class="source-line-no">3584</span><span id="line-3584"></span> |
| <span class="source-line-no">3585</span><span id="line-3585"> if (!isOperationPending(lastIndexExclusive)) {</span> |
| <span class="source-line-no">3586</span><span id="line-3586"> continue;</span> |
| <span class="source-line-no">3587</span><span id="line-3587"> }</span> |
| <span class="source-line-no">3588</span><span id="line-3588"></span> |
| <span class="source-line-no">3589</span><span id="line-3589"> // HBASE-19389 Limit concurrency of put with dense (hundreds) columns to avoid exhausting</span> |
| <span class="source-line-no">3590</span><span id="line-3590"> // RS handlers, covering both MutationBatchOperation and ReplayBatchOperation</span> |
| <span class="source-line-no">3591</span><span id="line-3591"> // The BAD_FAMILY/SANITY_CHECK_FAILURE cases are handled in checkAndPrepare phase and won't</span> |
| <span class="source-line-no">3592</span><span id="line-3592"> // pass the isOperationPending check</span> |
| <span class="source-line-no">3593</span><span id="line-3593"> Map<byte[], List<Cell>> curFamilyCellMap =</span> |
| <span class="source-line-no">3594</span><span id="line-3594"> getMutation(lastIndexExclusive).getFamilyCellMap();</span> |
| <span class="source-line-no">3595</span><span id="line-3595"> try {</span> |
| <span class="source-line-no">3596</span><span id="line-3596"> // start the protector before acquiring row lock considering performance, and will finish</span> |
| <span class="source-line-no">3597</span><span id="line-3597"> // it when encountering exception</span> |
| <span class="source-line-no">3598</span><span id="line-3598"> region.storeHotnessProtector.start(curFamilyCellMap);</span> |
| <span class="source-line-no">3599</span><span id="line-3599"> } catch (RegionTooBusyException rtbe) {</span> |
| <span class="source-line-no">3600</span><span id="line-3600"> region.storeHotnessProtector.finish(curFamilyCellMap);</span> |
| <span class="source-line-no">3601</span><span id="line-3601"> if (isAtomic()) {</span> |
| <span class="source-line-no">3602</span><span id="line-3602"> throw rtbe;</span> |
| <span class="source-line-no">3603</span><span id="line-3603"> }</span> |
| <span class="source-line-no">3604</span><span id="line-3604"> retCodeDetails[lastIndexExclusive] =</span> |
| <span class="source-line-no">3605</span><span id="line-3605"> new OperationStatus(OperationStatusCode.STORE_TOO_BUSY, rtbe.getMessage());</span> |
| <span class="source-line-no">3606</span><span id="line-3606"> continue;</span> |
| <span class="source-line-no">3607</span><span id="line-3607"> }</span> |
| <span class="source-line-no">3608</span><span id="line-3608"></span> |
| <span class="source-line-no">3609</span><span id="line-3609"> Mutation mutation = getMutation(lastIndexExclusive);</span> |
| <span class="source-line-no">3610</span><span id="line-3610"> // If we haven't got any rows in our batch, we should block to get the next one.</span> |
| <span class="source-line-no">3611</span><span id="line-3611"> RowLock rowLock = null;</span> |
| <span class="source-line-no">3612</span><span id="line-3612"> boolean throwException = false;</span> |
| <span class="source-line-no">3613</span><span id="line-3613"> try {</span> |
| <span class="source-line-no">3614</span><span id="line-3614"> // if atomic then get exclusive lock, else shared lock</span> |
| <span class="source-line-no">3615</span><span id="line-3615"> rowLock = region.getRowLock(mutation.getRow(), !isAtomic(), prevRowLock);</span> |
| <span class="source-line-no">3616</span><span id="line-3616"> } catch (TimeoutIOException | InterruptedIOException e) {</span> |
| <span class="source-line-no">3617</span><span id="line-3617"> // NOTE: We will retry when other exceptions, but we should stop if we receive</span> |
| <span class="source-line-no">3618</span><span id="line-3618"> // TimeoutIOException or InterruptedIOException as operation has timed out or</span> |
| <span class="source-line-no">3619</span><span id="line-3619"> // interrupted respectively.</span> |
| <span class="source-line-no">3620</span><span id="line-3620"> throwException = true;</span> |
| <span class="source-line-no">3621</span><span id="line-3621"> throw e;</span> |
| <span class="source-line-no">3622</span><span id="line-3622"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">3623</span><span id="line-3623"> LOG.warn("Failed getting lock, row={}, in region {}",</span> |
| <span class="source-line-no">3624</span><span id="line-3624"> Bytes.toStringBinary(mutation.getRow()), this, ioe);</span> |
| <span class="source-line-no">3625</span><span id="line-3625"> if (isAtomic()) { // fail, atomic means all or none</span> |
| <span class="source-line-no">3626</span><span id="line-3626"> throwException = true;</span> |
| <span class="source-line-no">3627</span><span id="line-3627"> throw ioe;</span> |
| <span class="source-line-no">3628</span><span id="line-3628"> }</span> |
| <span class="source-line-no">3629</span><span id="line-3629"> } catch (Throwable throwable) {</span> |
| <span class="source-line-no">3630</span><span id="line-3630"> throwException = true;</span> |
| <span class="source-line-no">3631</span><span id="line-3631"> throw throwable;</span> |
| <span class="source-line-no">3632</span><span id="line-3632"> } finally {</span> |
| <span class="source-line-no">3633</span><span id="line-3633"> if (throwException) {</span> |
| <span class="source-line-no">3634</span><span id="line-3634"> region.storeHotnessProtector.finish(curFamilyCellMap);</span> |
| <span class="source-line-no">3635</span><span id="line-3635"> }</span> |
| <span class="source-line-no">3636</span><span id="line-3636"> }</span> |
| <span class="source-line-no">3637</span><span id="line-3637"> if (rowLock == null) {</span> |
| <span class="source-line-no">3638</span><span id="line-3638"> // We failed to grab another lock</span> |
| <span class="source-line-no">3639</span><span id="line-3639"> if (isAtomic()) {</span> |
| <span class="source-line-no">3640</span><span id="line-3640"> region.storeHotnessProtector.finish(curFamilyCellMap);</span> |
| <span class="source-line-no">3641</span><span id="line-3641"> throw new IOException("Can't apply all operations atomically!");</span> |
| <span class="source-line-no">3642</span><span id="line-3642"> }</span> |
| <span class="source-line-no">3643</span><span id="line-3643"> break; // Stop acquiring more rows for this batch</span> |
| <span class="source-line-no">3644</span><span id="line-3644"> } else {</span> |
| <span class="source-line-no">3645</span><span id="line-3645"> if (rowLock != prevRowLock) {</span> |
| <span class="source-line-no">3646</span><span id="line-3646"> // It is a different row now, add this to the acquiredRowLocks and</span> |
| <span class="source-line-no">3647</span><span id="line-3647"> // set prevRowLock to the new returned rowLock</span> |
| <span class="source-line-no">3648</span><span id="line-3648"> acquiredRowLocks.add(rowLock);</span> |
| <span class="source-line-no">3649</span><span id="line-3649"> prevRowLock = rowLock;</span> |
| <span class="source-line-no">3650</span><span id="line-3650"> }</span> |
| <span class="source-line-no">3651</span><span id="line-3651"> }</span> |
| <span class="source-line-no">3652</span><span id="line-3652"></span> |
| <span class="source-line-no">3653</span><span id="line-3653"> readyToWriteCount++;</span> |
| <span class="source-line-no">3654</span><span id="line-3654"> }</span> |
| <span class="source-line-no">3655</span><span id="line-3655"> return createMiniBatch(lastIndexExclusive, readyToWriteCount);</span> |
| <span class="source-line-no">3656</span><span id="line-3656"> }</span> |
| <span class="source-line-no">3657</span><span id="line-3657"></span> |
| <span class="source-line-no">3658</span><span id="line-3658"> protected MiniBatchOperationInProgress<Mutation> createMiniBatch(final int lastIndexExclusive,</span> |
| <span class="source-line-no">3659</span><span id="line-3659"> final int readyToWriteCount) {</span> |
| <span class="source-line-no">3660</span><span id="line-3660"> return new MiniBatchOperationInProgress<>(getMutationsForCoprocs(), retCodeDetails,</span> |
| <span class="source-line-no">3661</span><span id="line-3661"> walEditsFromCoprocessors, nextIndexToProcess, lastIndexExclusive, readyToWriteCount);</span> |
| <span class="source-line-no">3662</span><span id="line-3662"> }</span> |
| <span class="source-line-no">3663</span><span id="line-3663"></span> |
| <span class="source-line-no">3664</span><span id="line-3664"> protected WALEdit createWALEdit(final MiniBatchOperationInProgress<Mutation> miniBatchOp) {</span> |
| <span class="source-line-no">3665</span><span id="line-3665"> return new WALEdit(miniBatchOp.getCellCount(), isInReplay());</span> |
| <span class="source-line-no">3666</span><span id="line-3666"> }</span> |
| <span class="source-line-no">3667</span><span id="line-3667"></span> |
| <span class="source-line-no">3668</span><span id="line-3668"> /**</span> |
| <span class="source-line-no">3669</span><span id="line-3669"> * Builds separate WALEdit per nonce by applying input mutations. If WALEdits from CP are</span> |
| <span class="source-line-no">3670</span><span id="line-3670"> * present, they are merged to result WALEdit.</span> |
| <span class="source-line-no">3671</span><span id="line-3671"> */</span> |
| <span class="source-line-no">3672</span><span id="line-3672"> public List<Pair<NonceKey, WALEdit>></span> |
| <span class="source-line-no">3673</span><span id="line-3673"> buildWALEdits(final MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {</span> |
| <span class="source-line-no">3674</span><span id="line-3674"> List<Pair<NonceKey, WALEdit>> walEdits = new ArrayList<>();</span> |
| <span class="source-line-no">3675</span><span id="line-3675"></span> |
| <span class="source-line-no">3676</span><span id="line-3676"> visitBatchOperations(true, nextIndexToProcess + miniBatchOp.size(), new Visitor() {</span> |
| <span class="source-line-no">3677</span><span id="line-3677"> private Pair<NonceKey, WALEdit> curWALEditForNonce;</span> |
| <span class="source-line-no">3678</span><span id="line-3678"></span> |
| <span class="source-line-no">3679</span><span id="line-3679"> @Override</span> |
| <span class="source-line-no">3680</span><span id="line-3680"> public boolean visit(int index) throws IOException {</span> |
| <span class="source-line-no">3681</span><span id="line-3681"> Mutation m = getMutation(index);</span> |
| <span class="source-line-no">3682</span><span id="line-3682"> // we use durability of the original mutation for the mutation passed by CP.</span> |
| <span class="source-line-no">3683</span><span id="line-3683"> if (region.getEffectiveDurability(m.getDurability()) == Durability.SKIP_WAL) {</span> |
| <span class="source-line-no">3684</span><span id="line-3684"> region.recordMutationWithoutWal(m.getFamilyCellMap());</span> |
| <span class="source-line-no">3685</span><span id="line-3685"> /**</span> |
| <span class="source-line-no">3686</span><span id="line-3686"> * Here is for HBASE-26993,in order to make the new framework for region replication</span> |
| <span class="source-line-no">3687</span><span id="line-3687"> * could work for SKIP_WAL, we save the {@link Mutation} which</span> |
| <span class="source-line-no">3688</span><span id="line-3688"> * {@link Mutation#getDurability} is {@link Durability#SKIP_WAL} in miniBatchOp.</span> |
| <span class="source-line-no">3689</span><span id="line-3689"> */</span> |
| <span class="source-line-no">3690</span><span id="line-3690"> cacheSkipWALMutationForRegionReplication(miniBatchOp, walEdits, familyCellMaps[index]);</span> |
| <span class="source-line-no">3691</span><span id="line-3691"> return true;</span> |
| <span class="source-line-no">3692</span><span id="line-3692"> }</span> |
| <span class="source-line-no">3693</span><span id="line-3693"></span> |
| <span class="source-line-no">3694</span><span id="line-3694"> // the batch may contain multiple nonce keys (replay case). If so, write WALEdit for each.</span> |
| <span class="source-line-no">3695</span><span id="line-3695"> // Given how nonce keys are originally written, these should be contiguous.</span> |
| <span class="source-line-no">3696</span><span id="line-3696"> // They don't have to be, it will still work, just write more WALEdits than needed.</span> |
| <span class="source-line-no">3697</span><span id="line-3697"> long nonceGroup = getNonceGroup(index);</span> |
| <span class="source-line-no">3698</span><span id="line-3698"> long nonce = getNonce(index);</span> |
| <span class="source-line-no">3699</span><span id="line-3699"> if (</span> |
| <span class="source-line-no">3700</span><span id="line-3700"> curWALEditForNonce == null</span> |
| <span class="source-line-no">3701</span><span id="line-3701"> || curWALEditForNonce.getFirst().getNonceGroup() != nonceGroup</span> |
| <span class="source-line-no">3702</span><span id="line-3702"> || curWALEditForNonce.getFirst().getNonce() != nonce</span> |
| <span class="source-line-no">3703</span><span id="line-3703"> ) {</span> |
| <span class="source-line-no">3704</span><span id="line-3704"> curWALEditForNonce =</span> |
| <span class="source-line-no">3705</span><span id="line-3705"> new Pair<>(new NonceKey(nonceGroup, nonce), createWALEdit(miniBatchOp));</span> |
| <span class="source-line-no">3706</span><span id="line-3706"> walEdits.add(curWALEditForNonce);</span> |
| <span class="source-line-no">3707</span><span id="line-3707"> }</span> |
| <span class="source-line-no">3708</span><span id="line-3708"> WALEdit walEdit = curWALEditForNonce.getSecond();</span> |
| <span class="source-line-no">3709</span><span id="line-3709"></span> |
| <span class="source-line-no">3710</span><span id="line-3710"> // Add WAL edits from CPs.</span> |
| <span class="source-line-no">3711</span><span id="line-3711"> WALEdit fromCP = walEditsFromCoprocessors[index];</span> |
| <span class="source-line-no">3712</span><span id="line-3712"> List<Cell> cellsFromCP = fromCP == null ? Collections.emptyList() : fromCP.getCells();</span> |
| <span class="source-line-no">3713</span><span id="line-3713"> addNonSkipWALMutationsToWALEdit(miniBatchOp, walEdit, cellsFromCP, familyCellMaps[index]);</span> |
| <span class="source-line-no">3714</span><span id="line-3714"> return true;</span> |
| <span class="source-line-no">3715</span><span id="line-3715"> }</span> |
| <span class="source-line-no">3716</span><span id="line-3716"> });</span> |
| <span class="source-line-no">3717</span><span id="line-3717"> return walEdits;</span> |
| <span class="source-line-no">3718</span><span id="line-3718"> }</span> |
| <span class="source-line-no">3719</span><span id="line-3719"></span> |
| <span class="source-line-no">3720</span><span id="line-3720"> protected void addNonSkipWALMutationsToWALEdit(</span> |
| <span class="source-line-no">3721</span><span id="line-3721"> final MiniBatchOperationInProgress<Mutation> miniBatchOp, WALEdit walEdit,</span> |
| <span class="source-line-no">3722</span><span id="line-3722"> List<Cell> cellsFromCP, Map<byte[], List<ExtendedCell>> familyCellMap) {</span> |
| <span class="source-line-no">3723</span><span id="line-3723"> doAddCellsToWALEdit(walEdit, cellsFromCP, familyCellMap);</span> |
| <span class="source-line-no">3724</span><span id="line-3724"> }</span> |
| <span class="source-line-no">3725</span><span id="line-3725"></span> |
| <span class="source-line-no">3726</span><span id="line-3726"> protected static void doAddCellsToWALEdit(WALEdit walEdit, List<Cell> cellsFromCP,</span> |
| <span class="source-line-no">3727</span><span id="line-3727"> Map<byte[], List<ExtendedCell>> familyCellMap) {</span> |
| <span class="source-line-no">3728</span><span id="line-3728"> walEdit.add(cellsFromCP);</span> |
| <span class="source-line-no">3729</span><span id="line-3729"> walEdit.add((Map) familyCellMap);</span> |
| <span class="source-line-no">3730</span><span id="line-3730"> }</span> |
| <span class="source-line-no">3731</span><span id="line-3731"></span> |
| <span class="source-line-no">3732</span><span id="line-3732"> protected abstract void cacheSkipWALMutationForRegionReplication(</span> |
| <span class="source-line-no">3733</span><span id="line-3733"> final MiniBatchOperationInProgress<Mutation> miniBatchOp,</span> |
| <span class="source-line-no">3734</span><span id="line-3734"> List<Pair<NonceKey, WALEdit>> walEdits, Map<byte[], List<ExtendedCell>> familyCellMap);</span> |
| <span class="source-line-no">3735</span><span id="line-3735"></span> |
| <span class="source-line-no">3736</span><span id="line-3736"> /**</span> |
| <span class="source-line-no">3737</span><span id="line-3737"> * This method completes mini-batch operations by calling postBatchMutate() CP hook (if</span> |
| <span class="source-line-no">3738</span><span id="line-3738"> * required) and completing mvcc.</span> |
| <span class="source-line-no">3739</span><span id="line-3739"> */</span> |
| <span class="source-line-no">3740</span><span id="line-3740"> public void completeMiniBatchOperations(</span> |
| <span class="source-line-no">3741</span><span id="line-3741"> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final WriteEntry writeEntry)</span> |
| <span class="source-line-no">3742</span><span id="line-3742"> throws IOException {</span> |
| <span class="source-line-no">3743</span><span id="line-3743"> if (writeEntry != null) {</span> |
| <span class="source-line-no">3744</span><span id="line-3744"> region.mvcc.completeAndWait(writeEntry);</span> |
| <span class="source-line-no">3745</span><span id="line-3745"> }</span> |
| <span class="source-line-no">3746</span><span id="line-3746"> }</span> |
| <span class="source-line-no">3747</span><span id="line-3747"></span> |
| <span class="source-line-no">3748</span><span id="line-3748"> public void doPostOpCleanupForMiniBatch(</span> |
| <span class="source-line-no">3749</span><span id="line-3749"> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final WALEdit walEdit,</span> |
| <span class="source-line-no">3750</span><span id="line-3750"> boolean success) throws IOException {</span> |
| <span class="source-line-no">3751</span><span id="line-3751"> doFinishHotnessProtector(miniBatchOp);</span> |
| <span class="source-line-no">3752</span><span id="line-3752"> }</span> |
| <span class="source-line-no">3753</span><span id="line-3753"></span> |
| <span class="source-line-no">3754</span><span id="line-3754"> private void</span> |
| <span class="source-line-no">3755</span><span id="line-3755"> doFinishHotnessProtector(final MiniBatchOperationInProgress<Mutation> miniBatchOp) {</span> |
| <span class="source-line-no">3756</span><span id="line-3756"> // check and return if the protector is not enabled</span> |
| <span class="source-line-no">3757</span><span id="line-3757"> if (!region.storeHotnessProtector.isEnable()) {</span> |
| <span class="source-line-no">3758</span><span id="line-3758"> return;</span> |
| <span class="source-line-no">3759</span><span id="line-3759"> }</span> |
| <span class="source-line-no">3760</span><span id="line-3760"> // miniBatchOp is null, if and only if lockRowsAndBuildMiniBatch throwing exception.</span> |
| <span class="source-line-no">3761</span><span id="line-3761"> // This case was handled.</span> |
| <span class="source-line-no">3762</span><span id="line-3762"> if (miniBatchOp == null) {</span> |
| <span class="source-line-no">3763</span><span id="line-3763"> return;</span> |
| <span class="source-line-no">3764</span><span id="line-3764"> }</span> |
| <span class="source-line-no">3765</span><span id="line-3765"></span> |
| <span class="source-line-no">3766</span><span id="line-3766"> final int finalLastIndexExclusive = miniBatchOp.getLastIndexExclusive();</span> |
| <span class="source-line-no">3767</span><span id="line-3767"></span> |
| <span class="source-line-no">3768</span><span id="line-3768"> for (int i = nextIndexToProcess; i < finalLastIndexExclusive; i++) {</span> |
| <span class="source-line-no">3769</span><span id="line-3769"> switch (retCodeDetails[i].getOperationStatusCode()) {</span> |
| <span class="source-line-no">3770</span><span id="line-3770"> case SUCCESS:</span> |
| <span class="source-line-no">3771</span><span id="line-3771"> case FAILURE:</span> |
| <span class="source-line-no">3772</span><span id="line-3772"> region.storeHotnessProtector.finish(getMutation(i).getFamilyCellMap());</span> |
| <span class="source-line-no">3773</span><span id="line-3773"> break;</span> |
| <span class="source-line-no">3774</span><span id="line-3774"> default:</span> |
| <span class="source-line-no">3775</span><span id="line-3775"> // do nothing</span> |
| <span class="source-line-no">3776</span><span id="line-3776"> // We won't start the protector for NOT_RUN/BAD_FAMILY/SANITY_CHECK_FAILURE and the</span> |
| <span class="source-line-no">3777</span><span id="line-3777"> // STORE_TOO_BUSY case is handled in StoreHotnessProtector#start</span> |
| <span class="source-line-no">3778</span><span id="line-3778"> break;</span> |
| <span class="source-line-no">3779</span><span id="line-3779"> }</span> |
| <span class="source-line-no">3780</span><span id="line-3780"> }</span> |
| <span class="source-line-no">3781</span><span id="line-3781"> }</span> |
| <span class="source-line-no">3782</span><span id="line-3782"></span> |
| <span class="source-line-no">3783</span><span id="line-3783"> /**</span> |
| <span class="source-line-no">3784</span><span id="line-3784"> * Atomically apply the given map of family->edits to the memstore. This handles the consistency</span> |
| <span class="source-line-no">3785</span><span id="line-3785"> * control on its own, but the caller should already have locked updatesLock.readLock(). This</span> |
| <span class="source-line-no">3786</span><span id="line-3786"> * also does <b>not</b> check the families for validity.</span> |
| <span class="source-line-no">3787</span><span id="line-3787"> * @param familyMap Map of Cells by family</span> |
| <span class="source-line-no">3788</span><span id="line-3788"> */</span> |
| <span class="source-line-no">3789</span><span id="line-3789"> protected void applyFamilyMapToMemStore(Map<byte[], List<ExtendedCell>> familyMap,</span> |
| <span class="source-line-no">3790</span><span id="line-3790"> MemStoreSizing memstoreAccounting) {</span> |
| <span class="source-line-no">3791</span><span id="line-3791"> for (Map.Entry<byte[], List<ExtendedCell>> e : familyMap.entrySet()) {</span> |
| <span class="source-line-no">3792</span><span id="line-3792"> byte[] family = e.getKey();</span> |
| <span class="source-line-no">3793</span><span id="line-3793"> List<ExtendedCell> cells = e.getValue();</span> |
| <span class="source-line-no">3794</span><span id="line-3794"> assert cells instanceof RandomAccess;</span> |
| <span class="source-line-no">3795</span><span id="line-3795"> region.applyToMemStore(region.getStore(family), cells, false, memstoreAccounting);</span> |
| <span class="source-line-no">3796</span><span id="line-3796"> }</span> |
| <span class="source-line-no">3797</span><span id="line-3797"> }</span> |
| <span class="source-line-no">3798</span><span id="line-3798"> }</span> |
| <span class="source-line-no">3799</span><span id="line-3799"></span> |
| <span class="source-line-no">3800</span><span id="line-3800"> /**</span> |
| <span class="source-line-no">3801</span><span id="line-3801"> * Batch of mutation operations. Base class is shared with {@link ReplayBatchOperation} as most of</span> |
| <span class="source-line-no">3802</span><span id="line-3802"> * the logic is same.</span> |
| <span class="source-line-no">3803</span><span id="line-3803"> */</span> |
| <span class="source-line-no">3804</span><span id="line-3804"> private static class MutationBatchOperation extends BatchOperation<Mutation> {</span> |
| <span class="source-line-no">3805</span><span id="line-3805"></span> |
| <span class="source-line-no">3806</span><span id="line-3806"> // For nonce operations</span> |
| <span class="source-line-no">3807</span><span id="line-3807"> private long nonceGroup;</span> |
| <span class="source-line-no">3808</span><span id="line-3808"> private long nonce;</span> |
| <span class="source-line-no">3809</span><span id="line-3809"> protected boolean canProceed;</span> |
| <span class="source-line-no">3810</span><span id="line-3810"> private boolean regionReplicateEnable;</span> |
| <span class="source-line-no">3811</span><span id="line-3811"></span> |
| <span class="source-line-no">3812</span><span id="line-3812"> public MutationBatchOperation(final HRegion region, Mutation[] operations, boolean atomic,</span> |
| <span class="source-line-no">3813</span><span id="line-3813"> long nonceGroup, long nonce) {</span> |
| <span class="source-line-no">3814</span><span id="line-3814"> super(region, operations);</span> |
| <span class="source-line-no">3815</span><span id="line-3815"> this.atomic = atomic;</span> |
| <span class="source-line-no">3816</span><span id="line-3816"> this.nonceGroup = nonceGroup;</span> |
| <span class="source-line-no">3817</span><span id="line-3817"> this.nonce = nonce;</span> |
| <span class="source-line-no">3818</span><span id="line-3818"> this.regionReplicateEnable = region.regionReplicationSink.isPresent();</span> |
| <span class="source-line-no">3819</span><span id="line-3819"> }</span> |
| <span class="source-line-no">3820</span><span id="line-3820"></span> |
| <span class="source-line-no">3821</span><span id="line-3821"> @Override</span> |
| <span class="source-line-no">3822</span><span id="line-3822"> public Mutation getMutation(int index) {</span> |
| <span class="source-line-no">3823</span><span id="line-3823"> return this.operations[index];</span> |
| <span class="source-line-no">3824</span><span id="line-3824"> }</span> |
| <span class="source-line-no">3825</span><span id="line-3825"></span> |
| <span class="source-line-no">3826</span><span id="line-3826"> @Override</span> |
| <span class="source-line-no">3827</span><span id="line-3827"> public long getNonceGroup(int index) {</span> |
| <span class="source-line-no">3828</span><span id="line-3828"> return nonceGroup;</span> |
| <span class="source-line-no">3829</span><span id="line-3829"> }</span> |
| <span class="source-line-no">3830</span><span id="line-3830"></span> |
| <span class="source-line-no">3831</span><span id="line-3831"> @Override</span> |
| <span class="source-line-no">3832</span><span id="line-3832"> public long getNonce(int index) {</span> |
| <span class="source-line-no">3833</span><span id="line-3833"> return nonce;</span> |
| <span class="source-line-no">3834</span><span id="line-3834"> }</span> |
| <span class="source-line-no">3835</span><span id="line-3835"></span> |
| <span class="source-line-no">3836</span><span id="line-3836"> @Override</span> |
| <span class="source-line-no">3837</span><span id="line-3837"> public Mutation[] getMutationsForCoprocs() {</span> |
| <span class="source-line-no">3838</span><span id="line-3838"> return this.operations;</span> |
| <span class="source-line-no">3839</span><span id="line-3839"> }</span> |
| <span class="source-line-no">3840</span><span id="line-3840"></span> |
| <span class="source-line-no">3841</span><span id="line-3841"> @Override</span> |
| <span class="source-line-no">3842</span><span id="line-3842"> public boolean isInReplay() {</span> |
| <span class="source-line-no">3843</span><span id="line-3843"> return false;</span> |
| <span class="source-line-no">3844</span><span id="line-3844"> }</span> |
| <span class="source-line-no">3845</span><span id="line-3845"></span> |
| <span class="source-line-no">3846</span><span id="line-3846"> @Override</span> |
| <span class="source-line-no">3847</span><span id="line-3847"> public long getOrigLogSeqNum() {</span> |
| <span class="source-line-no">3848</span><span id="line-3848"> return SequenceId.NO_SEQUENCE_ID;</span> |
| <span class="source-line-no">3849</span><span id="line-3849"> }</span> |
| <span class="source-line-no">3850</span><span id="line-3850"></span> |
| <span class="source-line-no">3851</span><span id="line-3851"> @Override</span> |
| <span class="source-line-no">3852</span><span id="line-3852"> public void startRegionOperation() throws IOException {</span> |
| <span class="source-line-no">3853</span><span id="line-3853"> region.startRegionOperation(Operation.BATCH_MUTATE);</span> |
| <span class="source-line-no">3854</span><span id="line-3854"> }</span> |
| <span class="source-line-no">3855</span><span id="line-3855"></span> |
| <span class="source-line-no">3856</span><span id="line-3856"> @Override</span> |
| <span class="source-line-no">3857</span><span id="line-3857"> public void closeRegionOperation() throws IOException {</span> |
| <span class="source-line-no">3858</span><span id="line-3858"> region.closeRegionOperation(Operation.BATCH_MUTATE);</span> |
| <span class="source-line-no">3859</span><span id="line-3859"> }</span> |
| <span class="source-line-no">3860</span><span id="line-3860"></span> |
| <span class="source-line-no">3861</span><span id="line-3861"> @Override</span> |
| <span class="source-line-no">3862</span><span id="line-3862"> public void checkAndPreparePut(Put p) throws IOException {</span> |
| <span class="source-line-no">3863</span><span id="line-3863"> region.checkFamilies(p.getFamilyCellMap().keySet(), p.getDurability());</span> |
| <span class="source-line-no">3864</span><span id="line-3864"> }</span> |
| <span class="source-line-no">3865</span><span id="line-3865"></span> |
| <span class="source-line-no">3866</span><span id="line-3866"> @Override</span> |
| <span class="source-line-no">3867</span><span id="line-3867"> public void checkAndPrepare() throws IOException {</span> |
| <span class="source-line-no">3868</span><span id="line-3868"> // index 0: puts, index 1: deletes, index 2: increments, index 3: append</span> |
| <span class="source-line-no">3869</span><span id="line-3869"> final int[] metrics = { 0, 0, 0, 0 };</span> |
| <span class="source-line-no">3870</span><span id="line-3870"></span> |
| <span class="source-line-no">3871</span><span id="line-3871"> visitBatchOperations(true, this.size(), new Visitor() {</span> |
| <span class="source-line-no">3872</span><span id="line-3872"> private long now = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">3873</span><span id="line-3873"> private WALEdit walEdit;</span> |
| <span class="source-line-no">3874</span><span id="line-3874"></span> |
| <span class="source-line-no">3875</span><span id="line-3875"> @Override</span> |
| <span class="source-line-no">3876</span><span id="line-3876"> public boolean visit(int index) throws IOException {</span> |
| <span class="source-line-no">3877</span><span id="line-3877"> // Run coprocessor pre hook outside of locks to avoid deadlock</span> |
| <span class="source-line-no">3878</span><span id="line-3878"> if (region.coprocessorHost != null) {</span> |
| <span class="source-line-no">3879</span><span id="line-3879"> if (walEdit == null) {</span> |
| <span class="source-line-no">3880</span><span id="line-3880"> walEdit = new WALEdit();</span> |
| <span class="source-line-no">3881</span><span id="line-3881"> }</span> |
| <span class="source-line-no">3882</span><span id="line-3882"> callPreMutateCPHook(index, walEdit, metrics);</span> |
| <span class="source-line-no">3883</span><span id="line-3883"> if (!walEdit.isEmpty()) {</span> |
| <span class="source-line-no">3884</span><span id="line-3884"> walEditsFromCoprocessors[index] = walEdit;</span> |
| <span class="source-line-no">3885</span><span id="line-3885"> walEdit = null;</span> |
| <span class="source-line-no">3886</span><span id="line-3886"> }</span> |
| <span class="source-line-no">3887</span><span id="line-3887"> }</span> |
| <span class="source-line-no">3888</span><span id="line-3888"> if (isOperationPending(index)) {</span> |
| <span class="source-line-no">3889</span><span id="line-3889"> // TODO: Currently validation is done with current time before acquiring locks and</span> |
| <span class="source-line-no">3890</span><span id="line-3890"> // updates are done with different timestamps after acquiring locks. This behavior is</span> |
| <span class="source-line-no">3891</span><span id="line-3891"> // inherited from the code prior to this change. Can this be changed?</span> |
| <span class="source-line-no">3892</span><span id="line-3892"> checkAndPrepareMutation(index, now);</span> |
| <span class="source-line-no">3893</span><span id="line-3893"> }</span> |
| <span class="source-line-no">3894</span><span id="line-3894"> return true;</span> |
| <span class="source-line-no">3895</span><span id="line-3895"> }</span> |
| <span class="source-line-no">3896</span><span id="line-3896"> });</span> |
| <span class="source-line-no">3897</span><span id="line-3897"></span> |
| <span class="source-line-no">3898</span><span id="line-3898"> // FIXME: we may update metrics twice! here for all operations bypassed by CP and later in</span> |
| <span class="source-line-no">3899</span><span id="line-3899"> // normal processing.</span> |
| <span class="source-line-no">3900</span><span id="line-3900"> // Update metrics in same way as it is done when we go the normal processing route (we now</span> |
| <span class="source-line-no">3901</span><span id="line-3901"> // update general metrics though a Coprocessor did the work).</span> |
| <span class="source-line-no">3902</span><span id="line-3902"> if (region.metricsRegion != null) {</span> |
| <span class="source-line-no">3903</span><span id="line-3903"> if (metrics[0] > 0) {</span> |
| <span class="source-line-no">3904</span><span id="line-3904"> // There were some Puts in the batch.</span> |
| <span class="source-line-no">3905</span><span id="line-3905"> region.metricsRegion.updatePut();</span> |
| <span class="source-line-no">3906</span><span id="line-3906"> }</span> |
| <span class="source-line-no">3907</span><span id="line-3907"> if (metrics[1] > 0) {</span> |
| <span class="source-line-no">3908</span><span id="line-3908"> // There were some Deletes in the batch.</span> |
| <span class="source-line-no">3909</span><span id="line-3909"> region.metricsRegion.updateDelete();</span> |
| <span class="source-line-no">3910</span><span id="line-3910"> }</span> |
| <span class="source-line-no">3911</span><span id="line-3911"> if (metrics[2] > 0) {</span> |
| <span class="source-line-no">3912</span><span id="line-3912"> // There were some Increment in the batch.</span> |
| <span class="source-line-no">3913</span><span id="line-3913"> region.metricsRegion.updateIncrement();</span> |
| <span class="source-line-no">3914</span><span id="line-3914"> }</span> |
| <span class="source-line-no">3915</span><span id="line-3915"> if (metrics[3] > 0) {</span> |
| <span class="source-line-no">3916</span><span id="line-3916"> // There were some Append in the batch.</span> |
| <span class="source-line-no">3917</span><span id="line-3917"> region.metricsRegion.updateAppend();</span> |
| <span class="source-line-no">3918</span><span id="line-3918"> }</span> |
| <span class="source-line-no">3919</span><span id="line-3919"> }</span> |
| <span class="source-line-no">3920</span><span id="line-3920"> }</span> |
| <span class="source-line-no">3921</span><span id="line-3921"></span> |
| <span class="source-line-no">3922</span><span id="line-3922"> @Override</span> |
| <span class="source-line-no">3923</span><span id="line-3923"> public void prepareMiniBatchOperations(MiniBatchOperationInProgress<Mutation> miniBatchOp,</span> |
| <span class="source-line-no">3924</span><span id="line-3924"> long timestamp, final List<RowLock> acquiredRowLocks) throws IOException {</span> |
| <span class="source-line-no">3925</span><span id="line-3925"> // For nonce operations</span> |
| <span class="source-line-no">3926</span><span id="line-3926"> canProceed = startNonceOperation();</span> |
| <span class="source-line-no">3927</span><span id="line-3927"></span> |
| <span class="source-line-no">3928</span><span id="line-3928"> visitBatchOperations(true, miniBatchOp.getLastIndexExclusive(), (int index) -> {</span> |
| <span class="source-line-no">3929</span><span id="line-3929"> Mutation mutation = getMutation(index);</span> |
| <span class="source-line-no">3930</span><span id="line-3930"> if (mutation instanceof Put) {</span> |
| <span class="source-line-no">3931</span><span id="line-3931"> HRegion.updateCellTimestamps(familyCellMaps[index].values(), Bytes.toBytes(timestamp));</span> |
| <span class="source-line-no">3932</span><span id="line-3932"> miniBatchOp.incrementNumOfPuts();</span> |
| <span class="source-line-no">3933</span><span id="line-3933"> } else if (mutation instanceof Delete) {</span> |
| <span class="source-line-no">3934</span><span id="line-3934"> region.prepareDeleteTimestamps(mutation, familyCellMaps[index], Bytes.toBytes(timestamp));</span> |
| <span class="source-line-no">3935</span><span id="line-3935"> miniBatchOp.incrementNumOfDeletes();</span> |
| <span class="source-line-no">3936</span><span id="line-3936"> } else if (mutation instanceof Increment || mutation instanceof Append) {</span> |
| <span class="source-line-no">3937</span><span id="line-3937"> boolean returnResults;</span> |
| <span class="source-line-no">3938</span><span id="line-3938"> if (mutation instanceof Increment) {</span> |
| <span class="source-line-no">3939</span><span id="line-3939"> returnResults = ((Increment) mutation).isReturnResults();</span> |
| <span class="source-line-no">3940</span><span id="line-3940"> } else {</span> |
| <span class="source-line-no">3941</span><span id="line-3941"> returnResults = ((Append) mutation).isReturnResults();</span> |
| <span class="source-line-no">3942</span><span id="line-3942"> }</span> |
| <span class="source-line-no">3943</span><span id="line-3943"></span> |
| <span class="source-line-no">3944</span><span id="line-3944"> // For nonce operations</span> |
| <span class="source-line-no">3945</span><span id="line-3945"> if (!canProceed) {</span> |
| <span class="source-line-no">3946</span><span id="line-3946"> Result result;</span> |
| <span class="source-line-no">3947</span><span id="line-3947"> if (returnResults) {</span> |
| <span class="source-line-no">3948</span><span id="line-3948"> // convert duplicate increment/append to get</span> |
| <span class="source-line-no">3949</span><span id="line-3949"> List<Cell> results = region.get(toGet(mutation), false, nonceGroup, nonce);</span> |
| <span class="source-line-no">3950</span><span id="line-3950"> result = Result.create(results);</span> |
| <span class="source-line-no">3951</span><span id="line-3951"> } else {</span> |
| <span class="source-line-no">3952</span><span id="line-3952"> result = Result.EMPTY_RESULT;</span> |
| <span class="source-line-no">3953</span><span id="line-3953"> }</span> |
| <span class="source-line-no">3954</span><span id="line-3954"> retCodeDetails[index] = new OperationStatus(OperationStatusCode.SUCCESS, result);</span> |
| <span class="source-line-no">3955</span><span id="line-3955"> return true;</span> |
| <span class="source-line-no">3956</span><span id="line-3956"> }</span> |
| <span class="source-line-no">3957</span><span id="line-3957"></span> |
| <span class="source-line-no">3958</span><span id="line-3958"> Result result = null;</span> |
| <span class="source-line-no">3959</span><span id="line-3959"> if (region.coprocessorHost != null) {</span> |
| <span class="source-line-no">3960</span><span id="line-3960"> if (mutation instanceof Increment) {</span> |
| <span class="source-line-no">3961</span><span id="line-3961"> result = region.coprocessorHost.preIncrementAfterRowLock((Increment) mutation);</span> |
| <span class="source-line-no">3962</span><span id="line-3962"> } else {</span> |
| <span class="source-line-no">3963</span><span id="line-3963"> result = region.coprocessorHost.preAppendAfterRowLock((Append) mutation);</span> |
| <span class="source-line-no">3964</span><span id="line-3964"> }</span> |
| <span class="source-line-no">3965</span><span id="line-3965"> }</span> |
| <span class="source-line-no">3966</span><span id="line-3966"> if (result != null) {</span> |
| <span class="source-line-no">3967</span><span id="line-3967"> retCodeDetails[index] = new OperationStatus(OperationStatusCode.SUCCESS,</span> |
| <span class="source-line-no">3968</span><span id="line-3968"> returnResults ? result : Result.EMPTY_RESULT);</span> |
| <span class="source-line-no">3969</span><span id="line-3969"> return true;</span> |
| <span class="source-line-no">3970</span><span id="line-3970"> }</span> |
| <span class="source-line-no">3971</span><span id="line-3971"></span> |
| <span class="source-line-no">3972</span><span id="line-3972"> List<ExtendedCell> results = returnResults ? new ArrayList<>(mutation.size()) : null;</span> |
| <span class="source-line-no">3973</span><span id="line-3973"> familyCellMaps[index] = reckonDeltas(mutation, results, timestamp);</span> |
| <span class="source-line-no">3974</span><span id="line-3974"> this.results[index] = results != null ? Result.create(results) : Result.EMPTY_RESULT;</span> |
| <span class="source-line-no">3975</span><span id="line-3975"></span> |
| <span class="source-line-no">3976</span><span id="line-3976"> if (mutation instanceof Increment) {</span> |
| <span class="source-line-no">3977</span><span id="line-3977"> miniBatchOp.incrementNumOfIncrements();</span> |
| <span class="source-line-no">3978</span><span id="line-3978"> } else {</span> |
| <span class="source-line-no">3979</span><span id="line-3979"> miniBatchOp.incrementNumOfAppends();</span> |
| <span class="source-line-no">3980</span><span id="line-3980"> }</span> |
| <span class="source-line-no">3981</span><span id="line-3981"> }</span> |
| <span class="source-line-no">3982</span><span id="line-3982"> region.rewriteCellTags(familyCellMaps[index], mutation);</span> |
| <span class="source-line-no">3983</span><span id="line-3983"></span> |
| <span class="source-line-no">3984</span><span id="line-3984"> // update cell count</span> |
| <span class="source-line-no">3985</span><span id="line-3985"> if (region.getEffectiveDurability(mutation.getDurability()) != Durability.SKIP_WAL) {</span> |
| <span class="source-line-no">3986</span><span id="line-3986"> for (List<Cell> cells : mutation.getFamilyCellMap().values()) {</span> |
| <span class="source-line-no">3987</span><span id="line-3987"> miniBatchOp.addCellCount(cells.size());</span> |
| <span class="source-line-no">3988</span><span id="line-3988"> }</span> |
| <span class="source-line-no">3989</span><span id="line-3989"> }</span> |
| <span class="source-line-no">3990</span><span id="line-3990"></span> |
| <span class="source-line-no">3991</span><span id="line-3991"> WALEdit fromCP = walEditsFromCoprocessors[index];</span> |
| <span class="source-line-no">3992</span><span id="line-3992"> if (fromCP != null) {</span> |
| <span class="source-line-no">3993</span><span id="line-3993"> miniBatchOp.addCellCount(fromCP.size());</span> |
| <span class="source-line-no">3994</span><span id="line-3994"> }</span> |
| <span class="source-line-no">3995</span><span id="line-3995"> return true;</span> |
| <span class="source-line-no">3996</span><span id="line-3996"> });</span> |
| <span class="source-line-no">3997</span><span id="line-3997"></span> |
| <span class="source-line-no">3998</span><span id="line-3998"> if (region.coprocessorHost != null) {</span> |
| <span class="source-line-no">3999</span><span id="line-3999"> // calling the pre CP hook for batch mutation</span> |
| <span class="source-line-no">4000</span><span id="line-4000"> region.coprocessorHost.preBatchMutate(miniBatchOp);</span> |
| <span class="source-line-no">4001</span><span id="line-4001"> checkAndMergeCPMutations(miniBatchOp, acquiredRowLocks, timestamp);</span> |
| <span class="source-line-no">4002</span><span id="line-4002"> }</span> |
| <span class="source-line-no">4003</span><span id="line-4003"> }</span> |
| <span class="source-line-no">4004</span><span id="line-4004"></span> |
| <span class="source-line-no">4005</span><span id="line-4005"> /**</span> |
| <span class="source-line-no">4006</span><span id="line-4006"> * Starts the nonce operation for a mutation, if needed.</span> |
| <span class="source-line-no">4007</span><span id="line-4007"> * @return whether to proceed this mutation.</span> |
| <span class="source-line-no">4008</span><span id="line-4008"> */</span> |
| <span class="source-line-no">4009</span><span id="line-4009"> private boolean startNonceOperation() throws IOException {</span> |
| <span class="source-line-no">4010</span><span id="line-4010"> if (</span> |
| <span class="source-line-no">4011</span><span id="line-4011"> region.rsServices == null || region.rsServices.getNonceManager() == null</span> |
| <span class="source-line-no">4012</span><span id="line-4012"> || nonce == HConstants.NO_NONCE</span> |
| <span class="source-line-no">4013</span><span id="line-4013"> ) {</span> |
| <span class="source-line-no">4014</span><span id="line-4014"> return true;</span> |
| <span class="source-line-no">4015</span><span id="line-4015"> }</span> |
| <span class="source-line-no">4016</span><span id="line-4016"> boolean canProceed;</span> |
| <span class="source-line-no">4017</span><span id="line-4017"> try {</span> |
| <span class="source-line-no">4018</span><span id="line-4018"> canProceed =</span> |
| <span class="source-line-no">4019</span><span id="line-4019"> region.rsServices.getNonceManager().startOperation(nonceGroup, nonce, region.rsServices);</span> |
| <span class="source-line-no">4020</span><span id="line-4020"> } catch (InterruptedException ex) {</span> |
| <span class="source-line-no">4021</span><span id="line-4021"> throw new InterruptedIOException("Nonce start operation interrupted");</span> |
| <span class="source-line-no">4022</span><span id="line-4022"> }</span> |
| <span class="source-line-no">4023</span><span id="line-4023"> return canProceed;</span> |
| <span class="source-line-no">4024</span><span id="line-4024"> }</span> |
| <span class="source-line-no">4025</span><span id="line-4025"></span> |
| <span class="source-line-no">4026</span><span id="line-4026"> /**</span> |
| <span class="source-line-no">4027</span><span id="line-4027"> * Ends nonce operation for a mutation, if needed.</span> |
| <span class="source-line-no">4028</span><span id="line-4028"> * @param success Whether the operation for this nonce has succeeded.</span> |
| <span class="source-line-no">4029</span><span id="line-4029"> */</span> |
| <span class="source-line-no">4030</span><span id="line-4030"> private void endNonceOperation(boolean success) {</span> |
| <span class="source-line-no">4031</span><span id="line-4031"> if (</span> |
| <span class="source-line-no">4032</span><span id="line-4032"> region.rsServices != null && region.rsServices.getNonceManager() != null</span> |
| <span class="source-line-no">4033</span><span id="line-4033"> && nonce != HConstants.NO_NONCE</span> |
| <span class="source-line-no">4034</span><span id="line-4034"> ) {</span> |
| <span class="source-line-no">4035</span><span id="line-4035"> region.rsServices.getNonceManager().endOperation(nonceGroup, nonce, success);</span> |
| <span class="source-line-no">4036</span><span id="line-4036"> }</span> |
| <span class="source-line-no">4037</span><span id="line-4037"> }</span> |
| <span class="source-line-no">4038</span><span id="line-4038"></span> |
| <span class="source-line-no">4039</span><span id="line-4039"> private static Get toGet(final Mutation mutation) throws IOException {</span> |
| <span class="source-line-no">4040</span><span id="line-4040"> assert mutation instanceof Increment || mutation instanceof Append;</span> |
| <span class="source-line-no">4041</span><span id="line-4041"> Get get = new Get(mutation.getRow());</span> |
| <span class="source-line-no">4042</span><span id="line-4042"> CellScanner cellScanner = mutation.cellScanner();</span> |
| <span class="source-line-no">4043</span><span id="line-4043"> while (cellScanner.advance()) {</span> |
| <span class="source-line-no">4044</span><span id="line-4044"> Cell cell = cellScanner.current();</span> |
| <span class="source-line-no">4045</span><span id="line-4045"> get.addColumn(CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell));</span> |
| <span class="source-line-no">4046</span><span id="line-4046"> }</span> |
| <span class="source-line-no">4047</span><span id="line-4047"> if (mutation instanceof Increment) {</span> |
| <span class="source-line-no">4048</span><span id="line-4048"> // Increment</span> |
| <span class="source-line-no">4049</span><span id="line-4049"> Increment increment = (Increment) mutation;</span> |
| <span class="source-line-no">4050</span><span id="line-4050"> get.setTimeRange(increment.getTimeRange().getMin(), increment.getTimeRange().getMax());</span> |
| <span class="source-line-no">4051</span><span id="line-4051"> } else {</span> |
| <span class="source-line-no">4052</span><span id="line-4052"> // Append</span> |
| <span class="source-line-no">4053</span><span id="line-4053"> Append append = (Append) mutation;</span> |
| <span class="source-line-no">4054</span><span id="line-4054"> get.setTimeRange(append.getTimeRange().getMin(), append.getTimeRange().getMax());</span> |
| <span class="source-line-no">4055</span><span id="line-4055"> }</span> |
| <span class="source-line-no">4056</span><span id="line-4056"> for (Entry<String, byte[]> entry : mutation.getAttributesMap().entrySet()) {</span> |
| <span class="source-line-no">4057</span><span id="line-4057"> get.setAttribute(entry.getKey(), entry.getValue());</span> |
| <span class="source-line-no">4058</span><span id="line-4058"> }</span> |
| <span class="source-line-no">4059</span><span id="line-4059"> return get;</span> |
| <span class="source-line-no">4060</span><span id="line-4060"> }</span> |
| <span class="source-line-no">4061</span><span id="line-4061"></span> |
| <span class="source-line-no">4062</span><span id="line-4062"> private Map<byte[], List<ExtendedCell>> reckonDeltas(Mutation mutation,</span> |
| <span class="source-line-no">4063</span><span id="line-4063"> List<ExtendedCell> results, long now) throws IOException {</span> |
| <span class="source-line-no">4064</span><span id="line-4064"> assert mutation instanceof Increment || mutation instanceof Append;</span> |
| <span class="source-line-no">4065</span><span id="line-4065"> Map<byte[], List<ExtendedCell>> ret = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">4066</span><span id="line-4066"> // Process a Store/family at a time.</span> |
| <span class="source-line-no">4067</span><span id="line-4067"> for (Map.Entry<byte[], List<ExtendedCell>> entry : PackagePrivateFieldAccessor</span> |
| <span class="source-line-no">4068</span><span id="line-4068"> .getExtendedFamilyCellMap(mutation).entrySet()) {</span> |
| <span class="source-line-no">4069</span><span id="line-4069"> final byte[] columnFamilyName = entry.getKey();</span> |
| <span class="source-line-no">4070</span><span id="line-4070"> List<ExtendedCell> deltas = (List) entry.getValue();</span> |
| <span class="source-line-no">4071</span><span id="line-4071"> // Reckon for the Store what to apply to WAL and MemStore.</span> |
| <span class="source-line-no">4072</span><span id="line-4072"> List<ExtendedCell> toApply =</span> |
| <span class="source-line-no">4073</span><span id="line-4073"> reckonDeltasByStore(region.stores.get(columnFamilyName), mutation, now, deltas, results);</span> |
| <span class="source-line-no">4074</span><span id="line-4074"> if (!toApply.isEmpty()) {</span> |
| <span class="source-line-no">4075</span><span id="line-4075"> for (ExtendedCell cell : toApply) {</span> |
| <span class="source-line-no">4076</span><span id="line-4076"> HStore store = region.getStore(cell);</span> |
| <span class="source-line-no">4077</span><span id="line-4077"> if (store == null) {</span> |
| <span class="source-line-no">4078</span><span id="line-4078"> region.checkFamily(CellUtil.cloneFamily(cell));</span> |
| <span class="source-line-no">4079</span><span id="line-4079"> } else {</span> |
| <span class="source-line-no">4080</span><span id="line-4080"> ret.computeIfAbsent(store.getColumnFamilyDescriptor().getName(),</span> |
| <span class="source-line-no">4081</span><span id="line-4081"> key -> new ArrayList<>()).add(cell);</span> |
| <span class="source-line-no">4082</span><span id="line-4082"> }</span> |
| <span class="source-line-no">4083</span><span id="line-4083"> }</span> |
| <span class="source-line-no">4084</span><span id="line-4084"> }</span> |
| <span class="source-line-no">4085</span><span id="line-4085"> }</span> |
| <span class="source-line-no">4086</span><span id="line-4086"> return ret;</span> |
| <span class="source-line-no">4087</span><span id="line-4087"> }</span> |
| <span class="source-line-no">4088</span><span id="line-4088"></span> |
| <span class="source-line-no">4089</span><span id="line-4089"> /**</span> |
| <span class="source-line-no">4090</span><span id="line-4090"> * Reckon the Cells to apply to WAL, memstore, and to return to the Client in passed column</span> |
| <span class="source-line-no">4091</span><span id="line-4091"> * family/Store. Does Get of current value and then adds passed in deltas for this Store</span> |
| <span class="source-line-no">4092</span><span id="line-4092"> * returning the result.</span> |
| <span class="source-line-no">4093</span><span id="line-4093"> * @param mutation The encompassing Mutation object</span> |
| <span class="source-line-no">4094</span><span id="line-4094"> * @param deltas Changes to apply to this Store; either increment amount or data to append</span> |
| <span class="source-line-no">4095</span><span id="line-4095"> * @param results In here we accumulate all the Cells we are to return to the client. If null,</span> |
| <span class="source-line-no">4096</span><span id="line-4096"> * client doesn't want results returned.</span> |
| <span class="source-line-no">4097</span><span id="line-4097"> * @return Resulting Cells after <code>deltas</code> have been applied to current values. Side</span> |
| <span class="source-line-no">4098</span><span id="line-4098"> * effect is our filling out of the <code>results</code> List.</span> |
| <span class="source-line-no">4099</span><span id="line-4099"> */</span> |
| <span class="source-line-no">4100</span><span id="line-4100"> private List<ExtendedCell> reckonDeltasByStore(HStore store, Mutation mutation, long now,</span> |
| <span class="source-line-no">4101</span><span id="line-4101"> List<ExtendedCell> deltas, List<ExtendedCell> results) throws IOException {</span> |
| <span class="source-line-no">4102</span><span id="line-4102"> assert mutation instanceof Increment || mutation instanceof Append;</span> |
| <span class="source-line-no">4103</span><span id="line-4103"> byte[] columnFamily = store.getColumnFamilyDescriptor().getName();</span> |
| <span class="source-line-no">4104</span><span id="line-4104"> List<Pair<ExtendedCell, ExtendedCell>> cellPairs = new ArrayList<>(deltas.size());</span> |
| <span class="source-line-no">4105</span><span id="line-4105"></span> |
| <span class="source-line-no">4106</span><span id="line-4106"> // Sort the cells so that they match the order that they appear in the Get results.</span> |
| <span class="source-line-no">4107</span><span id="line-4107"> // Otherwise, we won't be able to find the existing values if the cells are not specified</span> |
| <span class="source-line-no">4108</span><span id="line-4108"> // in order by the client since cells are in an array list.</span> |
| <span class="source-line-no">4109</span><span id="line-4109"> deltas.sort(store.getComparator());</span> |
| <span class="source-line-no">4110</span><span id="line-4110"></span> |
| <span class="source-line-no">4111</span><span id="line-4111"> // Get previous values for all columns in this family.</span> |
| <span class="source-line-no">4112</span><span id="line-4112"> Get get = new Get(mutation.getRow());</span> |
| <span class="source-line-no">4113</span><span id="line-4113"> for (ExtendedCell cell : deltas) {</span> |
| <span class="source-line-no">4114</span><span id="line-4114"> get.addColumn(columnFamily, CellUtil.cloneQualifier(cell));</span> |
| <span class="source-line-no">4115</span><span id="line-4115"> }</span> |
| <span class="source-line-no">4116</span><span id="line-4116"> TimeRange tr;</span> |
| <span class="source-line-no">4117</span><span id="line-4117"> if (mutation instanceof Increment) {</span> |
| <span class="source-line-no">4118</span><span id="line-4118"> tr = ((Increment) mutation).getTimeRange();</span> |
| <span class="source-line-no">4119</span><span id="line-4119"> } else {</span> |
| <span class="source-line-no">4120</span><span id="line-4120"> tr = ((Append) mutation).getTimeRange();</span> |
| <span class="source-line-no">4121</span><span id="line-4121"> }</span> |
| <span class="source-line-no">4122</span><span id="line-4122"></span> |
| <span class="source-line-no">4123</span><span id="line-4123"> if (tr != null) {</span> |
| <span class="source-line-no">4124</span><span id="line-4124"> get.setTimeRange(tr.getMin(), tr.getMax());</span> |
| <span class="source-line-no">4125</span><span id="line-4125"> }</span> |
| <span class="source-line-no">4126</span><span id="line-4126"></span> |
| <span class="source-line-no">4127</span><span id="line-4127"> try (RegionScanner scanner = region.getScanner(new Scan(get))) {</span> |
| <span class="source-line-no">4128</span><span id="line-4128"> // NOTE: Please don't use HRegion.get() instead,</span> |
| <span class="source-line-no">4129</span><span id="line-4129"> // because it will copy cells to heap. See HBASE-26036</span> |
| <span class="source-line-no">4130</span><span id="line-4130"> List<ExtendedCell> currentValues = new ArrayList<>();</span> |
| <span class="source-line-no">4131</span><span id="line-4131"> scanner.next((List) currentValues);</span> |
| <span class="source-line-no">4132</span><span id="line-4132"> // Iterate the input columns and update existing values if they were found, otherwise</span> |
| <span class="source-line-no">4133</span><span id="line-4133"> // add new column initialized to the delta amount</span> |
| <span class="source-line-no">4134</span><span id="line-4134"> int currentValuesIndex = 0;</span> |
| <span class="source-line-no">4135</span><span id="line-4135"> for (int i = 0; i < deltas.size(); i++) {</span> |
| <span class="source-line-no">4136</span><span id="line-4136"> ExtendedCell delta = deltas.get(i);</span> |
| <span class="source-line-no">4137</span><span id="line-4137"> ExtendedCell currentValue = null;</span> |
| <span class="source-line-no">4138</span><span id="line-4138"> if (</span> |
| <span class="source-line-no">4139</span><span id="line-4139"> currentValuesIndex < currentValues.size()</span> |
| <span class="source-line-no">4140</span><span id="line-4140"> && CellUtil.matchingQualifier(currentValues.get(currentValuesIndex), delta)</span> |
| <span class="source-line-no">4141</span><span id="line-4141"> ) {</span> |
| <span class="source-line-no">4142</span><span id="line-4142"> currentValue = currentValues.get(currentValuesIndex);</span> |
| <span class="source-line-no">4143</span><span id="line-4143"> if (i < (deltas.size() - 1) && !CellUtil.matchingQualifier(delta, deltas.get(i + 1))) {</span> |
| <span class="source-line-no">4144</span><span id="line-4144"> currentValuesIndex++;</span> |
| <span class="source-line-no">4145</span><span id="line-4145"> }</span> |
| <span class="source-line-no">4146</span><span id="line-4146"> }</span> |
| <span class="source-line-no">4147</span><span id="line-4147"> // Switch on whether this an increment or an append building the new Cell to apply.</span> |
| <span class="source-line-no">4148</span><span id="line-4148"> ExtendedCell newCell;</span> |
| <span class="source-line-no">4149</span><span id="line-4149"> if (mutation instanceof Increment) {</span> |
| <span class="source-line-no">4150</span><span id="line-4150"> long deltaAmount = getLongValue(delta);</span> |
| <span class="source-line-no">4151</span><span id="line-4151"> final long newValue =</span> |
| <span class="source-line-no">4152</span><span id="line-4152"> currentValue == null ? deltaAmount : getLongValue(currentValue) + deltaAmount;</span> |
| <span class="source-line-no">4153</span><span id="line-4153"> newCell = reckonDelta(delta, currentValue, columnFamily, now, mutation,</span> |
| <span class="source-line-no">4154</span><span id="line-4154"> (oldCell) -> Bytes.toBytes(newValue));</span> |
| <span class="source-line-no">4155</span><span id="line-4155"> } else {</span> |
| <span class="source-line-no">4156</span><span id="line-4156"> newCell = reckonDelta(delta, currentValue, columnFamily, now, mutation,</span> |
| <span class="source-line-no">4157</span><span id="line-4157"> (oldCell) -> ByteBuffer</span> |
| <span class="source-line-no">4158</span><span id="line-4158"> .wrap(new byte[delta.getValueLength() + oldCell.getValueLength()])</span> |
| <span class="source-line-no">4159</span><span id="line-4159"> .put(oldCell.getValueArray(), oldCell.getValueOffset(), oldCell.getValueLength())</span> |
| <span class="source-line-no">4160</span><span id="line-4160"> .put(delta.getValueArray(), delta.getValueOffset(), delta.getValueLength())</span> |
| <span class="source-line-no">4161</span><span id="line-4161"> .array());</span> |
| <span class="source-line-no">4162</span><span id="line-4162"> }</span> |
| <span class="source-line-no">4163</span><span id="line-4163"> if (region.maxCellSize > 0) {</span> |
| <span class="source-line-no">4164</span><span id="line-4164"> int newCellSize = PrivateCellUtil.estimatedSerializedSizeOf(newCell);</span> |
| <span class="source-line-no">4165</span><span id="line-4165"> if (newCellSize > region.maxCellSize) {</span> |
| <span class="source-line-no">4166</span><span id="line-4166"> String msg = "Cell with size " + newCellSize + " exceeds limit of "</span> |
| <span class="source-line-no">4167</span><span id="line-4167"> + region.maxCellSize + " bytes in region " + this;</span> |
| <span class="source-line-no">4168</span><span id="line-4168"> LOG.debug(msg);</span> |
| <span class="source-line-no">4169</span><span id="line-4169"> throw new DoNotRetryIOException(msg);</span> |
| <span class="source-line-no">4170</span><span id="line-4170"> }</span> |
| <span class="source-line-no">4171</span><span id="line-4171"> }</span> |
| <span class="source-line-no">4172</span><span id="line-4172"> cellPairs.add(new Pair<>(currentValue, newCell));</span> |
| <span class="source-line-no">4173</span><span id="line-4173"> // Add to results to get returned to the Client. If null, cilent does not want results.</span> |
| <span class="source-line-no">4174</span><span id="line-4174"> if (results != null) {</span> |
| <span class="source-line-no">4175</span><span id="line-4175"> results.add(newCell);</span> |
| <span class="source-line-no">4176</span><span id="line-4176"> }</span> |
| <span class="source-line-no">4177</span><span id="line-4177"> }</span> |
| <span class="source-line-no">4178</span><span id="line-4178"> // Give coprocessors a chance to update the new cells before apply to WAL or memstore</span> |
| <span class="source-line-no">4179</span><span id="line-4179"> if (region.coprocessorHost != null) {</span> |
| <span class="source-line-no">4180</span><span id="line-4180"> // Here the operation must be increment or append.</span> |
| <span class="source-line-no">4181</span><span id="line-4181"> cellPairs = mutation instanceof Increment</span> |
| <span class="source-line-no">4182</span><span id="line-4182"> ? region.coprocessorHost.postIncrementBeforeWAL(mutation, (List) cellPairs)</span> |
| <span class="source-line-no">4183</span><span id="line-4183"> : region.coprocessorHost.postAppendBeforeWAL(mutation, (List) cellPairs);</span> |
| <span class="source-line-no">4184</span><span id="line-4184"> }</span> |
| <span class="source-line-no">4185</span><span id="line-4185"> }</span> |
| <span class="source-line-no">4186</span><span id="line-4186"> return cellPairs.stream().map(Pair::getSecond).collect(Collectors.toList());</span> |
| <span class="source-line-no">4187</span><span id="line-4187"> }</span> |
| <span class="source-line-no">4188</span><span id="line-4188"></span> |
| <span class="source-line-no">4189</span><span id="line-4189"> private static ExtendedCell reckonDelta(final ExtendedCell delta,</span> |
| <span class="source-line-no">4190</span><span id="line-4190"> final ExtendedCell currentCell, final byte[] columnFamily, final long now, Mutation mutation,</span> |
| <span class="source-line-no">4191</span><span id="line-4191"> Function<ExtendedCell, byte[]> supplier) throws IOException {</span> |
| <span class="source-line-no">4192</span><span id="line-4192"> // Forward any tags found on the delta.</span> |
| <span class="source-line-no">4193</span><span id="line-4193"> List<Tag> tags = TagUtil.carryForwardTags(delta);</span> |
| <span class="source-line-no">4194</span><span id="line-4194"> if (currentCell != null) {</span> |
| <span class="source-line-no">4195</span><span id="line-4195"> tags = TagUtil.carryForwardTags(tags, currentCell);</span> |
| <span class="source-line-no">4196</span><span id="line-4196"> tags = TagUtil.carryForwardTTLTag(tags, mutation.getTTL());</span> |
| <span class="source-line-no">4197</span><span id="line-4197"> byte[] newValue = supplier.apply(currentCell);</span> |
| <span class="source-line-no">4198</span><span id="line-4198"> return ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)</span> |
| <span class="source-line-no">4199</span><span id="line-4199"> .setRow(mutation.getRow(), 0, mutation.getRow().length)</span> |
| <span class="source-line-no">4200</span><span id="line-4200"> .setFamily(columnFamily, 0, columnFamily.length)</span> |
| <span class="source-line-no">4201</span><span id="line-4201"> // copy the qualifier if the cell is located in shared memory.</span> |
| <span class="source-line-no">4202</span><span id="line-4202"> .setQualifier(CellUtil.cloneQualifier(delta))</span> |
| <span class="source-line-no">4203</span><span id="line-4203"> .setTimestamp(Math.max(currentCell.getTimestamp() + 1, now))</span> |
| <span class="source-line-no">4204</span><span id="line-4204"> .setType(KeyValue.Type.Put.getCode()).setValue(newValue, 0, newValue.length)</span> |
| <span class="source-line-no">4205</span><span id="line-4205"> .setTags(TagUtil.fromList(tags)).build();</span> |
| <span class="source-line-no">4206</span><span id="line-4206"> } else {</span> |
| <span class="source-line-no">4207</span><span id="line-4207"> tags = TagUtil.carryForwardTTLTag(tags, mutation.getTTL());</span> |
| <span class="source-line-no">4208</span><span id="line-4208"> PrivateCellUtil.updateLatestStamp(delta, now);</span> |
| <span class="source-line-no">4209</span><span id="line-4209"> ExtendedCell deltaCell = (ExtendedCell) delta;</span> |
| <span class="source-line-no">4210</span><span id="line-4210"> return CollectionUtils.isEmpty(tags)</span> |
| <span class="source-line-no">4211</span><span id="line-4211"> ? deltaCell</span> |
| <span class="source-line-no">4212</span><span id="line-4212"> : PrivateCellUtil.createCell(deltaCell, tags);</span> |
| <span class="source-line-no">4213</span><span id="line-4213"> }</span> |
| <span class="source-line-no">4214</span><span id="line-4214"> }</span> |
| <span class="source-line-no">4215</span><span id="line-4215"></span> |
| <span class="source-line-no">4216</span><span id="line-4216"> /** Returns Get the long out of the passed in Cell */</span> |
| <span class="source-line-no">4217</span><span id="line-4217"> private static long getLongValue(final Cell cell) throws DoNotRetryIOException {</span> |
| <span class="source-line-no">4218</span><span id="line-4218"> int len = cell.getValueLength();</span> |
| <span class="source-line-no">4219</span><span id="line-4219"> if (len != Bytes.SIZEOF_LONG) {</span> |
| <span class="source-line-no">4220</span><span id="line-4220"> // throw DoNotRetryIOException instead of IllegalArgumentException</span> |
| <span class="source-line-no">4221</span><span id="line-4221"> throw new DoNotRetryIOException("Field is not a long, it's " + len + " bytes wide");</span> |
| <span class="source-line-no">4222</span><span id="line-4222"> }</span> |
| <span class="source-line-no">4223</span><span id="line-4223"> return PrivateCellUtil.getValueAsLong(cell);</span> |
| <span class="source-line-no">4224</span><span id="line-4224"> }</span> |
| <span class="source-line-no">4225</span><span id="line-4225"></span> |
| <span class="source-line-no">4226</span><span id="line-4226"> @Override</span> |
| <span class="source-line-no">4227</span><span id="line-4227"> public List<Pair<NonceKey, WALEdit>></span> |
| <span class="source-line-no">4228</span><span id="line-4228"> buildWALEdits(final MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {</span> |
| <span class="source-line-no">4229</span><span id="line-4229"> List<Pair<NonceKey, WALEdit>> walEdits = super.buildWALEdits(miniBatchOp);</span> |
| <span class="source-line-no">4230</span><span id="line-4230"> // for MutationBatchOperation, more than one nonce is not allowed</span> |
| <span class="source-line-no">4231</span><span id="line-4231"> if (walEdits.size() > 1) {</span> |
| <span class="source-line-no">4232</span><span id="line-4232"> throw new IOException("Found multiple nonce keys per batch!");</span> |
| <span class="source-line-no">4233</span><span id="line-4233"> }</span> |
| <span class="source-line-no">4234</span><span id="line-4234"> return walEdits;</span> |
| <span class="source-line-no">4235</span><span id="line-4235"> }</span> |
| <span class="source-line-no">4236</span><span id="line-4236"></span> |
| <span class="source-line-no">4237</span><span id="line-4237"> /**</span> |
| <span class="source-line-no">4238</span><span id="line-4238"> * Here is for HBASE-26993,in order to make the new framework for region replication could work</span> |
| <span class="source-line-no">4239</span><span id="line-4239"> * for SKIP_WAL, we save the {@link Mutation} which {@link Mutation#getDurability} is</span> |
| <span class="source-line-no">4240</span><span id="line-4240"> * {@link Durability#SKIP_WAL} in miniBatchOp.</span> |
| <span class="source-line-no">4241</span><span id="line-4241"> */</span> |
| <span class="source-line-no">4242</span><span id="line-4242"> @Override</span> |
| <span class="source-line-no">4243</span><span id="line-4243"> protected void cacheSkipWALMutationForRegionReplication(</span> |
| <span class="source-line-no">4244</span><span id="line-4244"> MiniBatchOperationInProgress<Mutation> miniBatchOp,</span> |
| <span class="source-line-no">4245</span><span id="line-4245"> List<Pair<NonceKey, WALEdit>> nonceKeyAndWALEdits,</span> |
| <span class="source-line-no">4246</span><span id="line-4246"> Map<byte[], List<ExtendedCell>> familyCellMap) {</span> |
| <span class="source-line-no">4247</span><span id="line-4247"> if (!this.regionReplicateEnable) {</span> |
| <span class="source-line-no">4248</span><span id="line-4248"> return;</span> |
| <span class="source-line-no">4249</span><span id="line-4249"> }</span> |
| <span class="source-line-no">4250</span><span id="line-4250"></span> |
| <span class="source-line-no">4251</span><span id="line-4251"> WALEdit walEditForReplicateIfExistsSkipWAL =</span> |
| <span class="source-line-no">4252</span><span id="line-4252"> miniBatchOp.getWalEditForReplicateIfExistsSkipWAL();</span> |
| <span class="source-line-no">4253</span><span id="line-4253"> /**</span> |
| <span class="source-line-no">4254</span><span id="line-4254"> * When there is a SKIP_WAL {@link Mutation},we create a new {@link WALEdit} for replicating</span> |
| <span class="source-line-no">4255</span><span id="line-4255"> * to region replica,first we fill the existing {@link WALEdit} to it and then add the</span> |
| <span class="source-line-no">4256</span><span id="line-4256"> * {@link Mutation} which is SKIP_WAL to it.</span> |
| <span class="source-line-no">4257</span><span id="line-4257"> */</span> |
| <span class="source-line-no">4258</span><span id="line-4258"> if (walEditForReplicateIfExistsSkipWAL == null) {</span> |
| <span class="source-line-no">4259</span><span id="line-4259"> walEditForReplicateIfExistsSkipWAL =</span> |
| <span class="source-line-no">4260</span><span id="line-4260"> this.createWALEditForReplicateSkipWAL(miniBatchOp, nonceKeyAndWALEdits);</span> |
| <span class="source-line-no">4261</span><span id="line-4261"> miniBatchOp.setWalEditForReplicateIfExistsSkipWAL(walEditForReplicateIfExistsSkipWAL);</span> |
| <span class="source-line-no">4262</span><span id="line-4262"> }</span> |
| <span class="source-line-no">4263</span><span id="line-4263"> walEditForReplicateIfExistsSkipWAL.add((Map) familyCellMap);</span> |
| <span class="source-line-no">4264</span><span id="line-4264"></span> |
| <span class="source-line-no">4265</span><span id="line-4265"> }</span> |
| <span class="source-line-no">4266</span><span id="line-4266"></span> |
| <span class="source-line-no">4267</span><span id="line-4267"> private WALEdit createWALEditForReplicateSkipWAL(</span> |
| <span class="source-line-no">4268</span><span id="line-4268"> MiniBatchOperationInProgress<Mutation> miniBatchOp,</span> |
| <span class="source-line-no">4269</span><span id="line-4269"> List<Pair<NonceKey, WALEdit>> nonceKeyAndWALEdits) {</span> |
| <span class="source-line-no">4270</span><span id="line-4270"> if (nonceKeyAndWALEdits.isEmpty()) {</span> |
| <span class="source-line-no">4271</span><span id="line-4271"> return this.createWALEdit(miniBatchOp);</span> |
| <span class="source-line-no">4272</span><span id="line-4272"> }</span> |
| <span class="source-line-no">4273</span><span id="line-4273"> // for MutationBatchOperation, more than one nonce is not allowed</span> |
| <span class="source-line-no">4274</span><span id="line-4274"> assert nonceKeyAndWALEdits.size() == 1;</span> |
| <span class="source-line-no">4275</span><span id="line-4275"> WALEdit currentWALEdit = nonceKeyAndWALEdits.get(0).getSecond();</span> |
| <span class="source-line-no">4276</span><span id="line-4276"> return new WALEdit(currentWALEdit);</span> |
| <span class="source-line-no">4277</span><span id="line-4277"> }</span> |
| <span class="source-line-no">4278</span><span id="line-4278"></span> |
| <span class="source-line-no">4279</span><span id="line-4279"> @Override</span> |
| <span class="source-line-no">4280</span><span id="line-4280"> protected void addNonSkipWALMutationsToWALEdit(</span> |
| <span class="source-line-no">4281</span><span id="line-4281"> final MiniBatchOperationInProgress<Mutation> miniBatchOp, WALEdit walEdit,</span> |
| <span class="source-line-no">4282</span><span id="line-4282"> List<Cell> cellsFromCP, Map<byte[], List<ExtendedCell>> familyCellMap) {</span> |
| <span class="source-line-no">4283</span><span id="line-4283"></span> |
| <span class="source-line-no">4284</span><span id="line-4284"> super.addNonSkipWALMutationsToWALEdit(miniBatchOp, walEdit, cellsFromCP, familyCellMap);</span> |
| <span class="source-line-no">4285</span><span id="line-4285"> WALEdit walEditForReplicateIfExistsSkipWAL =</span> |
| <span class="source-line-no">4286</span><span id="line-4286"> miniBatchOp.getWalEditForReplicateIfExistsSkipWAL();</span> |
| <span class="source-line-no">4287</span><span id="line-4287"> if (walEditForReplicateIfExistsSkipWAL == null) {</span> |
| <span class="source-line-no">4288</span><span id="line-4288"> return;</span> |
| <span class="source-line-no">4289</span><span id="line-4289"> }</span> |
| <span class="source-line-no">4290</span><span id="line-4290"> /**</span> |
| <span class="source-line-no">4291</span><span id="line-4291"> * When walEditForReplicateIfExistsSkipWAL is not null,it means there exists SKIP_WAL</span> |
| <span class="source-line-no">4292</span><span id="line-4292"> * {@link Mutation} and we create a new {@link WALEdit} in</span> |
| <span class="source-line-no">4293</span><span id="line-4293"> * {@link MutationBatchOperation#cacheSkipWALMutationForReplicateRegionReplica} for</span> |
| <span class="source-line-no">4294</span><span id="line-4294"> * replicating to region replica, so here we also add non SKIP_WAL{@link Mutation}s to</span> |
| <span class="source-line-no">4295</span><span id="line-4295"> * walEditForReplicateIfExistsSkipWAL.</span> |
| <span class="source-line-no">4296</span><span id="line-4296"> */</span> |
| <span class="source-line-no">4297</span><span id="line-4297"> doAddCellsToWALEdit(walEditForReplicateIfExistsSkipWAL, cellsFromCP, familyCellMap);</span> |
| <span class="source-line-no">4298</span><span id="line-4298"> }</span> |
| <span class="source-line-no">4299</span><span id="line-4299"></span> |
| <span class="source-line-no">4300</span><span id="line-4300"> @Override</span> |
| <span class="source-line-no">4301</span><span id="line-4301"> public WriteEntry writeMiniBatchOperationsToMemStore(</span> |
| <span class="source-line-no">4302</span><span id="line-4302"> final MiniBatchOperationInProgress<Mutation> miniBatchOp, @Nullable WriteEntry writeEntry,</span> |
| <span class="source-line-no">4303</span><span id="line-4303"> long now) throws IOException {</span> |
| <span class="source-line-no">4304</span><span id="line-4304"> boolean newWriteEntry = false;</span> |
| <span class="source-line-no">4305</span><span id="line-4305"> if (writeEntry == null) {</span> |
| <span class="source-line-no">4306</span><span id="line-4306"> writeEntry = region.mvcc.begin();</span> |
| <span class="source-line-no">4307</span><span id="line-4307"> newWriteEntry = true;</span> |
| <span class="source-line-no">4308</span><span id="line-4308"> }</span> |
| <span class="source-line-no">4309</span><span id="line-4309"> super.writeMiniBatchOperationsToMemStore(miniBatchOp, writeEntry.getWriteNumber());</span> |
| <span class="source-line-no">4310</span><span id="line-4310"> if (newWriteEntry) {</span> |
| <span class="source-line-no">4311</span><span id="line-4311"> /**</span> |
| <span class="source-line-no">4312</span><span id="line-4312"> * Here is for HBASE-26993 case 2,all {@link Mutation}s are {@link Durability#SKIP_WAL}. In</span> |
| <span class="source-line-no">4313</span><span id="line-4313"> * order to make the new framework for region replication could work for SKIP_WAL,because</span> |
| <span class="source-line-no">4314</span><span id="line-4314"> * there is no {@link RegionReplicationSink#add} attached in {@link HRegion#doWALAppend},so</span> |
| <span class="source-line-no">4315</span><span id="line-4315"> * here we get {@link WALEdit} from</span> |
| <span class="source-line-no">4316</span><span id="line-4316"> * {@link MiniBatchOperationInProgress#getWalEditForReplicateIfExistsSkipWAL} and attach</span> |
| <span class="source-line-no">4317</span><span id="line-4317"> * {@link RegionReplicationSink#add} to the new mvcc writeEntry.</span> |
| <span class="source-line-no">4318</span><span id="line-4318"> */</span> |
| <span class="source-line-no">4319</span><span id="line-4319"> attachRegionReplicationToMVCCEntry(miniBatchOp, writeEntry, now);</span> |
| <span class="source-line-no">4320</span><span id="line-4320"> }</span> |
| <span class="source-line-no">4321</span><span id="line-4321"> return writeEntry;</span> |
| <span class="source-line-no">4322</span><span id="line-4322"> }</span> |
| <span class="source-line-no">4323</span><span id="line-4323"></span> |
| <span class="source-line-no">4324</span><span id="line-4324"> private WALKeyImpl createWALKey(long now) {</span> |
| <span class="source-line-no">4325</span><span id="line-4325"> // for MutationBatchOperation,isReplay is false.</span> |
| <span class="source-line-no">4326</span><span id="line-4326"> return this.region.createWALKeyForWALAppend(false, this, now, this.nonceGroup, this.nonce);</span> |
| <span class="source-line-no">4327</span><span id="line-4327"> }</span> |
| <span class="source-line-no">4328</span><span id="line-4328"></span> |
| <span class="source-line-no">4329</span><span id="line-4329"> /**</span> |
| <span class="source-line-no">4330</span><span id="line-4330"> * Create {@link WALKeyImpl} and get {@link WALEdit} from miniBatchOp and attach</span> |
| <span class="source-line-no">4331</span><span id="line-4331"> * {@link RegionReplicationSink#add} to the mvccWriteEntry.</span> |
| <span class="source-line-no">4332</span><span id="line-4332"> */</span> |
| <span class="source-line-no">4333</span><span id="line-4333"> private void attachRegionReplicationToMVCCEntry(</span> |
| <span class="source-line-no">4334</span><span id="line-4334"> final MiniBatchOperationInProgress<Mutation> miniBatchOp, WriteEntry mvccWriteEntry, long now)</span> |
| <span class="source-line-no">4335</span><span id="line-4335"> throws IOException {</span> |
| <span class="source-line-no">4336</span><span id="line-4336"> if (!this.regionReplicateEnable) {</span> |
| <span class="source-line-no">4337</span><span id="line-4337"> return;</span> |
| <span class="source-line-no">4338</span><span id="line-4338"> }</span> |
| <span class="source-line-no">4339</span><span id="line-4339"> assert !mvccWriteEntry.getCompletionAction().isPresent();</span> |
| <span class="source-line-no">4340</span><span id="line-4340"></span> |
| <span class="source-line-no">4341</span><span id="line-4341"> final WALKeyImpl walKey = this.createWALKey(now);</span> |
| <span class="source-line-no">4342</span><span id="line-4342"> walKey.setWriteEntry(mvccWriteEntry);</span> |
| <span class="source-line-no">4343</span><span id="line-4343"> region.doAttachReplicateRegionReplicaAction(walKey,</span> |
| <span class="source-line-no">4344</span><span id="line-4344"> miniBatchOp.getWalEditForReplicateIfExistsSkipWAL(), mvccWriteEntry);</span> |
| <span class="source-line-no">4345</span><span id="line-4345"> }</span> |
| <span class="source-line-no">4346</span><span id="line-4346"></span> |
| <span class="source-line-no">4347</span><span id="line-4347"> @Override</span> |
| <span class="source-line-no">4348</span><span id="line-4348"> public void completeMiniBatchOperations(</span> |
| <span class="source-line-no">4349</span><span id="line-4349"> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final WriteEntry writeEntry)</span> |
| <span class="source-line-no">4350</span><span id="line-4350"> throws IOException {</span> |
| <span class="source-line-no">4351</span><span id="line-4351"> // TODO: can it be done after completing mvcc?</span> |
| <span class="source-line-no">4352</span><span id="line-4352"> // calling the post CP hook for batch mutation</span> |
| <span class="source-line-no">4353</span><span id="line-4353"> if (region.coprocessorHost != null) {</span> |
| <span class="source-line-no">4354</span><span id="line-4354"> region.coprocessorHost.postBatchMutate(miniBatchOp);</span> |
| <span class="source-line-no">4355</span><span id="line-4355"> }</span> |
| <span class="source-line-no">4356</span><span id="line-4356"> super.completeMiniBatchOperations(miniBatchOp, writeEntry);</span> |
| <span class="source-line-no">4357</span><span id="line-4357"></span> |
| <span class="source-line-no">4358</span><span id="line-4358"> if (nonce != HConstants.NO_NONCE) {</span> |
| <span class="source-line-no">4359</span><span id="line-4359"> if (region.rsServices != null && region.rsServices.getNonceManager() != null) {</span> |
| <span class="source-line-no">4360</span><span id="line-4360"> region.rsServices.getNonceManager().addMvccToOperationContext(nonceGroup, nonce,</span> |
| <span class="source-line-no">4361</span><span id="line-4361"> writeEntry.getWriteNumber());</span> |
| <span class="source-line-no">4362</span><span id="line-4362"> }</span> |
| <span class="source-line-no">4363</span><span id="line-4363"> }</span> |
| <span class="source-line-no">4364</span><span id="line-4364"> }</span> |
| <span class="source-line-no">4365</span><span id="line-4365"></span> |
| <span class="source-line-no">4366</span><span id="line-4366"> @Override</span> |
| <span class="source-line-no">4367</span><span id="line-4367"> public void doPostOpCleanupForMiniBatch(MiniBatchOperationInProgress<Mutation> miniBatchOp,</span> |
| <span class="source-line-no">4368</span><span id="line-4368"> final WALEdit walEdit, boolean success) throws IOException {</span> |
| <span class="source-line-no">4369</span><span id="line-4369"></span> |
| <span class="source-line-no">4370</span><span id="line-4370"> super.doPostOpCleanupForMiniBatch(miniBatchOp, walEdit, success);</span> |
| <span class="source-line-no">4371</span><span id="line-4371"> if (miniBatchOp != null) {</span> |
| <span class="source-line-no">4372</span><span id="line-4372"> // synced so that the coprocessor contract is adhered to.</span> |
| <span class="source-line-no">4373</span><span id="line-4373"> if (region.coprocessorHost != null) {</span> |
| <span class="source-line-no">4374</span><span id="line-4374"> visitBatchOperations(false, miniBatchOp.getLastIndexExclusive(), (int i) -> {</span> |
| <span class="source-line-no">4375</span><span id="line-4375"> // only for successful puts/deletes/increments/appends</span> |
| <span class="source-line-no">4376</span><span id="line-4376"> if (retCodeDetails[i].getOperationStatusCode() == OperationStatusCode.SUCCESS) {</span> |
| <span class="source-line-no">4377</span><span id="line-4377"> Mutation m = getMutation(i);</span> |
| <span class="source-line-no">4378</span><span id="line-4378"> if (m instanceof Put) {</span> |
| <span class="source-line-no">4379</span><span id="line-4379"> region.coprocessorHost.postPut((Put) m, walEdit);</span> |
| <span class="source-line-no">4380</span><span id="line-4380"> } else if (m instanceof Delete) {</span> |
| <span class="source-line-no">4381</span><span id="line-4381"> region.coprocessorHost.postDelete((Delete) m, walEdit);</span> |
| <span class="source-line-no">4382</span><span id="line-4382"> } else if (m instanceof Increment) {</span> |
| <span class="source-line-no">4383</span><span id="line-4383"> Result result =</span> |
| <span class="source-line-no">4384</span><span id="line-4384"> region.getCoprocessorHost().postIncrement((Increment) m, results[i], walEdit);</span> |
| <span class="source-line-no">4385</span><span id="line-4385"> if (result != results[i]) {</span> |
| <span class="source-line-no">4386</span><span id="line-4386"> retCodeDetails[i] =</span> |
| <span class="source-line-no">4387</span><span id="line-4387"> new OperationStatus(retCodeDetails[i].getOperationStatusCode(), result);</span> |
| <span class="source-line-no">4388</span><span id="line-4388"> }</span> |
| <span class="source-line-no">4389</span><span id="line-4389"> } else if (m instanceof Append) {</span> |
| <span class="source-line-no">4390</span><span id="line-4390"> Result result =</span> |
| <span class="source-line-no">4391</span><span id="line-4391"> region.getCoprocessorHost().postAppend((Append) m, results[i], walEdit);</span> |
| <span class="source-line-no">4392</span><span id="line-4392"> if (result != results[i]) {</span> |
| <span class="source-line-no">4393</span><span id="line-4393"> retCodeDetails[i] =</span> |
| <span class="source-line-no">4394</span><span id="line-4394"> new OperationStatus(retCodeDetails[i].getOperationStatusCode(), result);</span> |
| <span class="source-line-no">4395</span><span id="line-4395"> }</span> |
| <span class="source-line-no">4396</span><span id="line-4396"> }</span> |
| <span class="source-line-no">4397</span><span id="line-4397"> }</span> |
| <span class="source-line-no">4398</span><span id="line-4398"> return true;</span> |
| <span class="source-line-no">4399</span><span id="line-4399"> });</span> |
| <span class="source-line-no">4400</span><span id="line-4400"> }</span> |
| <span class="source-line-no">4401</span><span id="line-4401"></span> |
| <span class="source-line-no">4402</span><span id="line-4402"> // For nonce operations</span> |
| <span class="source-line-no">4403</span><span id="line-4403"> if (canProceed && nonce != HConstants.NO_NONCE) {</span> |
| <span class="source-line-no">4404</span><span id="line-4404"> boolean[] areAllIncrementsAndAppendsSuccessful = new boolean[] { true };</span> |
| <span class="source-line-no">4405</span><span id="line-4405"> visitBatchOperations(false, miniBatchOp.getLastIndexExclusive(), (int i) -> {</span> |
| <span class="source-line-no">4406</span><span id="line-4406"> Mutation mutation = getMutation(i);</span> |
| <span class="source-line-no">4407</span><span id="line-4407"> if (mutation instanceof Increment || mutation instanceof Append) {</span> |
| <span class="source-line-no">4408</span><span id="line-4408"> if (retCodeDetails[i].getOperationStatusCode() != OperationStatusCode.SUCCESS) {</span> |
| <span class="source-line-no">4409</span><span id="line-4409"> areAllIncrementsAndAppendsSuccessful[0] = false;</span> |
| <span class="source-line-no">4410</span><span id="line-4410"> return false;</span> |
| <span class="source-line-no">4411</span><span id="line-4411"> }</span> |
| <span class="source-line-no">4412</span><span id="line-4412"> }</span> |
| <span class="source-line-no">4413</span><span id="line-4413"> return true;</span> |
| <span class="source-line-no">4414</span><span id="line-4414"> });</span> |
| <span class="source-line-no">4415</span><span id="line-4415"> endNonceOperation(areAllIncrementsAndAppendsSuccessful[0]);</span> |
| <span class="source-line-no">4416</span><span id="line-4416"> }</span> |
| <span class="source-line-no">4417</span><span id="line-4417"></span> |
| <span class="source-line-no">4418</span><span id="line-4418"> // See if the column families were consistent through the whole thing.</span> |
| <span class="source-line-no">4419</span><span id="line-4419"> // if they were then keep them. If they were not then pass a null.</span> |
| <span class="source-line-no">4420</span><span id="line-4420"> // null will be treated as unknown.</span> |
| <span class="source-line-no">4421</span><span id="line-4421"> // Total time taken might be involving Puts, Deletes, Increments and Appends.</span> |
| <span class="source-line-no">4422</span><span id="line-4422"> // Split the time for puts and deletes based on the total number of Puts, Deletes,</span> |
| <span class="source-line-no">4423</span><span id="line-4423"> // Increments and Appends.</span> |
| <span class="source-line-no">4424</span><span id="line-4424"> if (region.metricsRegion != null) {</span> |
| <span class="source-line-no">4425</span><span id="line-4425"> if (miniBatchOp.getNumOfPuts() > 0) {</span> |
| <span class="source-line-no">4426</span><span id="line-4426"> // There were some Puts in the batch.</span> |
| <span class="source-line-no">4427</span><span id="line-4427"> region.metricsRegion.updatePut();</span> |
| <span class="source-line-no">4428</span><span id="line-4428"> }</span> |
| <span class="source-line-no">4429</span><span id="line-4429"> if (miniBatchOp.getNumOfDeletes() > 0) {</span> |
| <span class="source-line-no">4430</span><span id="line-4430"> // There were some Deletes in the batch.</span> |
| <span class="source-line-no">4431</span><span id="line-4431"> region.metricsRegion.updateDelete();</span> |
| <span class="source-line-no">4432</span><span id="line-4432"> }</span> |
| <span class="source-line-no">4433</span><span id="line-4433"> if (miniBatchOp.getNumOfIncrements() > 0) {</span> |
| <span class="source-line-no">4434</span><span id="line-4434"> // There were some Increments in the batch.</span> |
| <span class="source-line-no">4435</span><span id="line-4435"> region.metricsRegion.updateIncrement();</span> |
| <span class="source-line-no">4436</span><span id="line-4436"> }</span> |
| <span class="source-line-no">4437</span><span id="line-4437"> if (miniBatchOp.getNumOfAppends() > 0) {</span> |
| <span class="source-line-no">4438</span><span id="line-4438"> // There were some Appends in the batch.</span> |
| <span class="source-line-no">4439</span><span id="line-4439"> region.metricsRegion.updateAppend();</span> |
| <span class="source-line-no">4440</span><span id="line-4440"> }</span> |
| <span class="source-line-no">4441</span><span id="line-4441"> }</span> |
| <span class="source-line-no">4442</span><span id="line-4442"> }</span> |
| <span class="source-line-no">4443</span><span id="line-4443"></span> |
| <span class="source-line-no">4444</span><span id="line-4444"> if (region.coprocessorHost != null) {</span> |
| <span class="source-line-no">4445</span><span id="line-4445"> // call the coprocessor hook to do any finalization steps after the put is done</span> |
| <span class="source-line-no">4446</span><span id="line-4446"> region.coprocessorHost.postBatchMutateIndispensably(</span> |
| <span class="source-line-no">4447</span><span id="line-4447"> miniBatchOp != null ? miniBatchOp : createMiniBatch(size(), 0), success);</span> |
| <span class="source-line-no">4448</span><span id="line-4448"> }</span> |
| <span class="source-line-no">4449</span><span id="line-4449"> }</span> |
| <span class="source-line-no">4450</span><span id="line-4450"></span> |
| <span class="source-line-no">4451</span><span id="line-4451"> /**</span> |
| <span class="source-line-no">4452</span><span id="line-4452"> * Runs prePut/preDelete/preIncrement/preAppend coprocessor hook for input mutation in a batch</span> |
| <span class="source-line-no">4453</span><span id="line-4453"> * @param metrics Array of 2 ints. index 0: count of puts, index 1: count of deletes, index 2:</span> |
| <span class="source-line-no">4454</span><span id="line-4454"> * count of increments and 3: count of appends</span> |
| <span class="source-line-no">4455</span><span id="line-4455"> */</span> |
| <span class="source-line-no">4456</span><span id="line-4456"> private void callPreMutateCPHook(int index, final WALEdit walEdit, final int[] metrics)</span> |
| <span class="source-line-no">4457</span><span id="line-4457"> throws IOException {</span> |
| <span class="source-line-no">4458</span><span id="line-4458"> Mutation m = getMutation(index);</span> |
| <span class="source-line-no">4459</span><span id="line-4459"> if (m instanceof Put) {</span> |
| <span class="source-line-no">4460</span><span id="line-4460"> if (region.coprocessorHost.prePut((Put) m, walEdit)) {</span> |
| <span class="source-line-no">4461</span><span id="line-4461"> // pre hook says skip this Put</span> |
| <span class="source-line-no">4462</span><span id="line-4462"> // mark as success and skip in doMiniBatchMutation</span> |
| <span class="source-line-no">4463</span><span id="line-4463"> metrics[0]++;</span> |
| <span class="source-line-no">4464</span><span id="line-4464"> retCodeDetails[index] = OperationStatus.SUCCESS;</span> |
| <span class="source-line-no">4465</span><span id="line-4465"> }</span> |
| <span class="source-line-no">4466</span><span id="line-4466"> } else if (m instanceof Delete) {</span> |
| <span class="source-line-no">4467</span><span id="line-4467"> Delete curDel = (Delete) m;</span> |
| <span class="source-line-no">4468</span><span id="line-4468"> if (curDel.getFamilyCellMap().isEmpty()) {</span> |
| <span class="source-line-no">4469</span><span id="line-4469"> // handle deleting a row case</span> |
| <span class="source-line-no">4470</span><span id="line-4470"> // TODO: prepareDelete() has been called twice, before and after preDelete() CP hook.</span> |
| <span class="source-line-no">4471</span><span id="line-4471"> // Can this be avoided?</span> |
| <span class="source-line-no">4472</span><span id="line-4472"> region.prepareDelete(curDel);</span> |
| <span class="source-line-no">4473</span><span id="line-4473"> }</span> |
| <span class="source-line-no">4474</span><span id="line-4474"> if (region.coprocessorHost.preDelete(curDel, walEdit)) {</span> |
| <span class="source-line-no">4475</span><span id="line-4475"> // pre hook says skip this Delete</span> |
| <span class="source-line-no">4476</span><span id="line-4476"> // mark as success and skip in doMiniBatchMutation</span> |
| <span class="source-line-no">4477</span><span id="line-4477"> metrics[1]++;</span> |
| <span class="source-line-no">4478</span><span id="line-4478"> retCodeDetails[index] = OperationStatus.SUCCESS;</span> |
| <span class="source-line-no">4479</span><span id="line-4479"> }</span> |
| <span class="source-line-no">4480</span><span id="line-4480"> } else if (m instanceof Increment) {</span> |
| <span class="source-line-no">4481</span><span id="line-4481"> Increment increment = (Increment) m;</span> |
| <span class="source-line-no">4482</span><span id="line-4482"> Result result = region.coprocessorHost.preIncrement(increment, walEdit);</span> |
| <span class="source-line-no">4483</span><span id="line-4483"> if (result != null) {</span> |
| <span class="source-line-no">4484</span><span id="line-4484"> // pre hook says skip this Increment</span> |
| <span class="source-line-no">4485</span><span id="line-4485"> // mark as success and skip in doMiniBatchMutation</span> |
| <span class="source-line-no">4486</span><span id="line-4486"> metrics[2]++;</span> |
| <span class="source-line-no">4487</span><span id="line-4487"> retCodeDetails[index] = new OperationStatus(OperationStatusCode.SUCCESS, result);</span> |
| <span class="source-line-no">4488</span><span id="line-4488"> }</span> |
| <span class="source-line-no">4489</span><span id="line-4489"> } else if (m instanceof Append) {</span> |
| <span class="source-line-no">4490</span><span id="line-4490"> Append append = (Append) m;</span> |
| <span class="source-line-no">4491</span><span id="line-4491"> Result result = region.coprocessorHost.preAppend(append, walEdit);</span> |
| <span class="source-line-no">4492</span><span id="line-4492"> if (result != null) {</span> |
| <span class="source-line-no">4493</span><span id="line-4493"> // pre hook says skip this Append</span> |
| <span class="source-line-no">4494</span><span id="line-4494"> // mark as success and skip in doMiniBatchMutation</span> |
| <span class="source-line-no">4495</span><span id="line-4495"> metrics[3]++;</span> |
| <span class="source-line-no">4496</span><span id="line-4496"> retCodeDetails[index] = new OperationStatus(OperationStatusCode.SUCCESS, result);</span> |
| <span class="source-line-no">4497</span><span id="line-4497"> }</span> |
| <span class="source-line-no">4498</span><span id="line-4498"> } else {</span> |
| <span class="source-line-no">4499</span><span id="line-4499"> String msg = "Put/Delete/Increment/Append mutations only supported in a batch";</span> |
| <span class="source-line-no">4500</span><span id="line-4500"> retCodeDetails[index] = new OperationStatus(OperationStatusCode.FAILURE, msg);</span> |
| <span class="source-line-no">4501</span><span id="line-4501"> if (isAtomic()) { // fail, atomic means all or none</span> |
| <span class="source-line-no">4502</span><span id="line-4502"> throw new IOException(msg);</span> |
| <span class="source-line-no">4503</span><span id="line-4503"> }</span> |
| <span class="source-line-no">4504</span><span id="line-4504"> }</span> |
| <span class="source-line-no">4505</span><span id="line-4505"> }</span> |
| <span class="source-line-no">4506</span><span id="line-4506"></span> |
| <span class="source-line-no">4507</span><span id="line-4507"> // TODO Support Increment/Append operations</span> |
| <span class="source-line-no">4508</span><span id="line-4508"> private void checkAndMergeCPMutations(final MiniBatchOperationInProgress<Mutation> miniBatchOp,</span> |
| <span class="source-line-no">4509</span><span id="line-4509"> final List<RowLock> acquiredRowLocks, final long timestamp) throws IOException {</span> |
| <span class="source-line-no">4510</span><span id="line-4510"> visitBatchOperations(true, nextIndexToProcess + miniBatchOp.size(), (int i) -> {</span> |
| <span class="source-line-no">4511</span><span id="line-4511"> // we pass (i - firstIndex) below since the call expects a relative index</span> |
| <span class="source-line-no">4512</span><span id="line-4512"> Mutation[] cpMutations = miniBatchOp.getOperationsFromCoprocessors(i - nextIndexToProcess);</span> |
| <span class="source-line-no">4513</span><span id="line-4513"> if (cpMutations == null) {</span> |
| <span class="source-line-no">4514</span><span id="line-4514"> return true;</span> |
| <span class="source-line-no">4515</span><span id="line-4515"> }</span> |
| <span class="source-line-no">4516</span><span id="line-4516"> // Else Coprocessor added more Mutations corresponding to the Mutation at this index.</span> |
| <span class="source-line-no">4517</span><span id="line-4517"> Mutation mutation = getMutation(i);</span> |
| <span class="source-line-no">4518</span><span id="line-4518"> for (Mutation cpMutation : cpMutations) {</span> |
| <span class="source-line-no">4519</span><span id="line-4519"> this.checkAndPrepareMutation(cpMutation, timestamp);</span> |
| <span class="source-line-no">4520</span><span id="line-4520"></span> |
| <span class="source-line-no">4521</span><span id="line-4521"> // Acquire row locks. If not, the whole batch will fail.</span> |
| <span class="source-line-no">4522</span><span id="line-4522"> acquiredRowLocks.add(region.getRowLock(cpMutation.getRow(), true, null));</span> |
| <span class="source-line-no">4523</span><span id="line-4523"></span> |
| <span class="source-line-no">4524</span><span id="line-4524"> // Returned mutations from coprocessor correspond to the Mutation at index i. We can</span> |
| <span class="source-line-no">4525</span><span id="line-4525"> // directly add the cells from those mutations to the familyMaps of this mutation.</span> |
| <span class="source-line-no">4526</span><span id="line-4526"> Map<byte[], List<ExtendedCell>> cpFamilyMap =</span> |
| <span class="source-line-no">4527</span><span id="line-4527"> PackagePrivateFieldAccessor.getExtendedFamilyCellMap(cpMutation);</span> |
| <span class="source-line-no">4528</span><span id="line-4528"> region.rewriteCellTags(cpFamilyMap, mutation);</span> |
| <span class="source-line-no">4529</span><span id="line-4529"> // will get added to the memStore later</span> |
| <span class="source-line-no">4530</span><span id="line-4530"> mergeFamilyMaps(familyCellMaps[i], cpFamilyMap);</span> |
| <span class="source-line-no">4531</span><span id="line-4531"></span> |
| <span class="source-line-no">4532</span><span id="line-4532"> // The durability of returned mutation is replaced by the corresponding mutation.</span> |
| <span class="source-line-no">4533</span><span id="line-4533"> // If the corresponding mutation contains the SKIP_WAL, we shouldn't count the</span> |
| <span class="source-line-no">4534</span><span id="line-4534"> // cells of returned mutation.</span> |
| <span class="source-line-no">4535</span><span id="line-4535"> if (region.getEffectiveDurability(mutation.getDurability()) != Durability.SKIP_WAL) {</span> |
| <span class="source-line-no">4536</span><span id="line-4536"> for (List<ExtendedCell> cells : cpFamilyMap.values()) {</span> |
| <span class="source-line-no">4537</span><span id="line-4537"> miniBatchOp.addCellCount(cells.size());</span> |
| <span class="source-line-no">4538</span><span id="line-4538"> }</span> |
| <span class="source-line-no">4539</span><span id="line-4539"> }</span> |
| <span class="source-line-no">4540</span><span id="line-4540"> }</span> |
| <span class="source-line-no">4541</span><span id="line-4541"> return true;</span> |
| <span class="source-line-no">4542</span><span id="line-4542"> });</span> |
| <span class="source-line-no">4543</span><span id="line-4543"> }</span> |
| <span class="source-line-no">4544</span><span id="line-4544"></span> |
| <span class="source-line-no">4545</span><span id="line-4545"> private void mergeFamilyMaps(Map<byte[], List<ExtendedCell>> familyMap,</span> |
| <span class="source-line-no">4546</span><span id="line-4546"> Map<byte[], List<ExtendedCell>> toBeMerged) {</span> |
| <span class="source-line-no">4547</span><span id="line-4547"> for (Map.Entry<byte[], List<ExtendedCell>> entry : toBeMerged.entrySet()) {</span> |
| <span class="source-line-no">4548</span><span id="line-4548"> List<ExtendedCell> cells = familyMap.get(entry.getKey());</span> |
| <span class="source-line-no">4549</span><span id="line-4549"> if (cells == null) {</span> |
| <span class="source-line-no">4550</span><span id="line-4550"> familyMap.put(entry.getKey(), entry.getValue());</span> |
| <span class="source-line-no">4551</span><span id="line-4551"> } else {</span> |
| <span class="source-line-no">4552</span><span id="line-4552"> cells.addAll(entry.getValue());</span> |
| <span class="source-line-no">4553</span><span id="line-4553"> }</span> |
| <span class="source-line-no">4554</span><span id="line-4554"> }</span> |
| <span class="source-line-no">4555</span><span id="line-4555"> }</span> |
| <span class="source-line-no">4556</span><span id="line-4556"> }</span> |
| <span class="source-line-no">4557</span><span id="line-4557"></span> |
| <span class="source-line-no">4558</span><span id="line-4558"> /**</span> |
| <span class="source-line-no">4559</span><span id="line-4559"> * Batch of mutations for replay. Base class is shared with {@link MutationBatchOperation} as most</span> |
| <span class="source-line-no">4560</span><span id="line-4560"> * of the logic is same.</span> |
| <span class="source-line-no">4561</span><span id="line-4561"> * @deprecated Since 3.0.0, will be removed in 4.0.0. Now we will not use this operation to apply</span> |
| <span class="source-line-no">4562</span><span id="line-4562"> * edits at secondary replica side.</span> |
| <span class="source-line-no">4563</span><span id="line-4563"> */</span> |
| <span class="source-line-no">4564</span><span id="line-4564"> @Deprecated</span> |
| <span class="source-line-no">4565</span><span id="line-4565"> private static final class ReplayBatchOperation extends BatchOperation<MutationReplay> {</span> |
| <span class="source-line-no">4566</span><span id="line-4566"></span> |
| <span class="source-line-no">4567</span><span id="line-4567"> private long origLogSeqNum = 0;</span> |
| <span class="source-line-no">4568</span><span id="line-4568"></span> |
| <span class="source-line-no">4569</span><span id="line-4569"> public ReplayBatchOperation(final HRegion region, MutationReplay[] operations,</span> |
| <span class="source-line-no">4570</span><span id="line-4570"> long origLogSeqNum) {</span> |
| <span class="source-line-no">4571</span><span id="line-4571"> super(region, operations);</span> |
| <span class="source-line-no">4572</span><span id="line-4572"> this.origLogSeqNum = origLogSeqNum;</span> |
| <span class="source-line-no">4573</span><span id="line-4573"> }</span> |
| <span class="source-line-no">4574</span><span id="line-4574"></span> |
| <span class="source-line-no">4575</span><span id="line-4575"> @Override</span> |
| <span class="source-line-no">4576</span><span id="line-4576"> public Mutation getMutation(int index) {</span> |
| <span class="source-line-no">4577</span><span id="line-4577"> return this.operations[index].mutation;</span> |
| <span class="source-line-no">4578</span><span id="line-4578"> }</span> |
| <span class="source-line-no">4579</span><span id="line-4579"></span> |
| <span class="source-line-no">4580</span><span id="line-4580"> @Override</span> |
| <span class="source-line-no">4581</span><span id="line-4581"> public long getNonceGroup(int index) {</span> |
| <span class="source-line-no">4582</span><span id="line-4582"> return this.operations[index].nonceGroup;</span> |
| <span class="source-line-no">4583</span><span id="line-4583"> }</span> |
| <span class="source-line-no">4584</span><span id="line-4584"></span> |
| <span class="source-line-no">4585</span><span id="line-4585"> @Override</span> |
| <span class="source-line-no">4586</span><span id="line-4586"> public long getNonce(int index) {</span> |
| <span class="source-line-no">4587</span><span id="line-4587"> return this.operations[index].nonce;</span> |
| <span class="source-line-no">4588</span><span id="line-4588"> }</span> |
| <span class="source-line-no">4589</span><span id="line-4589"></span> |
| <span class="source-line-no">4590</span><span id="line-4590"> @Override</span> |
| <span class="source-line-no">4591</span><span id="line-4591"> public Mutation[] getMutationsForCoprocs() {</span> |
| <span class="source-line-no">4592</span><span id="line-4592"> return null;</span> |
| <span class="source-line-no">4593</span><span id="line-4593"> }</span> |
| <span class="source-line-no">4594</span><span id="line-4594"></span> |
| <span class="source-line-no">4595</span><span id="line-4595"> @Override</span> |
| <span class="source-line-no">4596</span><span id="line-4596"> public boolean isInReplay() {</span> |
| <span class="source-line-no">4597</span><span id="line-4597"> return true;</span> |
| <span class="source-line-no">4598</span><span id="line-4598"> }</span> |
| <span class="source-line-no">4599</span><span id="line-4599"></span> |
| <span class="source-line-no">4600</span><span id="line-4600"> @Override</span> |
| <span class="source-line-no">4601</span><span id="line-4601"> public long getOrigLogSeqNum() {</span> |
| <span class="source-line-no">4602</span><span id="line-4602"> return this.origLogSeqNum;</span> |
| <span class="source-line-no">4603</span><span id="line-4603"> }</span> |
| <span class="source-line-no">4604</span><span id="line-4604"></span> |
| <span class="source-line-no">4605</span><span id="line-4605"> @Override</span> |
| <span class="source-line-no">4606</span><span id="line-4606"> public void startRegionOperation() throws IOException {</span> |
| <span class="source-line-no">4607</span><span id="line-4607"> region.startRegionOperation(Operation.REPLAY_BATCH_MUTATE);</span> |
| <span class="source-line-no">4608</span><span id="line-4608"> }</span> |
| <span class="source-line-no">4609</span><span id="line-4609"></span> |
| <span class="source-line-no">4610</span><span id="line-4610"> @Override</span> |
| <span class="source-line-no">4611</span><span id="line-4611"> public void closeRegionOperation() throws IOException {</span> |
| <span class="source-line-no">4612</span><span id="line-4612"> region.closeRegionOperation(Operation.REPLAY_BATCH_MUTATE);</span> |
| <span class="source-line-no">4613</span><span id="line-4613"> }</span> |
| <span class="source-line-no">4614</span><span id="line-4614"></span> |
| <span class="source-line-no">4615</span><span id="line-4615"> /**</span> |
| <span class="source-line-no">4616</span><span id="line-4616"> * During replay, there could exist column families which are removed between region server</span> |
| <span class="source-line-no">4617</span><span id="line-4617"> * failure and replay</span> |
| <span class="source-line-no">4618</span><span id="line-4618"> */</span> |
| <span class="source-line-no">4619</span><span id="line-4619"> @Override</span> |
| <span class="source-line-no">4620</span><span id="line-4620"> protected void checkAndPreparePut(Put p) throws IOException {</span> |
| <span class="source-line-no">4621</span><span id="line-4621"> Map<byte[], List<Cell>> familyCellMap = p.getFamilyCellMap();</span> |
| <span class="source-line-no">4622</span><span id="line-4622"> List<byte[]> nonExistentList = null;</span> |
| <span class="source-line-no">4623</span><span id="line-4623"> for (byte[] family : familyCellMap.keySet()) {</span> |
| <span class="source-line-no">4624</span><span id="line-4624"> if (!region.htableDescriptor.hasColumnFamily(family)) {</span> |
| <span class="source-line-no">4625</span><span id="line-4625"> if (nonExistentList == null) {</span> |
| <span class="source-line-no">4626</span><span id="line-4626"> nonExistentList = new ArrayList<>();</span> |
| <span class="source-line-no">4627</span><span id="line-4627"> }</span> |
| <span class="source-line-no">4628</span><span id="line-4628"> nonExistentList.add(family);</span> |
| <span class="source-line-no">4629</span><span id="line-4629"> }</span> |
| <span class="source-line-no">4630</span><span id="line-4630"> }</span> |
| <span class="source-line-no">4631</span><span id="line-4631"> if (nonExistentList != null) {</span> |
| <span class="source-line-no">4632</span><span id="line-4632"> for (byte[] family : nonExistentList) {</span> |
| <span class="source-line-no">4633</span><span id="line-4633"> // Perhaps schema was changed between crash and replay</span> |
| <span class="source-line-no">4634</span><span id="line-4634"> LOG.info("No family for {} omit from reply in region {}.", Bytes.toString(family), this);</span> |
| <span class="source-line-no">4635</span><span id="line-4635"> familyCellMap.remove(family);</span> |
| <span class="source-line-no">4636</span><span id="line-4636"> }</span> |
| <span class="source-line-no">4637</span><span id="line-4637"> }</span> |
| <span class="source-line-no">4638</span><span id="line-4638"> }</span> |
| <span class="source-line-no">4639</span><span id="line-4639"></span> |
| <span class="source-line-no">4640</span><span id="line-4640"> @Override</span> |
| <span class="source-line-no">4641</span><span id="line-4641"> public void checkAndPrepare() throws IOException {</span> |
| <span class="source-line-no">4642</span><span id="line-4642"> long now = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">4643</span><span id="line-4643"> visitBatchOperations(true, this.size(), (int index) -> {</span> |
| <span class="source-line-no">4644</span><span id="line-4644"> checkAndPrepareMutation(index, now);</span> |
| <span class="source-line-no">4645</span><span id="line-4645"> return true;</span> |
| <span class="source-line-no">4646</span><span id="line-4646"> });</span> |
| <span class="source-line-no">4647</span><span id="line-4647"> }</span> |
| <span class="source-line-no">4648</span><span id="line-4648"></span> |
| <span class="source-line-no">4649</span><span id="line-4649"> @Override</span> |
| <span class="source-line-no">4650</span><span id="line-4650"> public void prepareMiniBatchOperations(MiniBatchOperationInProgress<Mutation> miniBatchOp,</span> |
| <span class="source-line-no">4651</span><span id="line-4651"> long timestamp, final List<RowLock> acquiredRowLocks) throws IOException {</span> |
| <span class="source-line-no">4652</span><span id="line-4652"> visitBatchOperations(true, miniBatchOp.getLastIndexExclusive(), (int index) -> {</span> |
| <span class="source-line-no">4653</span><span id="line-4653"> // update cell count</span> |
| <span class="source-line-no">4654</span><span id="line-4654"> for (List<Cell> cells : getMutation(index).getFamilyCellMap().values()) {</span> |
| <span class="source-line-no">4655</span><span id="line-4655"> miniBatchOp.addCellCount(cells.size());</span> |
| <span class="source-line-no">4656</span><span id="line-4656"> }</span> |
| <span class="source-line-no">4657</span><span id="line-4657"> return true;</span> |
| <span class="source-line-no">4658</span><span id="line-4658"> });</span> |
| <span class="source-line-no">4659</span><span id="line-4659"> }</span> |
| <span class="source-line-no">4660</span><span id="line-4660"></span> |
| <span class="source-line-no">4661</span><span id="line-4661"> @Override</span> |
| <span class="source-line-no">4662</span><span id="line-4662"> public WriteEntry writeMiniBatchOperationsToMemStore(</span> |
| <span class="source-line-no">4663</span><span id="line-4663"> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final WriteEntry writeEntry,</span> |
| <span class="source-line-no">4664</span><span id="line-4664"> long now) throws IOException {</span> |
| <span class="source-line-no">4665</span><span id="line-4665"> super.writeMiniBatchOperationsToMemStore(miniBatchOp, getOrigLogSeqNum());</span> |
| <span class="source-line-no">4666</span><span id="line-4666"> return writeEntry;</span> |
| <span class="source-line-no">4667</span><span id="line-4667"> }</span> |
| <span class="source-line-no">4668</span><span id="line-4668"></span> |
| <span class="source-line-no">4669</span><span id="line-4669"> @Override</span> |
| <span class="source-line-no">4670</span><span id="line-4670"> public void completeMiniBatchOperations(</span> |
| <span class="source-line-no">4671</span><span id="line-4671"> final MiniBatchOperationInProgress<Mutation> miniBatchOp, final WriteEntry writeEntry)</span> |
| <span class="source-line-no">4672</span><span id="line-4672"> throws IOException {</span> |
| <span class="source-line-no">4673</span><span id="line-4673"> super.completeMiniBatchOperations(miniBatchOp, writeEntry);</span> |
| <span class="source-line-no">4674</span><span id="line-4674"> region.mvcc.advanceTo(getOrigLogSeqNum());</span> |
| <span class="source-line-no">4675</span><span id="line-4675"> }</span> |
| <span class="source-line-no">4676</span><span id="line-4676"></span> |
| <span class="source-line-no">4677</span><span id="line-4677"> @Override</span> |
| <span class="source-line-no">4678</span><span id="line-4678"> protected void cacheSkipWALMutationForRegionReplication(</span> |
| <span class="source-line-no">4679</span><span id="line-4679"> MiniBatchOperationInProgress<Mutation> miniBatchOp, List<Pair<NonceKey, WALEdit>> walEdits,</span> |
| <span class="source-line-no">4680</span><span id="line-4680"> Map<byte[], List<ExtendedCell>> familyCellMap) {</span> |
| <span class="source-line-no">4681</span><span id="line-4681"> // There is no action to do if current region is secondary replica</span> |
| <span class="source-line-no">4682</span><span id="line-4682"> }</span> |
| <span class="source-line-no">4683</span><span id="line-4683"></span> |
| <span class="source-line-no">4684</span><span id="line-4684"> }</span> |
| <span class="source-line-no">4685</span><span id="line-4685"></span> |
| <span class="source-line-no">4686</span><span id="line-4686"> public OperationStatus[] batchMutate(Mutation[] mutations, boolean atomic, long nonceGroup,</span> |
| <span class="source-line-no">4687</span><span id="line-4687"> long nonce) throws IOException {</span> |
| <span class="source-line-no">4688</span><span id="line-4688"> // As it stands, this is used for 3 things</span> |
| <span class="source-line-no">4689</span><span id="line-4689"> // * batchMutate with single mutation - put/delete/increment/append, separate or from</span> |
| <span class="source-line-no">4690</span><span id="line-4690"> // checkAndMutate.</span> |
| <span class="source-line-no">4691</span><span id="line-4691"> // * coprocessor calls (see ex. BulkDeleteEndpoint).</span> |
| <span class="source-line-no">4692</span><span id="line-4692"> // So nonces are not really ever used by HBase. They could be by coprocs, and checkAnd...</span> |
| <span class="source-line-no">4693</span><span id="line-4693"> return batchMutate(new MutationBatchOperation(this, mutations, atomic, nonceGroup, nonce));</span> |
| <span class="source-line-no">4694</span><span id="line-4694"> }</span> |
| <span class="source-line-no">4695</span><span id="line-4695"></span> |
| <span class="source-line-no">4696</span><span id="line-4696"> @Override</span> |
| <span class="source-line-no">4697</span><span id="line-4697"> public OperationStatus[] batchMutate(Mutation[] mutations) throws IOException {</span> |
| <span class="source-line-no">4698</span><span id="line-4698"> // If the mutations has any Increment/Append operations, we need to do batchMutate atomically</span> |
| <span class="source-line-no">4699</span><span id="line-4699"> boolean atomic =</span> |
| <span class="source-line-no">4700</span><span id="line-4700"> Arrays.stream(mutations).anyMatch(m -> m instanceof Increment || m instanceof Append);</span> |
| <span class="source-line-no">4701</span><span id="line-4701"> return batchMutate(mutations, atomic);</span> |
| <span class="source-line-no">4702</span><span id="line-4702"> }</span> |
| <span class="source-line-no">4703</span><span id="line-4703"></span> |
| <span class="source-line-no">4704</span><span id="line-4704"> OperationStatus[] batchMutate(Mutation[] mutations, boolean atomic) throws IOException {</span> |
| <span class="source-line-no">4705</span><span id="line-4705"> return TraceUtil.trace(</span> |
| <span class="source-line-no">4706</span><span id="line-4706"> () -> batchMutate(mutations, atomic, HConstants.NO_NONCE, HConstants.NO_NONCE),</span> |
| <span class="source-line-no">4707</span><span id="line-4707"> () -> createRegionSpan("Region.batchMutate"));</span> |
| <span class="source-line-no">4708</span><span id="line-4708"> }</span> |
| <span class="source-line-no">4709</span><span id="line-4709"></span> |
| <span class="source-line-no">4710</span><span id="line-4710"> /**</span> |
| <span class="source-line-no">4711</span><span id="line-4711"> * @deprecated Since 3.0.0, will be removed in 4.0.0. Now we use</span> |
| <span class="source-line-no">4712</span><span id="line-4712"> * {@link #replayWALEntry(WALEntry, CellScanner)} for replaying edits at secondary</span> |
| <span class="source-line-no">4713</span><span id="line-4713"> * replica side.</span> |
| <span class="source-line-no">4714</span><span id="line-4714"> */</span> |
| <span class="source-line-no">4715</span><span id="line-4715"> @Deprecated</span> |
| <span class="source-line-no">4716</span><span id="line-4716"> OperationStatus[] batchReplay(MutationReplay[] mutations, long replaySeqId) throws IOException {</span> |
| <span class="source-line-no">4717</span><span id="line-4717"> if (</span> |
| <span class="source-line-no">4718</span><span id="line-4718"> !RegionReplicaUtil.isDefaultReplica(getRegionInfo())</span> |
| <span class="source-line-no">4719</span><span id="line-4719"> && replaySeqId < lastReplayedOpenRegionSeqId</span> |
| <span class="source-line-no">4720</span><span id="line-4720"> ) {</span> |
| <span class="source-line-no">4721</span><span id="line-4721"> // if it is a secondary replica we should ignore these entries silently</span> |
| <span class="source-line-no">4722</span><span id="line-4722"> // since they are coming out of order</span> |
| <span class="source-line-no">4723</span><span id="line-4723"> if (LOG.isTraceEnabled()) {</span> |
| <span class="source-line-no">4724</span><span id="line-4724"> LOG.trace(getRegionInfo().getEncodedName() + " : " + "Skipping " + mutations.length</span> |
| <span class="source-line-no">4725</span><span id="line-4725"> + " mutations with replaySeqId=" + replaySeqId</span> |
| <span class="source-line-no">4726</span><span id="line-4726"> + " which is < than lastReplayedOpenRegionSeqId=" + lastReplayedOpenRegionSeqId);</span> |
| <span class="source-line-no">4727</span><span id="line-4727"> for (MutationReplay mut : mutations) {</span> |
| <span class="source-line-no">4728</span><span id="line-4728"> LOG.trace(getRegionInfo().getEncodedName() + " : Skipping : " + mut.mutation);</span> |
| <span class="source-line-no">4729</span><span id="line-4729"> }</span> |
| <span class="source-line-no">4730</span><span id="line-4730"> }</span> |
| <span class="source-line-no">4731</span><span id="line-4731"></span> |
| <span class="source-line-no">4732</span><span id="line-4732"> OperationStatus[] statuses = new OperationStatus[mutations.length];</span> |
| <span class="source-line-no">4733</span><span id="line-4733"> for (int i = 0; i < statuses.length; i++) {</span> |
| <span class="source-line-no">4734</span><span id="line-4734"> statuses[i] = OperationStatus.SUCCESS;</span> |
| <span class="source-line-no">4735</span><span id="line-4735"> }</span> |
| <span class="source-line-no">4736</span><span id="line-4736"> return statuses;</span> |
| <span class="source-line-no">4737</span><span id="line-4737"> }</span> |
| <span class="source-line-no">4738</span><span id="line-4738"> return batchMutate(new ReplayBatchOperation(this, mutations, replaySeqId));</span> |
| <span class="source-line-no">4739</span><span id="line-4739"> }</span> |
| <span class="source-line-no">4740</span><span id="line-4740"></span> |
| <span class="source-line-no">4741</span><span id="line-4741"> /**</span> |
| <span class="source-line-no">4742</span><span id="line-4742"> * Perform a batch of mutations.</span> |
| <span class="source-line-no">4743</span><span id="line-4743"> * <p/></span> |
| <span class="source-line-no">4744</span><span id="line-4744"> * Operations in a batch are stored with highest durability specified of for all operations in a</span> |
| <span class="source-line-no">4745</span><span id="line-4745"> * batch, except for {@link Durability#SKIP_WAL}.</span> |
| <span class="source-line-no">4746</span><span id="line-4746"> * <p/></span> |
| <span class="source-line-no">4747</span><span id="line-4747"> * This function is called from {@link #batchReplay(WALSplitUtil.MutationReplay[], long)} with</span> |
| <span class="source-line-no">4748</span><span id="line-4748"> * {@link ReplayBatchOperation} instance and {@link #batchMutate(Mutation[])} with</span> |
| <span class="source-line-no">4749</span><span id="line-4749"> * {@link MutationBatchOperation} instance as an argument. As the processing of replay batch and</span> |
| <span class="source-line-no">4750</span><span id="line-4750"> * mutation batch is very similar, lot of code is shared by providing generic methods in base</span> |
| <span class="source-line-no">4751</span><span id="line-4751"> * class {@link BatchOperation}. The logic for this method and</span> |
| <span class="source-line-no">4752</span><span id="line-4752"> * {@link #doMiniBatchMutate(BatchOperation)} is implemented using methods in base class which are</span> |
| <span class="source-line-no">4753</span><span id="line-4753"> * overridden by derived classes to implement special behavior.</span> |
| <span class="source-line-no">4754</span><span id="line-4754"> * @param batchOp contains the list of mutations</span> |
| <span class="source-line-no">4755</span><span id="line-4755"> * @return an array of OperationStatus which internally contains the OperationStatusCode and the</span> |
| <span class="source-line-no">4756</span><span id="line-4756"> * exceptionMessage if any.</span> |
| <span class="source-line-no">4757</span><span id="line-4757"> * @throws IOException if an IO problem is encountered</span> |
| <span class="source-line-no">4758</span><span id="line-4758"> */</span> |
| <span class="source-line-no">4759</span><span id="line-4759"> private OperationStatus[] batchMutate(BatchOperation<?> batchOp) throws IOException {</span> |
| <span class="source-line-no">4760</span><span id="line-4760"> boolean initialized = false;</span> |
| <span class="source-line-no">4761</span><span id="line-4761"> batchOp.startRegionOperation();</span> |
| <span class="source-line-no">4762</span><span id="line-4762"> try {</span> |
| <span class="source-line-no">4763</span><span id="line-4763"> while (!batchOp.isDone()) {</span> |
| <span class="source-line-no">4764</span><span id="line-4764"> if (!batchOp.isInReplay()) {</span> |
| <span class="source-line-no">4765</span><span id="line-4765"> checkReadOnly();</span> |
| <span class="source-line-no">4766</span><span id="line-4766"> }</span> |
| <span class="source-line-no">4767</span><span id="line-4767"> checkResources();</span> |
| <span class="source-line-no">4768</span><span id="line-4768"></span> |
| <span class="source-line-no">4769</span><span id="line-4769"> if (!initialized) {</span> |
| <span class="source-line-no">4770</span><span id="line-4770"> this.writeRequestsCount.add(batchOp.size());</span> |
| <span class="source-line-no">4771</span><span id="line-4771"> // validate and prepare batch for write, for MutationBatchOperation it also calls CP</span> |
| <span class="source-line-no">4772</span><span id="line-4772"> // prePut()/preDelete()/preIncrement()/preAppend() hooks</span> |
| <span class="source-line-no">4773</span><span id="line-4773"> batchOp.checkAndPrepare();</span> |
| <span class="source-line-no">4774</span><span id="line-4774"> initialized = true;</span> |
| <span class="source-line-no">4775</span><span id="line-4775"> }</span> |
| <span class="source-line-no">4776</span><span id="line-4776"> doMiniBatchMutate(batchOp);</span> |
| <span class="source-line-no">4777</span><span id="line-4777"> requestFlushIfNeeded();</span> |
| <span class="source-line-no">4778</span><span id="line-4778"> }</span> |
| <span class="source-line-no">4779</span><span id="line-4779"> } finally {</span> |
| <span class="source-line-no">4780</span><span id="line-4780"> if (rsServices != null && rsServices.getMetrics() != null) {</span> |
| <span class="source-line-no">4781</span><span id="line-4781"> rsServices.getMetrics().updateWriteQueryMeter(this, batchOp.size());</span> |
| <span class="source-line-no">4782</span><span id="line-4782"> }</span> |
| <span class="source-line-no">4783</span><span id="line-4783"> batchOp.closeRegionOperation();</span> |
| <span class="source-line-no">4784</span><span id="line-4784"> }</span> |
| <span class="source-line-no">4785</span><span id="line-4785"> return batchOp.retCodeDetails;</span> |
| <span class="source-line-no">4786</span><span id="line-4786"> }</span> |
| <span class="source-line-no">4787</span><span id="line-4787"></span> |
| <span class="source-line-no">4788</span><span id="line-4788"> /**</span> |
| <span class="source-line-no">4789</span><span id="line-4789"> * Called to do a piece of the batch that came in to {@link #batchMutate(Mutation[])} In here we</span> |
| <span class="source-line-no">4790</span><span id="line-4790"> * also handle replay of edits on region recover. Also gets change in size brought about by</span> |
| <span class="source-line-no">4791</span><span id="line-4791"> * applying {@code batchOp}.</span> |
| <span class="source-line-no">4792</span><span id="line-4792"> */</span> |
| <span class="source-line-no">4793</span><span id="line-4793"> private void doMiniBatchMutate(BatchOperation<?> batchOp) throws IOException {</span> |
| <span class="source-line-no">4794</span><span id="line-4794"> boolean success = false;</span> |
| <span class="source-line-no">4795</span><span id="line-4795"> WALEdit walEdit = null;</span> |
| <span class="source-line-no">4796</span><span id="line-4796"> WriteEntry writeEntry = null;</span> |
| <span class="source-line-no">4797</span><span id="line-4797"> boolean locked = false;</span> |
| <span class="source-line-no">4798</span><span id="line-4798"> // We try to set up a batch in the range [batchOp.nextIndexToProcess,lastIndexExclusive)</span> |
| <span class="source-line-no">4799</span><span id="line-4799"> MiniBatchOperationInProgress<Mutation> miniBatchOp = null;</span> |
| <span class="source-line-no">4800</span><span id="line-4800"> /** Keep track of the locks we hold so we can release them in finally clause */</span> |
| <span class="source-line-no">4801</span><span id="line-4801"> List<RowLock> acquiredRowLocks = Lists.newArrayListWithCapacity(batchOp.size());</span> |
| <span class="source-line-no">4802</span><span id="line-4802"></span> |
| <span class="source-line-no">4803</span><span id="line-4803"> // Check for thread interrupt status in case we have been signaled from</span> |
| <span class="source-line-no">4804</span><span id="line-4804"> // #interruptRegionOperation.</span> |
| <span class="source-line-no">4805</span><span id="line-4805"> checkInterrupt();</span> |
| <span class="source-line-no">4806</span><span id="line-4806"></span> |
| <span class="source-line-no">4807</span><span id="line-4807"> try {</span> |
| <span class="source-line-no">4808</span><span id="line-4808"> // STEP 1. Try to acquire as many locks as we can and build mini-batch of operations with</span> |
| <span class="source-line-no">4809</span><span id="line-4809"> // locked rows</span> |
| <span class="source-line-no">4810</span><span id="line-4810"> miniBatchOp = batchOp.lockRowsAndBuildMiniBatch(acquiredRowLocks);</span> |
| <span class="source-line-no">4811</span><span id="line-4811"></span> |
| <span class="source-line-no">4812</span><span id="line-4812"> // We've now grabbed as many mutations off the list as we can</span> |
| <span class="source-line-no">4813</span><span id="line-4813"> // Ensure we acquire at least one.</span> |
| <span class="source-line-no">4814</span><span id="line-4814"> if (miniBatchOp.getReadyToWriteCount() <= 0) {</span> |
| <span class="source-line-no">4815</span><span id="line-4815"> // Nothing to put/delete/increment/append -- an exception in the above such as</span> |
| <span class="source-line-no">4816</span><span id="line-4816"> // NoSuchColumnFamily?</span> |
| <span class="source-line-no">4817</span><span id="line-4817"> return;</span> |
| <span class="source-line-no">4818</span><span id="line-4818"> }</span> |
| <span class="source-line-no">4819</span><span id="line-4819"></span> |
| <span class="source-line-no">4820</span><span id="line-4820"> // Check for thread interrupt status in case we have been signaled from</span> |
| <span class="source-line-no">4821</span><span id="line-4821"> // #interruptRegionOperation. Do it before we take the lock and disable interrupts for</span> |
| <span class="source-line-no">4822</span><span id="line-4822"> // the WAL append.</span> |
| <span class="source-line-no">4823</span><span id="line-4823"> checkInterrupt();</span> |
| <span class="source-line-no">4824</span><span id="line-4824"></span> |
| <span class="source-line-no">4825</span><span id="line-4825"> lock(this.updatesLock.readLock(), miniBatchOp.getReadyToWriteCount());</span> |
| <span class="source-line-no">4826</span><span id="line-4826"> locked = true;</span> |
| <span class="source-line-no">4827</span><span id="line-4827"></span> |
| <span class="source-line-no">4828</span><span id="line-4828"> // From this point until memstore update this operation should not be interrupted.</span> |
| <span class="source-line-no">4829</span><span id="line-4829"> disableInterrupts();</span> |
| <span class="source-line-no">4830</span><span id="line-4830"></span> |
| <span class="source-line-no">4831</span><span id="line-4831"> // STEP 2. Update mini batch of all operations in progress with LATEST_TIMESTAMP timestamp</span> |
| <span class="source-line-no">4832</span><span id="line-4832"> // We should record the timestamp only after we have acquired the rowLock,</span> |
| <span class="source-line-no">4833</span><span id="line-4833"> // otherwise, newer puts/deletes/increment/append are not guaranteed to have a newer</span> |
| <span class="source-line-no">4834</span><span id="line-4834"> // timestamp</span> |
| <span class="source-line-no">4835</span><span id="line-4835"></span> |
| <span class="source-line-no">4836</span><span id="line-4836"> long now = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">4837</span><span id="line-4837"> batchOp.prepareMiniBatchOperations(miniBatchOp, now, acquiredRowLocks);</span> |
| <span class="source-line-no">4838</span><span id="line-4838"></span> |
| <span class="source-line-no">4839</span><span id="line-4839"> // STEP 3. Build WAL edit</span> |
| <span class="source-line-no">4840</span><span id="line-4840"></span> |
| <span class="source-line-no">4841</span><span id="line-4841"> List<Pair<NonceKey, WALEdit>> walEdits = batchOp.buildWALEdits(miniBatchOp);</span> |
| <span class="source-line-no">4842</span><span id="line-4842"></span> |
| <span class="source-line-no">4843</span><span id="line-4843"> // STEP 4. Append the WALEdits to WAL and sync.</span> |
| <span class="source-line-no">4844</span><span id="line-4844"></span> |
| <span class="source-line-no">4845</span><span id="line-4845"> for (Iterator<Pair<NonceKey, WALEdit>> it = walEdits.iterator(); it.hasNext();) {</span> |
| <span class="source-line-no">4846</span><span id="line-4846"> Pair<NonceKey, WALEdit> nonceKeyWALEditPair = it.next();</span> |
| <span class="source-line-no">4847</span><span id="line-4847"> walEdit = nonceKeyWALEditPair.getSecond();</span> |
| <span class="source-line-no">4848</span><span id="line-4848"> NonceKey nonceKey = nonceKeyWALEditPair.getFirst();</span> |
| <span class="source-line-no">4849</span><span id="line-4849"></span> |
| <span class="source-line-no">4850</span><span id="line-4850"> if (walEdit != null && !walEdit.isEmpty()) {</span> |
| <span class="source-line-no">4851</span><span id="line-4851"> writeEntry = doWALAppend(walEdit, batchOp, miniBatchOp, now, nonceKey);</span> |
| <span class="source-line-no">4852</span><span id="line-4852"> }</span> |
| <span class="source-line-no">4853</span><span id="line-4853"></span> |
| <span class="source-line-no">4854</span><span id="line-4854"> // Complete mvcc for all but last writeEntry (for replay case)</span> |
| <span class="source-line-no">4855</span><span id="line-4855"> if (it.hasNext() && writeEntry != null) {</span> |
| <span class="source-line-no">4856</span><span id="line-4856"> mvcc.complete(writeEntry);</span> |
| <span class="source-line-no">4857</span><span id="line-4857"> writeEntry = null;</span> |
| <span class="source-line-no">4858</span><span id="line-4858"> }</span> |
| <span class="source-line-no">4859</span><span id="line-4859"> }</span> |
| <span class="source-line-no">4860</span><span id="line-4860"></span> |
| <span class="source-line-no">4861</span><span id="line-4861"> // STEP 5. Write back to memStore</span> |
| <span class="source-line-no">4862</span><span id="line-4862"> // NOTE: writeEntry can be null here</span> |
| <span class="source-line-no">4863</span><span id="line-4863"> writeEntry = batchOp.writeMiniBatchOperationsToMemStore(miniBatchOp, writeEntry, now);</span> |
| <span class="source-line-no">4864</span><span id="line-4864"></span> |
| <span class="source-line-no">4865</span><span id="line-4865"> // STEP 6. Complete MiniBatchOperations: If required calls postBatchMutate() CP hook and</span> |
| <span class="source-line-no">4866</span><span id="line-4866"> // complete mvcc for last writeEntry</span> |
| <span class="source-line-no">4867</span><span id="line-4867"> batchOp.completeMiniBatchOperations(miniBatchOp, writeEntry);</span> |
| <span class="source-line-no">4868</span><span id="line-4868"> writeEntry = null;</span> |
| <span class="source-line-no">4869</span><span id="line-4869"> success = true;</span> |
| <span class="source-line-no">4870</span><span id="line-4870"> } finally {</span> |
| <span class="source-line-no">4871</span><span id="line-4871"> // Call complete rather than completeAndWait because we probably had error if walKey != null</span> |
| <span class="source-line-no">4872</span><span id="line-4872"> if (writeEntry != null) mvcc.complete(writeEntry);</span> |
| <span class="source-line-no">4873</span><span id="line-4873"></span> |
| <span class="source-line-no">4874</span><span id="line-4874"> if (locked) {</span> |
| <span class="source-line-no">4875</span><span id="line-4875"> this.updatesLock.readLock().unlock();</span> |
| <span class="source-line-no">4876</span><span id="line-4876"> }</span> |
| <span class="source-line-no">4877</span><span id="line-4877"> releaseRowLocks(acquiredRowLocks);</span> |
| <span class="source-line-no">4878</span><span id="line-4878"></span> |
| <span class="source-line-no">4879</span><span id="line-4879"> enableInterrupts();</span> |
| <span class="source-line-no">4880</span><span id="line-4880"></span> |
| <span class="source-line-no">4881</span><span id="line-4881"> final int finalLastIndexExclusive =</span> |
| <span class="source-line-no">4882</span><span id="line-4882"> miniBatchOp != null ? miniBatchOp.getLastIndexExclusive() : batchOp.size();</span> |
| <span class="source-line-no">4883</span><span id="line-4883"> final boolean finalSuccess = success;</span> |
| <span class="source-line-no">4884</span><span id="line-4884"> batchOp.visitBatchOperations(true, finalLastIndexExclusive, (int i) -> {</span> |
| <span class="source-line-no">4885</span><span id="line-4885"> Mutation mutation = batchOp.getMutation(i);</span> |
| <span class="source-line-no">4886</span><span id="line-4886"> if (mutation instanceof Increment || mutation instanceof Append) {</span> |
| <span class="source-line-no">4887</span><span id="line-4887"> if (finalSuccess) {</span> |
| <span class="source-line-no">4888</span><span id="line-4888"> batchOp.retCodeDetails[i] =</span> |
| <span class="source-line-no">4889</span><span id="line-4889"> new OperationStatus(OperationStatusCode.SUCCESS, batchOp.results[i]);</span> |
| <span class="source-line-no">4890</span><span id="line-4890"> } else {</span> |
| <span class="source-line-no">4891</span><span id="line-4891"> batchOp.retCodeDetails[i] = OperationStatus.FAILURE;</span> |
| <span class="source-line-no">4892</span><span id="line-4892"> }</span> |
| <span class="source-line-no">4893</span><span id="line-4893"> } else {</span> |
| <span class="source-line-no">4894</span><span id="line-4894"> batchOp.retCodeDetails[i] =</span> |
| <span class="source-line-no">4895</span><span id="line-4895"> finalSuccess ? OperationStatus.SUCCESS : OperationStatus.FAILURE;</span> |
| <span class="source-line-no">4896</span><span id="line-4896"> }</span> |
| <span class="source-line-no">4897</span><span id="line-4897"> return true;</span> |
| <span class="source-line-no">4898</span><span id="line-4898"> });</span> |
| <span class="source-line-no">4899</span><span id="line-4899"></span> |
| <span class="source-line-no">4900</span><span id="line-4900"> batchOp.doPostOpCleanupForMiniBatch(miniBatchOp, walEdit, finalSuccess);</span> |
| <span class="source-line-no">4901</span><span id="line-4901"></span> |
| <span class="source-line-no">4902</span><span id="line-4902"> batchOp.nextIndexToProcess = finalLastIndexExclusive;</span> |
| <span class="source-line-no">4903</span><span id="line-4903"> }</span> |
| <span class="source-line-no">4904</span><span id="line-4904"> }</span> |
| <span class="source-line-no">4905</span><span id="line-4905"></span> |
| <span class="source-line-no">4906</span><span id="line-4906"> /**</span> |
| <span class="source-line-no">4907</span><span id="line-4907"> * Returns effective durability from the passed durability and the table descriptor.</span> |
| <span class="source-line-no">4908</span><span id="line-4908"> */</span> |
| <span class="source-line-no">4909</span><span id="line-4909"> private Durability getEffectiveDurability(Durability d) {</span> |
| <span class="source-line-no">4910</span><span id="line-4910"> return d == Durability.USE_DEFAULT ? this.regionDurability : d;</span> |
| <span class="source-line-no">4911</span><span id="line-4911"> }</span> |
| <span class="source-line-no">4912</span><span id="line-4912"></span> |
| <span class="source-line-no">4913</span><span id="line-4913"> @Override</span> |
| <span class="source-line-no">4914</span><span id="line-4914"> @Deprecated</span> |
| <span class="source-line-no">4915</span><span id="line-4915"> public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareOperator op,</span> |
| <span class="source-line-no">4916</span><span id="line-4916"> ByteArrayComparable comparator, TimeRange timeRange, Mutation mutation) throws IOException {</span> |
| <span class="source-line-no">4917</span><span id="line-4917"> CheckAndMutate checkAndMutate;</span> |
| <span class="source-line-no">4918</span><span id="line-4918"> try {</span> |
| <span class="source-line-no">4919</span><span id="line-4919"> CheckAndMutate.Builder builder = CheckAndMutate.newBuilder(row)</span> |
| <span class="source-line-no">4920</span><span id="line-4920"> .ifMatches(family, qualifier, op, comparator.getValue()).timeRange(timeRange);</span> |
| <span class="source-line-no">4921</span><span id="line-4921"> if (mutation instanceof Put) {</span> |
| <span class="source-line-no">4922</span><span id="line-4922"> checkAndMutate = builder.build((Put) mutation);</span> |
| <span class="source-line-no">4923</span><span id="line-4923"> } else if (mutation instanceof Delete) {</span> |
| <span class="source-line-no">4924</span><span id="line-4924"> checkAndMutate = builder.build((Delete) mutation);</span> |
| <span class="source-line-no">4925</span><span id="line-4925"> } else {</span> |
| <span class="source-line-no">4926</span><span id="line-4926"> throw new DoNotRetryIOException(</span> |
| <span class="source-line-no">4927</span><span id="line-4927"> "Unsupported mutate type: " + mutation.getClass().getSimpleName().toUpperCase());</span> |
| <span class="source-line-no">4928</span><span id="line-4928"> }</span> |
| <span class="source-line-no">4929</span><span id="line-4929"> } catch (IllegalArgumentException e) {</span> |
| <span class="source-line-no">4930</span><span id="line-4930"> throw new DoNotRetryIOException(e.getMessage());</span> |
| <span class="source-line-no">4931</span><span id="line-4931"> }</span> |
| <span class="source-line-no">4932</span><span id="line-4932"> return checkAndMutate(checkAndMutate).isSuccess();</span> |
| <span class="source-line-no">4933</span><span id="line-4933"> }</span> |
| <span class="source-line-no">4934</span><span id="line-4934"></span> |
| <span class="source-line-no">4935</span><span id="line-4935"> @Override</span> |
| <span class="source-line-no">4936</span><span id="line-4936"> @Deprecated</span> |
| <span class="source-line-no">4937</span><span id="line-4937"> public boolean checkAndMutate(byte[] row, Filter filter, TimeRange timeRange, Mutation mutation)</span> |
| <span class="source-line-no">4938</span><span id="line-4938"> throws IOException {</span> |
| <span class="source-line-no">4939</span><span id="line-4939"> CheckAndMutate checkAndMutate;</span> |
| <span class="source-line-no">4940</span><span id="line-4940"> try {</span> |
| <span class="source-line-no">4941</span><span id="line-4941"> CheckAndMutate.Builder builder =</span> |
| <span class="source-line-no">4942</span><span id="line-4942"> CheckAndMutate.newBuilder(row).ifMatches(filter).timeRange(timeRange);</span> |
| <span class="source-line-no">4943</span><span id="line-4943"> if (mutation instanceof Put) {</span> |
| <span class="source-line-no">4944</span><span id="line-4944"> checkAndMutate = builder.build((Put) mutation);</span> |
| <span class="source-line-no">4945</span><span id="line-4945"> } else if (mutation instanceof Delete) {</span> |
| <span class="source-line-no">4946</span><span id="line-4946"> checkAndMutate = builder.build((Delete) mutation);</span> |
| <span class="source-line-no">4947</span><span id="line-4947"> } else {</span> |
| <span class="source-line-no">4948</span><span id="line-4948"> throw new DoNotRetryIOException(</span> |
| <span class="source-line-no">4949</span><span id="line-4949"> "Unsupported mutate type: " + mutation.getClass().getSimpleName().toUpperCase());</span> |
| <span class="source-line-no">4950</span><span id="line-4950"> }</span> |
| <span class="source-line-no">4951</span><span id="line-4951"> } catch (IllegalArgumentException e) {</span> |
| <span class="source-line-no">4952</span><span id="line-4952"> throw new DoNotRetryIOException(e.getMessage());</span> |
| <span class="source-line-no">4953</span><span id="line-4953"> }</span> |
| <span class="source-line-no">4954</span><span id="line-4954"> return checkAndMutate(checkAndMutate).isSuccess();</span> |
| <span class="source-line-no">4955</span><span id="line-4955"> }</span> |
| <span class="source-line-no">4956</span><span id="line-4956"></span> |
| <span class="source-line-no">4957</span><span id="line-4957"> @Override</span> |
| <span class="source-line-no">4958</span><span id="line-4958"> @Deprecated</span> |
| <span class="source-line-no">4959</span><span id="line-4959"> public boolean checkAndRowMutate(byte[] row, byte[] family, byte[] qualifier, CompareOperator op,</span> |
| <span class="source-line-no">4960</span><span id="line-4960"> ByteArrayComparable comparator, TimeRange timeRange, RowMutations rm) throws IOException {</span> |
| <span class="source-line-no">4961</span><span id="line-4961"> CheckAndMutate checkAndMutate;</span> |
| <span class="source-line-no">4962</span><span id="line-4962"> try {</span> |
| <span class="source-line-no">4963</span><span id="line-4963"> checkAndMutate = CheckAndMutate.newBuilder(row)</span> |
| <span class="source-line-no">4964</span><span id="line-4964"> .ifMatches(family, qualifier, op, comparator.getValue()).timeRange(timeRange).build(rm);</span> |
| <span class="source-line-no">4965</span><span id="line-4965"> } catch (IllegalArgumentException e) {</span> |
| <span class="source-line-no">4966</span><span id="line-4966"> throw new DoNotRetryIOException(e.getMessage());</span> |
| <span class="source-line-no">4967</span><span id="line-4967"> }</span> |
| <span class="source-line-no">4968</span><span id="line-4968"> return checkAndMutate(checkAndMutate).isSuccess();</span> |
| <span class="source-line-no">4969</span><span id="line-4969"> }</span> |
| <span class="source-line-no">4970</span><span id="line-4970"></span> |
| <span class="source-line-no">4971</span><span id="line-4971"> @Override</span> |
| <span class="source-line-no">4972</span><span id="line-4972"> @Deprecated</span> |
| <span class="source-line-no">4973</span><span id="line-4973"> public boolean checkAndRowMutate(byte[] row, Filter filter, TimeRange timeRange, RowMutations rm)</span> |
| <span class="source-line-no">4974</span><span id="line-4974"> throws IOException {</span> |
| <span class="source-line-no">4975</span><span id="line-4975"> CheckAndMutate checkAndMutate;</span> |
| <span class="source-line-no">4976</span><span id="line-4976"> try {</span> |
| <span class="source-line-no">4977</span><span id="line-4977"> checkAndMutate =</span> |
| <span class="source-line-no">4978</span><span id="line-4978"> CheckAndMutate.newBuilder(row).ifMatches(filter).timeRange(timeRange).build(rm);</span> |
| <span class="source-line-no">4979</span><span id="line-4979"> } catch (IllegalArgumentException e) {</span> |
| <span class="source-line-no">4980</span><span id="line-4980"> throw new DoNotRetryIOException(e.getMessage());</span> |
| <span class="source-line-no">4981</span><span id="line-4981"> }</span> |
| <span class="source-line-no">4982</span><span id="line-4982"> return checkAndMutate(checkAndMutate).isSuccess();</span> |
| <span class="source-line-no">4983</span><span id="line-4983"> }</span> |
| <span class="source-line-no">4984</span><span id="line-4984"></span> |
| <span class="source-line-no">4985</span><span id="line-4985"> @Override</span> |
| <span class="source-line-no">4986</span><span id="line-4986"> public CheckAndMutateResult checkAndMutate(CheckAndMutate checkAndMutate) throws IOException {</span> |
| <span class="source-line-no">4987</span><span id="line-4987"> return checkAndMutate(checkAndMutate, HConstants.NO_NONCE, HConstants.NO_NONCE);</span> |
| <span class="source-line-no">4988</span><span id="line-4988"> }</span> |
| <span class="source-line-no">4989</span><span id="line-4989"></span> |
| <span class="source-line-no">4990</span><span id="line-4990"> public CheckAndMutateResult checkAndMutate(CheckAndMutate checkAndMutate, long nonceGroup,</span> |
| <span class="source-line-no">4991</span><span id="line-4991"> long nonce) throws IOException {</span> |
| <span class="source-line-no">4992</span><span id="line-4992"> return TraceUtil.trace(() -> checkAndMutateInternal(checkAndMutate, nonceGroup, nonce),</span> |
| <span class="source-line-no">4993</span><span id="line-4993"> () -> createRegionSpan("Region.checkAndMutate"));</span> |
| <span class="source-line-no">4994</span><span id="line-4994"> }</span> |
| <span class="source-line-no">4995</span><span id="line-4995"></span> |
| <span class="source-line-no">4996</span><span id="line-4996"> private CheckAndMutateResult checkAndMutateInternal(CheckAndMutate checkAndMutate,</span> |
| <span class="source-line-no">4997</span><span id="line-4997"> long nonceGroup, long nonce) throws IOException {</span> |
| <span class="source-line-no">4998</span><span id="line-4998"> byte[] row = checkAndMutate.getRow();</span> |
| <span class="source-line-no">4999</span><span id="line-4999"> Filter filter = null;</span> |
| <span class="source-line-no">5000</span><span id="line-5000"> byte[] family = null;</span> |
| <span class="source-line-no">5001</span><span id="line-5001"> byte[] qualifier = null;</span> |
| <span class="source-line-no">5002</span><span id="line-5002"> CompareOperator op = null;</span> |
| <span class="source-line-no">5003</span><span id="line-5003"> ByteArrayComparable comparator = null;</span> |
| <span class="source-line-no">5004</span><span id="line-5004"> if (checkAndMutate.hasFilter()) {</span> |
| <span class="source-line-no">5005</span><span id="line-5005"> filter = checkAndMutate.getFilter();</span> |
| <span class="source-line-no">5006</span><span id="line-5006"> } else {</span> |
| <span class="source-line-no">5007</span><span id="line-5007"> family = checkAndMutate.getFamily();</span> |
| <span class="source-line-no">5008</span><span id="line-5008"> qualifier = checkAndMutate.getQualifier();</span> |
| <span class="source-line-no">5009</span><span id="line-5009"> op = checkAndMutate.getCompareOp();</span> |
| <span class="source-line-no">5010</span><span id="line-5010"> comparator = new BinaryComparator(checkAndMutate.getValue());</span> |
| <span class="source-line-no">5011</span><span id="line-5011"> }</span> |
| <span class="source-line-no">5012</span><span id="line-5012"> TimeRange timeRange = checkAndMutate.getTimeRange();</span> |
| <span class="source-line-no">5013</span><span id="line-5013"></span> |
| <span class="source-line-no">5014</span><span id="line-5014"> Mutation mutation = null;</span> |
| <span class="source-line-no">5015</span><span id="line-5015"> RowMutations rowMutations = null;</span> |
| <span class="source-line-no">5016</span><span id="line-5016"> if (checkAndMutate.getAction() instanceof Mutation) {</span> |
| <span class="source-line-no">5017</span><span id="line-5017"> mutation = (Mutation) checkAndMutate.getAction();</span> |
| <span class="source-line-no">5018</span><span id="line-5018"> } else {</span> |
| <span class="source-line-no">5019</span><span id="line-5019"> rowMutations = (RowMutations) checkAndMutate.getAction();</span> |
| <span class="source-line-no">5020</span><span id="line-5020"> }</span> |
| <span class="source-line-no">5021</span><span id="line-5021"></span> |
| <span class="source-line-no">5022</span><span id="line-5022"> if (mutation != null) {</span> |
| <span class="source-line-no">5023</span><span id="line-5023"> checkMutationType(mutation);</span> |
| <span class="source-line-no">5024</span><span id="line-5024"> checkRow(mutation, row);</span> |
| <span class="source-line-no">5025</span><span id="line-5025"> } else {</span> |
| <span class="source-line-no">5026</span><span id="line-5026"> checkRow(rowMutations, row);</span> |
| <span class="source-line-no">5027</span><span id="line-5027"> }</span> |
| <span class="source-line-no">5028</span><span id="line-5028"> checkReadOnly();</span> |
| <span class="source-line-no">5029</span><span id="line-5029"> // TODO, add check for value length also move this check to the client</span> |
| <span class="source-line-no">5030</span><span id="line-5030"> checkResources();</span> |
| <span class="source-line-no">5031</span><span id="line-5031"> startRegionOperation();</span> |
| <span class="source-line-no">5032</span><span id="line-5032"> try {</span> |
| <span class="source-line-no">5033</span><span id="line-5033"> Get get = new Get(row);</span> |
| <span class="source-line-no">5034</span><span id="line-5034"> if (family != null) {</span> |
| <span class="source-line-no">5035</span><span id="line-5035"> checkFamily(family);</span> |
| <span class="source-line-no">5036</span><span id="line-5036"> get.addColumn(family, qualifier);</span> |
| <span class="source-line-no">5037</span><span id="line-5037"> }</span> |
| <span class="source-line-no">5038</span><span id="line-5038"> if (filter != null) {</span> |
| <span class="source-line-no">5039</span><span id="line-5039"> get.setFilter(filter);</span> |
| <span class="source-line-no">5040</span><span id="line-5040"> }</span> |
| <span class="source-line-no">5041</span><span id="line-5041"> if (timeRange != null) {</span> |
| <span class="source-line-no">5042</span><span id="line-5042"> get.setTimeRange(timeRange.getMin(), timeRange.getMax());</span> |
| <span class="source-line-no">5043</span><span id="line-5043"> }</span> |
| <span class="source-line-no">5044</span><span id="line-5044"> // Lock row - note that doBatchMutate will relock this row if called</span> |
| <span class="source-line-no">5045</span><span id="line-5045"> checkRow(row, "doCheckAndRowMutate");</span> |
| <span class="source-line-no">5046</span><span id="line-5046"> RowLock rowLock = getRowLock(get.getRow(), false, null);</span> |
| <span class="source-line-no">5047</span><span id="line-5047"> try {</span> |
| <span class="source-line-no">5048</span><span id="line-5048"> if (this.getCoprocessorHost() != null) {</span> |
| <span class="source-line-no">5049</span><span id="line-5049"> CheckAndMutateResult result =</span> |
| <span class="source-line-no">5050</span><span id="line-5050"> getCoprocessorHost().preCheckAndMutateAfterRowLock(checkAndMutate);</span> |
| <span class="source-line-no">5051</span><span id="line-5051"> if (result != null) {</span> |
| <span class="source-line-no">5052</span><span id="line-5052"> return result;</span> |
| <span class="source-line-no">5053</span><span id="line-5053"> }</span> |
| <span class="source-line-no">5054</span><span id="line-5054"> }</span> |
| <span class="source-line-no">5055</span><span id="line-5055"></span> |
| <span class="source-line-no">5056</span><span id="line-5056"> // NOTE: We used to wait here until mvcc caught up: mvcc.await();</span> |
| <span class="source-line-no">5057</span><span id="line-5057"> // Supposition is that now all changes are done under row locks, then when we go to read,</span> |
| <span class="source-line-no">5058</span><span id="line-5058"> // we'll get the latest on this row.</span> |
| <span class="source-line-no">5059</span><span id="line-5059"> boolean matches = false;</span> |
| <span class="source-line-no">5060</span><span id="line-5060"> long cellTs = 0;</span> |
| <span class="source-line-no">5061</span><span id="line-5061"> try (RegionScanner scanner = getScanner(new Scan(get))) {</span> |
| <span class="source-line-no">5062</span><span id="line-5062"> // NOTE: Please don't use HRegion.get() instead,</span> |
| <span class="source-line-no">5063</span><span id="line-5063"> // because it will copy cells to heap. See HBASE-26036</span> |
| <span class="source-line-no">5064</span><span id="line-5064"> List<Cell> result = new ArrayList<>(1);</span> |
| <span class="source-line-no">5065</span><span id="line-5065"> scanner.next(result);</span> |
| <span class="source-line-no">5066</span><span id="line-5066"> if (filter != null) {</span> |
| <span class="source-line-no">5067</span><span id="line-5067"> if (!result.isEmpty()) {</span> |
| <span class="source-line-no">5068</span><span id="line-5068"> matches = true;</span> |
| <span class="source-line-no">5069</span><span id="line-5069"> cellTs = result.get(0).getTimestamp();</span> |
| <span class="source-line-no">5070</span><span id="line-5070"> }</span> |
| <span class="source-line-no">5071</span><span id="line-5071"> } else {</span> |
| <span class="source-line-no">5072</span><span id="line-5072"> boolean valueIsNull =</span> |
| <span class="source-line-no">5073</span><span id="line-5073"> comparator.getValue() == null || comparator.getValue().length == 0;</span> |
| <span class="source-line-no">5074</span><span id="line-5074"> if (result.isEmpty() && valueIsNull) {</span> |
| <span class="source-line-no">5075</span><span id="line-5075"> matches = op != CompareOperator.NOT_EQUAL;</span> |
| <span class="source-line-no">5076</span><span id="line-5076"> } else if (result.size() > 0 && valueIsNull) {</span> |
| <span class="source-line-no">5077</span><span id="line-5077"> matches = (result.get(0).getValueLength() == 0) == (op != CompareOperator.NOT_EQUAL);</span> |
| <span class="source-line-no">5078</span><span id="line-5078"> cellTs = result.get(0).getTimestamp();</span> |
| <span class="source-line-no">5079</span><span id="line-5079"> } else if (result.size() == 1) {</span> |
| <span class="source-line-no">5080</span><span id="line-5080"> Cell kv = result.get(0);</span> |
| <span class="source-line-no">5081</span><span id="line-5081"> cellTs = kv.getTimestamp();</span> |
| <span class="source-line-no">5082</span><span id="line-5082"> int compareResult = PrivateCellUtil.compareValue(kv, comparator);</span> |
| <span class="source-line-no">5083</span><span id="line-5083"> matches = matches(op, compareResult);</span> |
| <span class="source-line-no">5084</span><span id="line-5084"> }</span> |
| <span class="source-line-no">5085</span><span id="line-5085"> }</span> |
| <span class="source-line-no">5086</span><span id="line-5086"> }</span> |
| <span class="source-line-no">5087</span><span id="line-5087"></span> |
| <span class="source-line-no">5088</span><span id="line-5088"> // If matches, perform the mutation or the rowMutations</span> |
| <span class="source-line-no">5089</span><span id="line-5089"> if (matches) {</span> |
| <span class="source-line-no">5090</span><span id="line-5090"> // We have acquired the row lock already. If the system clock is NOT monotonically</span> |
| <span class="source-line-no">5091</span><span id="line-5091"> // non-decreasing (see HBASE-14070) we should make sure that the mutation has a</span> |
| <span class="source-line-no">5092</span><span id="line-5092"> // larger timestamp than what was observed via Get. doBatchMutate already does this, but</span> |
| <span class="source-line-no">5093</span><span id="line-5093"> // there is no way to pass the cellTs. See HBASE-14054.</span> |
| <span class="source-line-no">5094</span><span id="line-5094"> long now = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">5095</span><span id="line-5095"> long ts = Math.max(now, cellTs); // ensure write is not eclipsed</span> |
| <span class="source-line-no">5096</span><span id="line-5096"> byte[] byteTs = Bytes.toBytes(ts);</span> |
| <span class="source-line-no">5097</span><span id="line-5097"> if (mutation != null) {</span> |
| <span class="source-line-no">5098</span><span id="line-5098"> if (mutation instanceof Put) {</span> |
| <span class="source-line-no">5099</span><span id="line-5099"> updateCellTimestamps(</span> |
| <span class="source-line-no">5100</span><span id="line-5100"> PackagePrivateFieldAccessor.getExtendedFamilyCellMap(mutation).values(), byteTs);</span> |
| <span class="source-line-no">5101</span><span id="line-5101"> }</span> |
| <span class="source-line-no">5102</span><span id="line-5102"> // And else 'delete' is not needed since it already does a second get, and sets the</span> |
| <span class="source-line-no">5103</span><span id="line-5103"> // timestamp from get (see prepareDeleteTimestamps).</span> |
| <span class="source-line-no">5104</span><span id="line-5104"> } else {</span> |
| <span class="source-line-no">5105</span><span id="line-5105"> for (Mutation m : rowMutations.getMutations()) {</span> |
| <span class="source-line-no">5106</span><span id="line-5106"> if (m instanceof Put) {</span> |
| <span class="source-line-no">5107</span><span id="line-5107"> updateCellTimestamps(</span> |
| <span class="source-line-no">5108</span><span id="line-5108"> PackagePrivateFieldAccessor.getExtendedFamilyCellMap(m).values(), byteTs);</span> |
| <span class="source-line-no">5109</span><span id="line-5109"> }</span> |
| <span class="source-line-no">5110</span><span id="line-5110"> }</span> |
| <span class="source-line-no">5111</span><span id="line-5111"> // And else 'delete' is not needed since it already does a second get, and sets the</span> |
| <span class="source-line-no">5112</span><span id="line-5112"> // timestamp from get (see prepareDeleteTimestamps).</span> |
| <span class="source-line-no">5113</span><span id="line-5113"> }</span> |
| <span class="source-line-no">5114</span><span id="line-5114"> // All edits for the given row (across all column families) must happen atomically.</span> |
| <span class="source-line-no">5115</span><span id="line-5115"> Result r;</span> |
| <span class="source-line-no">5116</span><span id="line-5116"> if (mutation != null) {</span> |
| <span class="source-line-no">5117</span><span id="line-5117"> r = mutate(mutation, true, nonceGroup, nonce).getResult();</span> |
| <span class="source-line-no">5118</span><span id="line-5118"> } else {</span> |
| <span class="source-line-no">5119</span><span id="line-5119"> r = mutateRow(rowMutations, nonceGroup, nonce);</span> |
| <span class="source-line-no">5120</span><span id="line-5120"> }</span> |
| <span class="source-line-no">5121</span><span id="line-5121"> this.checkAndMutateChecksPassed.increment();</span> |
| <span class="source-line-no">5122</span><span id="line-5122"> return new CheckAndMutateResult(true, r);</span> |
| <span class="source-line-no">5123</span><span id="line-5123"> }</span> |
| <span class="source-line-no">5124</span><span id="line-5124"> this.checkAndMutateChecksFailed.increment();</span> |
| <span class="source-line-no">5125</span><span id="line-5125"> return new CheckAndMutateResult(false, null);</span> |
| <span class="source-line-no">5126</span><span id="line-5126"> } finally {</span> |
| <span class="source-line-no">5127</span><span id="line-5127"> rowLock.release();</span> |
| <span class="source-line-no">5128</span><span id="line-5128"> }</span> |
| <span class="source-line-no">5129</span><span id="line-5129"> } finally {</span> |
| <span class="source-line-no">5130</span><span id="line-5130"> closeRegionOperation();</span> |
| <span class="source-line-no">5131</span><span id="line-5131"> }</span> |
| <span class="source-line-no">5132</span><span id="line-5132"> }</span> |
| <span class="source-line-no">5133</span><span id="line-5133"></span> |
| <span class="source-line-no">5134</span><span id="line-5134"> private void checkMutationType(final Mutation mutation) throws DoNotRetryIOException {</span> |
| <span class="source-line-no">5135</span><span id="line-5135"> if (</span> |
| <span class="source-line-no">5136</span><span id="line-5136"> !(mutation instanceof Put) && !(mutation instanceof Delete)</span> |
| <span class="source-line-no">5137</span><span id="line-5137"> && !(mutation instanceof Increment) && !(mutation instanceof Append)</span> |
| <span class="source-line-no">5138</span><span id="line-5138"> ) {</span> |
| <span class="source-line-no">5139</span><span id="line-5139"> throw new org.apache.hadoop.hbase.DoNotRetryIOException(</span> |
| <span class="source-line-no">5140</span><span id="line-5140"> "Action must be Put or Delete or Increment or Delete");</span> |
| <span class="source-line-no">5141</span><span id="line-5141"> }</span> |
| <span class="source-line-no">5142</span><span id="line-5142"> }</span> |
| <span class="source-line-no">5143</span><span id="line-5143"></span> |
| <span class="source-line-no">5144</span><span id="line-5144"> private void checkRow(final Row action, final byte[] row) throws DoNotRetryIOException {</span> |
| <span class="source-line-no">5145</span><span id="line-5145"> if (!Bytes.equals(row, action.getRow())) {</span> |
| <span class="source-line-no">5146</span><span id="line-5146"> throw new org.apache.hadoop.hbase.DoNotRetryIOException("Action's getRow must match");</span> |
| <span class="source-line-no">5147</span><span id="line-5147"> }</span> |
| <span class="source-line-no">5148</span><span id="line-5148"> }</span> |
| <span class="source-line-no">5149</span><span id="line-5149"></span> |
| <span class="source-line-no">5150</span><span id="line-5150"> private boolean matches(final CompareOperator op, final int compareResult) {</span> |
| <span class="source-line-no">5151</span><span id="line-5151"> boolean matches = false;</span> |
| <span class="source-line-no">5152</span><span id="line-5152"> switch (op) {</span> |
| <span class="source-line-no">5153</span><span id="line-5153"> case LESS:</span> |
| <span class="source-line-no">5154</span><span id="line-5154"> matches = compareResult < 0;</span> |
| <span class="source-line-no">5155</span><span id="line-5155"> break;</span> |
| <span class="source-line-no">5156</span><span id="line-5156"> case LESS_OR_EQUAL:</span> |
| <span class="source-line-no">5157</span><span id="line-5157"> matches = compareResult <= 0;</span> |
| <span class="source-line-no">5158</span><span id="line-5158"> break;</span> |
| <span class="source-line-no">5159</span><span id="line-5159"> case EQUAL:</span> |
| <span class="source-line-no">5160</span><span id="line-5160"> matches = compareResult == 0;</span> |
| <span class="source-line-no">5161</span><span id="line-5161"> break;</span> |
| <span class="source-line-no">5162</span><span id="line-5162"> case NOT_EQUAL:</span> |
| <span class="source-line-no">5163</span><span id="line-5163"> matches = compareResult != 0;</span> |
| <span class="source-line-no">5164</span><span id="line-5164"> break;</span> |
| <span class="source-line-no">5165</span><span id="line-5165"> case GREATER_OR_EQUAL:</span> |
| <span class="source-line-no">5166</span><span id="line-5166"> matches = compareResult >= 0;</span> |
| <span class="source-line-no">5167</span><span id="line-5167"> break;</span> |
| <span class="source-line-no">5168</span><span id="line-5168"> case GREATER:</span> |
| <span class="source-line-no">5169</span><span id="line-5169"> matches = compareResult > 0;</span> |
| <span class="source-line-no">5170</span><span id="line-5170"> break;</span> |
| <span class="source-line-no">5171</span><span id="line-5171"> default:</span> |
| <span class="source-line-no">5172</span><span id="line-5172"> throw new RuntimeException("Unknown Compare op " + op.name());</span> |
| <span class="source-line-no">5173</span><span id="line-5173"> }</span> |
| <span class="source-line-no">5174</span><span id="line-5174"> return matches;</span> |
| <span class="source-line-no">5175</span><span id="line-5175"> }</span> |
| <span class="source-line-no">5176</span><span id="line-5176"></span> |
| <span class="source-line-no">5177</span><span id="line-5177"> private OperationStatus mutate(Mutation mutation) throws IOException {</span> |
| <span class="source-line-no">5178</span><span id="line-5178"> return mutate(mutation, false);</span> |
| <span class="source-line-no">5179</span><span id="line-5179"> }</span> |
| <span class="source-line-no">5180</span><span id="line-5180"></span> |
| <span class="source-line-no">5181</span><span id="line-5181"> private OperationStatus mutate(Mutation mutation, boolean atomic) throws IOException {</span> |
| <span class="source-line-no">5182</span><span id="line-5182"> return mutate(mutation, atomic, HConstants.NO_NONCE, HConstants.NO_NONCE);</span> |
| <span class="source-line-no">5183</span><span id="line-5183"> }</span> |
| <span class="source-line-no">5184</span><span id="line-5184"></span> |
| <span class="source-line-no">5185</span><span id="line-5185"> private OperationStatus mutate(Mutation mutation, boolean atomic, long nonceGroup, long nonce)</span> |
| <span class="source-line-no">5186</span><span id="line-5186"> throws IOException {</span> |
| <span class="source-line-no">5187</span><span id="line-5187"> OperationStatus[] status =</span> |
| <span class="source-line-no">5188</span><span id="line-5188"> this.batchMutate(new Mutation[] { mutation }, atomic, nonceGroup, nonce);</span> |
| <span class="source-line-no">5189</span><span id="line-5189"> if (status[0].getOperationStatusCode().equals(OperationStatusCode.SANITY_CHECK_FAILURE)) {</span> |
| <span class="source-line-no">5190</span><span id="line-5190"> throw new FailedSanityCheckException(status[0].getExceptionMsg());</span> |
| <span class="source-line-no">5191</span><span id="line-5191"> } else if (status[0].getOperationStatusCode().equals(OperationStatusCode.BAD_FAMILY)) {</span> |
| <span class="source-line-no">5192</span><span id="line-5192"> throw new NoSuchColumnFamilyException(status[0].getExceptionMsg());</span> |
| <span class="source-line-no">5193</span><span id="line-5193"> } else if (status[0].getOperationStatusCode().equals(OperationStatusCode.STORE_TOO_BUSY)) {</span> |
| <span class="source-line-no">5194</span><span id="line-5194"> throw new RegionTooBusyException(status[0].getExceptionMsg());</span> |
| <span class="source-line-no">5195</span><span id="line-5195"> }</span> |
| <span class="source-line-no">5196</span><span id="line-5196"> return status[0];</span> |
| <span class="source-line-no">5197</span><span id="line-5197"> }</span> |
| <span class="source-line-no">5198</span><span id="line-5198"></span> |
| <span class="source-line-no">5199</span><span id="line-5199"> /**</span> |
| <span class="source-line-no">5200</span><span id="line-5200"> * Complete taking the snapshot on the region. Writes the region info and adds references to the</span> |
| <span class="source-line-no">5201</span><span id="line-5201"> * working snapshot directory. TODO for api consistency, consider adding another version with no</span> |
| <span class="source-line-no">5202</span><span id="line-5202"> * {@link ForeignExceptionSnare} arg. (In the future other cancellable HRegion methods could</span> |
| <span class="source-line-no">5203</span><span id="line-5203"> * eventually add a {@link ForeignExceptionSnare}, or we could do something fancier).</span> |
| <span class="source-line-no">5204</span><span id="line-5204"> * @param desc snapshot description object</span> |
| <span class="source-line-no">5205</span><span id="line-5205"> * @param exnSnare ForeignExceptionSnare that captures external exceptions in case we need to bail</span> |
| <span class="source-line-no">5206</span><span id="line-5206"> * out. This is allowed to be null and will just be ignored in that case.</span> |
| <span class="source-line-no">5207</span><span id="line-5207"> * @throws IOException if there is an external or internal error causing the snapshot to fail</span> |
| <span class="source-line-no">5208</span><span id="line-5208"> */</span> |
| <span class="source-line-no">5209</span><span id="line-5209"> public void addRegionToSnapshot(SnapshotDescription desc, ForeignExceptionSnare exnSnare)</span> |
| <span class="source-line-no">5210</span><span id="line-5210"> throws IOException {</span> |
| <span class="source-line-no">5211</span><span id="line-5211"> Path rootDir = CommonFSUtils.getRootDir(conf);</span> |
| <span class="source-line-no">5212</span><span id="line-5212"> Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);</span> |
| <span class="source-line-no">5213</span><span id="line-5213"></span> |
| <span class="source-line-no">5214</span><span id="line-5214"> SnapshotManifest manifest =</span> |
| <span class="source-line-no">5215</span><span id="line-5215"> SnapshotManifest.create(conf, getFilesystem(), snapshotDir, desc, exnSnare);</span> |
| <span class="source-line-no">5216</span><span id="line-5216"> manifest.addRegion(this);</span> |
| <span class="source-line-no">5217</span><span id="line-5217"> }</span> |
| <span class="source-line-no">5218</span><span id="line-5218"></span> |
| <span class="source-line-no">5219</span><span id="line-5219"> private void updateSequenceId(final Iterable<List<ExtendedCell>> cellItr, final long sequenceId)</span> |
| <span class="source-line-no">5220</span><span id="line-5220"> throws IOException {</span> |
| <span class="source-line-no">5221</span><span id="line-5221"> for (List<ExtendedCell> cells : cellItr) {</span> |
| <span class="source-line-no">5222</span><span id="line-5222"> if (cells == null) {</span> |
| <span class="source-line-no">5223</span><span id="line-5223"> return;</span> |
| <span class="source-line-no">5224</span><span id="line-5224"> }</span> |
| <span class="source-line-no">5225</span><span id="line-5225"> for (ExtendedCell cell : cells) {</span> |
| <span class="source-line-no">5226</span><span id="line-5226"> cell.setSequenceId(sequenceId);</span> |
| <span class="source-line-no">5227</span><span id="line-5227"> }</span> |
| <span class="source-line-no">5228</span><span id="line-5228"> }</span> |
| <span class="source-line-no">5229</span><span id="line-5229"> }</span> |
| <span class="source-line-no">5230</span><span id="line-5230"></span> |
| <span class="source-line-no">5231</span><span id="line-5231"> /**</span> |
| <span class="source-line-no">5232</span><span id="line-5232"> * Replace any cell timestamps set to {@link org.apache.hadoop.hbase.HConstants#LATEST_TIMESTAMP}</span> |
| <span class="source-line-no">5233</span><span id="line-5233"> * provided current timestamp.</span> |
| <span class="source-line-no">5234</span><span id="line-5234"> */</span> |
| <span class="source-line-no">5235</span><span id="line-5235"> private static void updateCellTimestamps(final Iterable<List<ExtendedCell>> cellItr,</span> |
| <span class="source-line-no">5236</span><span id="line-5236"> final byte[] now) throws IOException {</span> |
| <span class="source-line-no">5237</span><span id="line-5237"> for (List<ExtendedCell> cells : cellItr) {</span> |
| <span class="source-line-no">5238</span><span id="line-5238"> if (cells == null) {</span> |
| <span class="source-line-no">5239</span><span id="line-5239"> continue;</span> |
| <span class="source-line-no">5240</span><span id="line-5240"> }</span> |
| <span class="source-line-no">5241</span><span id="line-5241"> // Optimization: 'foreach' loop is not used. See:</span> |
| <span class="source-line-no">5242</span><span id="line-5242"> // HBASE-12023 HRegion.applyFamilyMapToMemstore creates too many iterator objects</span> |
| <span class="source-line-no">5243</span><span id="line-5243"> assert cells instanceof RandomAccess;</span> |
| <span class="source-line-no">5244</span><span id="line-5244"> int listSize = cells.size();</span> |
| <span class="source-line-no">5245</span><span id="line-5245"> for (int i = 0; i < listSize; i++) {</span> |
| <span class="source-line-no">5246</span><span id="line-5246"> PrivateCellUtil.updateLatestStamp(cells.get(i), now);</span> |
| <span class="source-line-no">5247</span><span id="line-5247"> }</span> |
| <span class="source-line-no">5248</span><span id="line-5248"> }</span> |
| <span class="source-line-no">5249</span><span id="line-5249"> }</span> |
| <span class="source-line-no">5250</span><span id="line-5250"></span> |
| <span class="source-line-no">5251</span><span id="line-5251"> /**</span> |
| <span class="source-line-no">5252</span><span id="line-5252"> * Possibly rewrite incoming cell tags.</span> |
| <span class="source-line-no">5253</span><span id="line-5253"> */</span> |
| <span class="source-line-no">5254</span><span id="line-5254"> private void rewriteCellTags(Map<byte[], List<ExtendedCell>> familyMap, final Mutation m) {</span> |
| <span class="source-line-no">5255</span><span id="line-5255"> // Check if we have any work to do and early out otherwise</span> |
| <span class="source-line-no">5256</span><span id="line-5256"> // Update these checks as more logic is added here</span> |
| <span class="source-line-no">5257</span><span id="line-5257"> if (m.getTTL() == Long.MAX_VALUE) {</span> |
| <span class="source-line-no">5258</span><span id="line-5258"> return;</span> |
| <span class="source-line-no">5259</span><span id="line-5259"> }</span> |
| <span class="source-line-no">5260</span><span id="line-5260"></span> |
| <span class="source-line-no">5261</span><span id="line-5261"> // From this point we know we have some work to do</span> |
| <span class="source-line-no">5262</span><span id="line-5262"> for (Map.Entry<byte[], List<ExtendedCell>> e : familyMap.entrySet()) {</span> |
| <span class="source-line-no">5263</span><span id="line-5263"> List<ExtendedCell> cells = e.getValue();</span> |
| <span class="source-line-no">5264</span><span id="line-5264"> assert cells instanceof RandomAccess;</span> |
| <span class="source-line-no">5265</span><span id="line-5265"> int listSize = cells.size();</span> |
| <span class="source-line-no">5266</span><span id="line-5266"> for (int i = 0; i < listSize; i++) {</span> |
| <span class="source-line-no">5267</span><span id="line-5267"> ExtendedCell cell = cells.get(i);</span> |
| <span class="source-line-no">5268</span><span id="line-5268"> List<Tag> newTags = TagUtil.carryForwardTags(null, cell);</span> |
| <span class="source-line-no">5269</span><span id="line-5269"> newTags = TagUtil.carryForwardTTLTag(newTags, m.getTTL());</span> |
| <span class="source-line-no">5270</span><span id="line-5270"> // Rewrite the cell with the updated set of tags</span> |
| <span class="source-line-no">5271</span><span id="line-5271"> cells.set(i, PrivateCellUtil.createCell(cell, newTags));</span> |
| <span class="source-line-no">5272</span><span id="line-5272"> }</span> |
| <span class="source-line-no">5273</span><span id="line-5273"> }</span> |
| <span class="source-line-no">5274</span><span id="line-5274"> }</span> |
| <span class="source-line-no">5275</span><span id="line-5275"></span> |
| <span class="source-line-no">5276</span><span id="line-5276"> /**</span> |
| <span class="source-line-no">5277</span><span id="line-5277"> * Check if resources to support an update.</span> |
| <span class="source-line-no">5278</span><span id="line-5278"> * <p/></span> |
| <span class="source-line-no">5279</span><span id="line-5279"> * We throw RegionTooBusyException if above memstore limit and expect client to retry using some</span> |
| <span class="source-line-no">5280</span><span id="line-5280"> * kind of backoff</span> |
| <span class="source-line-no">5281</span><span id="line-5281"> */</span> |
| <span class="source-line-no">5282</span><span id="line-5282"> private void checkResources() throws RegionTooBusyException {</span> |
| <span class="source-line-no">5283</span><span id="line-5283"> // If catalog region, do not impose resource constraints or block updates.</span> |
| <span class="source-line-no">5284</span><span id="line-5284"> if (this.getRegionInfo().isMetaRegion()) {</span> |
| <span class="source-line-no">5285</span><span id="line-5285"> return;</span> |
| <span class="source-line-no">5286</span><span id="line-5286"> }</span> |
| <span class="source-line-no">5287</span><span id="line-5287"></span> |
| <span class="source-line-no">5288</span><span id="line-5288"> MemStoreSize mss = this.memStoreSizing.getMemStoreSize();</span> |
| <span class="source-line-no">5289</span><span id="line-5289"> if (mss.getHeapSize() + mss.getOffHeapSize() > this.blockingMemStoreSize) {</span> |
| <span class="source-line-no">5290</span><span id="line-5290"> blockedRequestsCount.increment();</span> |
| <span class="source-line-no">5291</span><span id="line-5291"> requestFlush();</span> |
| <span class="source-line-no">5292</span><span id="line-5292"> // Don't print current limit because it will vary too much. The message is used as a key</span> |
| <span class="source-line-no">5293</span><span id="line-5293"> // over in RetriesExhaustedWithDetailsException processing.</span> |
| <span class="source-line-no">5294</span><span id="line-5294"> final String regionName =</span> |
| <span class="source-line-no">5295</span><span id="line-5295"> this.getRegionInfo() == null ? "unknown" : this.getRegionInfo().getEncodedName();</span> |
| <span class="source-line-no">5296</span><span id="line-5296"> final String serverName = this.getRegionServerServices() == null</span> |
| <span class="source-line-no">5297</span><span id="line-5297"> ? "unknown"</span> |
| <span class="source-line-no">5298</span><span id="line-5298"> : (this.getRegionServerServices().getServerName() == null</span> |
| <span class="source-line-no">5299</span><span id="line-5299"> ? "unknown"</span> |
| <span class="source-line-no">5300</span><span id="line-5300"> : this.getRegionServerServices().getServerName().toString());</span> |
| <span class="source-line-no">5301</span><span id="line-5301"> RegionTooBusyException rtbe = new RegionTooBusyException("Over memstore limit="</span> |
| <span class="source-line-no">5302</span><span id="line-5302"> + org.apache.hadoop.hbase.procedure2.util.StringUtils.humanSize(this.blockingMemStoreSize)</span> |
| <span class="source-line-no">5303</span><span id="line-5303"> + ", regionName=" + regionName + ", server=" + serverName);</span> |
| <span class="source-line-no">5304</span><span id="line-5304"> LOG.warn("Region is too busy due to exceeding memstore size limit.", rtbe);</span> |
| <span class="source-line-no">5305</span><span id="line-5305"> throw rtbe;</span> |
| <span class="source-line-no">5306</span><span id="line-5306"> }</span> |
| <span class="source-line-no">5307</span><span id="line-5307"> }</span> |
| <span class="source-line-no">5308</span><span id="line-5308"></span> |
| <span class="source-line-no">5309</span><span id="line-5309"> /**</span> |
| <span class="source-line-no">5310</span><span id="line-5310"> * @throws IOException Throws exception if region is in read-only mode.</span> |
| <span class="source-line-no">5311</span><span id="line-5311"> */</span> |
| <span class="source-line-no">5312</span><span id="line-5312"> private void checkReadOnly() throws IOException {</span> |
| <span class="source-line-no">5313</span><span id="line-5313"> if (isReadOnly()) {</span> |
| <span class="source-line-no">5314</span><span id="line-5314"> throw new DoNotRetryIOException("region is read only");</span> |
| <span class="source-line-no">5315</span><span id="line-5315"> }</span> |
| <span class="source-line-no">5316</span><span id="line-5316"> }</span> |
| <span class="source-line-no">5317</span><span id="line-5317"></span> |
| <span class="source-line-no">5318</span><span id="line-5318"> private void checkReadsEnabled() throws IOException {</span> |
| <span class="source-line-no">5319</span><span id="line-5319"> if (!this.writestate.readsEnabled) {</span> |
| <span class="source-line-no">5320</span><span id="line-5320"> throw new IOException(getRegionInfo().getEncodedName()</span> |
| <span class="source-line-no">5321</span><span id="line-5321"> + ": The region's reads are disabled. Cannot serve the request");</span> |
| <span class="source-line-no">5322</span><span id="line-5322"> }</span> |
| <span class="source-line-no">5323</span><span id="line-5323"> }</span> |
| <span class="source-line-no">5324</span><span id="line-5324"></span> |
| <span class="source-line-no">5325</span><span id="line-5325"> public void setReadsEnabled(boolean readsEnabled) {</span> |
| <span class="source-line-no">5326</span><span id="line-5326"> if (readsEnabled && !this.writestate.readsEnabled) {</span> |
| <span class="source-line-no">5327</span><span id="line-5327"> LOG.info("Enabling reads for {}", getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">5328</span><span id="line-5328"> }</span> |
| <span class="source-line-no">5329</span><span id="line-5329"> this.writestate.setReadsEnabled(readsEnabled);</span> |
| <span class="source-line-no">5330</span><span id="line-5330"> }</span> |
| <span class="source-line-no">5331</span><span id="line-5331"></span> |
| <span class="source-line-no">5332</span><span id="line-5332"> /**</span> |
| <span class="source-line-no">5333</span><span id="line-5333"> * @param delta If we are doing delta changes -- e.g. increment/append -- then this flag will be</span> |
| <span class="source-line-no">5334</span><span id="line-5334"> * set; when set we will run operations that make sense in the increment/append</span> |
| <span class="source-line-no">5335</span><span id="line-5335"> * scenario but that do not make sense otherwise.</span> |
| <span class="source-line-no">5336</span><span id="line-5336"> */</span> |
| <span class="source-line-no">5337</span><span id="line-5337"> private void applyToMemStore(HStore store, List<ExtendedCell> cells, boolean delta,</span> |
| <span class="source-line-no">5338</span><span id="line-5338"> MemStoreSizing memstoreAccounting) {</span> |
| <span class="source-line-no">5339</span><span id="line-5339"> // Any change in how we update Store/MemStore needs to also be done in other applyToMemStore!!!!</span> |
| <span class="source-line-no">5340</span><span id="line-5340"> boolean upsert = delta && store.getColumnFamilyDescriptor().getMaxVersions() == 1;</span> |
| <span class="source-line-no">5341</span><span id="line-5341"> if (upsert) {</span> |
| <span class="source-line-no">5342</span><span id="line-5342"> store.upsert(cells, getSmallestReadPoint(), memstoreAccounting);</span> |
| <span class="source-line-no">5343</span><span id="line-5343"> } else {</span> |
| <span class="source-line-no">5344</span><span id="line-5344"> store.add(cells, memstoreAccounting);</span> |
| <span class="source-line-no">5345</span><span id="line-5345"> }</span> |
| <span class="source-line-no">5346</span><span id="line-5346"> }</span> |
| <span class="source-line-no">5347</span><span id="line-5347"></span> |
| <span class="source-line-no">5348</span><span id="line-5348"> private void checkFamilies(Collection<byte[]> families, Durability durability)</span> |
| <span class="source-line-no">5349</span><span id="line-5349"> throws NoSuchColumnFamilyException, InvalidMutationDurabilityException {</span> |
| <span class="source-line-no">5350</span><span id="line-5350"> for (byte[] family : families) {</span> |
| <span class="source-line-no">5351</span><span id="line-5351"> checkFamily(family, durability);</span> |
| <span class="source-line-no">5352</span><span id="line-5352"> }</span> |
| <span class="source-line-no">5353</span><span id="line-5353"> }</span> |
| <span class="source-line-no">5354</span><span id="line-5354"></span> |
| <span class="source-line-no">5355</span><span id="line-5355"> private void checkFamily(final byte[] family, Durability durability)</span> |
| <span class="source-line-no">5356</span><span id="line-5356"> throws NoSuchColumnFamilyException, InvalidMutationDurabilityException {</span> |
| <span class="source-line-no">5357</span><span id="line-5357"> checkFamily(family);</span> |
| <span class="source-line-no">5358</span><span id="line-5358"> if (</span> |
| <span class="source-line-no">5359</span><span id="line-5359"> durability.equals(Durability.SKIP_WAL)</span> |
| <span class="source-line-no">5360</span><span id="line-5360"> && htableDescriptor.getColumnFamily(family).getScope() != HConstants.REPLICATION_SCOPE_LOCAL</span> |
| <span class="source-line-no">5361</span><span id="line-5361"> ) {</span> |
| <span class="source-line-no">5362</span><span id="line-5362"> throw new InvalidMutationDurabilityException(</span> |
| <span class="source-line-no">5363</span><span id="line-5363"> "Mutation's durability is SKIP_WAL but table's column family " + Bytes.toString(family)</span> |
| <span class="source-line-no">5364</span><span id="line-5364"> + " need replication");</span> |
| <span class="source-line-no">5365</span><span id="line-5365"> }</span> |
| <span class="source-line-no">5366</span><span id="line-5366"> }</span> |
| <span class="source-line-no">5367</span><span id="line-5367"></span> |
| <span class="source-line-no">5368</span><span id="line-5368"> private void checkFamily(final byte[] family) throws NoSuchColumnFamilyException {</span> |
| <span class="source-line-no">5369</span><span id="line-5369"> if (!this.htableDescriptor.hasColumnFamily(family)) {</span> |
| <span class="source-line-no">5370</span><span id="line-5370"> throw new NoSuchColumnFamilyException("Column family " + Bytes.toString(family)</span> |
| <span class="source-line-no">5371</span><span id="line-5371"> + " does not exist in region " + this + " in table " + this.htableDescriptor);</span> |
| <span class="source-line-no">5372</span><span id="line-5372"> }</span> |
| <span class="source-line-no">5373</span><span id="line-5373"> }</span> |
| <span class="source-line-no">5374</span><span id="line-5374"></span> |
| <span class="source-line-no">5375</span><span id="line-5375"> /**</span> |
| <span class="source-line-no">5376</span><span id="line-5376"> * Check the collection of families for valid timestamps</span> |
| <span class="source-line-no">5377</span><span id="line-5377"> * @param now current timestamp</span> |
| <span class="source-line-no">5378</span><span id="line-5378"> */</span> |
| <span class="source-line-no">5379</span><span id="line-5379"> public void checkTimestamps(final Map<byte[], List<Cell>> familyMap, long now)</span> |
| <span class="source-line-no">5380</span><span id="line-5380"> throws FailedSanityCheckException {</span> |
| <span class="source-line-no">5381</span><span id="line-5381"> if (timestampSlop == HConstants.LATEST_TIMESTAMP) {</span> |
| <span class="source-line-no">5382</span><span id="line-5382"> return;</span> |
| <span class="source-line-no">5383</span><span id="line-5383"> }</span> |
| <span class="source-line-no">5384</span><span id="line-5384"> long maxTs = now + timestampSlop;</span> |
| <span class="source-line-no">5385</span><span id="line-5385"> for (List<Cell> kvs : familyMap.values()) {</span> |
| <span class="source-line-no">5386</span><span id="line-5386"> // Optimization: 'foreach' loop is not used. See:</span> |
| <span class="source-line-no">5387</span><span id="line-5387"> // HBASE-12023 HRegion.applyFamilyMapToMemstore creates too many iterator objects</span> |
| <span class="source-line-no">5388</span><span id="line-5388"> assert kvs instanceof RandomAccess;</span> |
| <span class="source-line-no">5389</span><span id="line-5389"> int listSize = kvs.size();</span> |
| <span class="source-line-no">5390</span><span id="line-5390"> for (int i = 0; i < listSize; i++) {</span> |
| <span class="source-line-no">5391</span><span id="line-5391"> Cell cell = kvs.get(i);</span> |
| <span class="source-line-no">5392</span><span id="line-5392"> // see if the user-side TS is out of range. latest = server-side</span> |
| <span class="source-line-no">5393</span><span id="line-5393"> long ts = cell.getTimestamp();</span> |
| <span class="source-line-no">5394</span><span id="line-5394"> if (ts != HConstants.LATEST_TIMESTAMP && ts > maxTs) {</span> |
| <span class="source-line-no">5395</span><span id="line-5395"> throw new FailedSanityCheckException(</span> |
| <span class="source-line-no">5396</span><span id="line-5396"> "Timestamp for KV out of range " + cell + " (too.new=" + timestampSlop + ")");</span> |
| <span class="source-line-no">5397</span><span id="line-5397"> }</span> |
| <span class="source-line-no">5398</span><span id="line-5398"> }</span> |
| <span class="source-line-no">5399</span><span id="line-5399"> }</span> |
| <span class="source-line-no">5400</span><span id="line-5400"> }</span> |
| <span class="source-line-no">5401</span><span id="line-5401"></span> |
| <span class="source-line-no">5402</span><span id="line-5402"> /*</span> |
| <span class="source-line-no">5403</span><span id="line-5403"> * @return True if size is over the flush threshold</span> |
| <span class="source-line-no">5404</span><span id="line-5404"> */</span> |
| <span class="source-line-no">5405</span><span id="line-5405"> private boolean isFlushSize(MemStoreSize size) {</span> |
| <span class="source-line-no">5406</span><span id="line-5406"> return size.getHeapSize() + size.getOffHeapSize() > getMemStoreFlushSize();</span> |
| <span class="source-line-no">5407</span><span id="line-5407"> }</span> |
| <span class="source-line-no">5408</span><span id="line-5408"></span> |
| <span class="source-line-no">5409</span><span id="line-5409"> private void deleteRecoveredEdits(FileSystem fs, Iterable<Path> files) throws IOException {</span> |
| <span class="source-line-no">5410</span><span id="line-5410"> for (Path file : files) {</span> |
| <span class="source-line-no">5411</span><span id="line-5411"> if (!fs.delete(file, false)) {</span> |
| <span class="source-line-no">5412</span><span id="line-5412"> LOG.error("Failed delete of {}", file);</span> |
| <span class="source-line-no">5413</span><span id="line-5413"> } else {</span> |
| <span class="source-line-no">5414</span><span id="line-5414"> LOG.debug("Deleted recovered.edits file={}", file);</span> |
| <span class="source-line-no">5415</span><span id="line-5415"> }</span> |
| <span class="source-line-no">5416</span><span id="line-5416"> }</span> |
| <span class="source-line-no">5417</span><span id="line-5417"> }</span> |
| <span class="source-line-no">5418</span><span id="line-5418"></span> |
| <span class="source-line-no">5419</span><span id="line-5419"> /**</span> |
| <span class="source-line-no">5420</span><span id="line-5420"> * Read the edits put under this region by wal splitting process. Put the recovered edits back up</span> |
| <span class="source-line-no">5421</span><span id="line-5421"> * into this region.</span> |
| <span class="source-line-no">5422</span><span id="line-5422"> * <p></span> |
| <span class="source-line-no">5423</span><span id="line-5423"> * We can ignore any wal message that has a sequence ID that's equal to or lower than minSeqId.</span> |
| <span class="source-line-no">5424</span><span id="line-5424"> * (Because we know such messages are already reflected in the HFiles.)</span> |
| <span class="source-line-no">5425</span><span id="line-5425"> * <p></span> |
| <span class="source-line-no">5426</span><span id="line-5426"> * While this is running we are putting pressure on memory yet we are outside of our usual</span> |
| <span class="source-line-no">5427</span><span id="line-5427"> * accounting because we are not yet an onlined region (this stuff is being run as part of Region</span> |
| <span class="source-line-no">5428</span><span id="line-5428"> * initialization). This means that if we're up against global memory limits, we'll not be flagged</span> |
| <span class="source-line-no">5429</span><span id="line-5429"> * to flush because we are not online. We can't be flushed by usual mechanisms anyways; we're not</span> |
| <span class="source-line-no">5430</span><span id="line-5430"> * yet online so our relative sequenceids are not yet aligned with WAL sequenceids -- not till we</span> |
| <span class="source-line-no">5431</span><span id="line-5431"> * come up online, post processing of split edits.</span> |
| <span class="source-line-no">5432</span><span id="line-5432"> * <p></span> |
| <span class="source-line-no">5433</span><span id="line-5433"> * But to help relieve memory pressure, at least manage our own heap size flushing if are in</span> |
| <span class="source-line-no">5434</span><span id="line-5434"> * excess of per-region limits. Flushing, though, we have to be careful and avoid using the</span> |
| <span class="source-line-no">5435</span><span id="line-5435"> * regionserver/wal sequenceid. Its running on a different line to whats going on in here in this</span> |
| <span class="source-line-no">5436</span><span id="line-5436"> * region context so if we crashed replaying these edits, but in the midst had a flush that used</span> |
| <span class="source-line-no">5437</span><span id="line-5437"> * the regionserver wal with a sequenceid in excess of whats going on in here in this region and</span> |
| <span class="source-line-no">5438</span><span id="line-5438"> * with its split editlogs, then we could miss edits the next time we go to recover. So, we have</span> |
| <span class="source-line-no">5439</span><span id="line-5439"> * to flush inline, using seqids that make sense in a this single region context only -- until we</span> |
| <span class="source-line-no">5440</span><span id="line-5440"> * online.</span> |
| <span class="source-line-no">5441</span><span id="line-5441"> * @param maxSeqIdInStores Any edit found in split editlogs needs to be in excess of the maxSeqId</span> |
| <span class="source-line-no">5442</span><span id="line-5442"> * for the store to be applied, else its skipped.</span> |
| <span class="source-line-no">5443</span><span id="line-5443"> * @return the sequence id of the last edit added to this region out of the recovered edits log or</span> |
| <span class="source-line-no">5444</span><span id="line-5444"> * <code>minSeqId</code> if nothing added from editlogs.</span> |
| <span class="source-line-no">5445</span><span id="line-5445"> */</span> |
| <span class="source-line-no">5446</span><span id="line-5446"> long replayRecoveredEditsIfAny(Map<byte[], Long> maxSeqIdInStores,</span> |
| <span class="source-line-no">5447</span><span id="line-5447"> final CancelableProgressable reporter, final MonitoredTask status) throws IOException {</span> |
| <span class="source-line-no">5448</span><span id="line-5448"> long minSeqIdForTheRegion = -1;</span> |
| <span class="source-line-no">5449</span><span id="line-5449"> for (Long maxSeqIdInStore : maxSeqIdInStores.values()) {</span> |
| <span class="source-line-no">5450</span><span id="line-5450"> if (maxSeqIdInStore < minSeqIdForTheRegion || minSeqIdForTheRegion == -1) {</span> |
| <span class="source-line-no">5451</span><span id="line-5451"> minSeqIdForTheRegion = maxSeqIdInStore;</span> |
| <span class="source-line-no">5452</span><span id="line-5452"> }</span> |
| <span class="source-line-no">5453</span><span id="line-5453"> }</span> |
| <span class="source-line-no">5454</span><span id="line-5454"> long seqId = minSeqIdForTheRegion;</span> |
| <span class="source-line-no">5455</span><span id="line-5455"> String specialRecoveredEditsDirStr = conf.get(SPECIAL_RECOVERED_EDITS_DIR);</span> |
| <span class="source-line-no">5456</span><span id="line-5456"> if (org.apache.commons.lang3.StringUtils.isBlank(specialRecoveredEditsDirStr)) {</span> |
| <span class="source-line-no">5457</span><span id="line-5457"> FileSystem walFS = getWalFileSystem();</span> |
| <span class="source-line-no">5458</span><span id="line-5458"> FileSystem rootFS = getFilesystem();</span> |
| <span class="source-line-no">5459</span><span id="line-5459"> Path wrongRegionWALDir = CommonFSUtils.getWrongWALRegionDir(conf, getRegionInfo().getTable(),</span> |
| <span class="source-line-no">5460</span><span id="line-5460"> getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">5461</span><span id="line-5461"> Path regionWALDir = getWALRegionDir();</span> |
| <span class="source-line-no">5462</span><span id="line-5462"> Path regionDir =</span> |
| <span class="source-line-no">5463</span><span id="line-5463"> FSUtils.getRegionDirFromRootDir(CommonFSUtils.getRootDir(conf), getRegionInfo());</span> |
| <span class="source-line-no">5464</span><span id="line-5464"></span> |
| <span class="source-line-no">5465</span><span id="line-5465"> // We made a mistake in HBASE-20734 so we need to do this dirty hack...</span> |
| <span class="source-line-no">5466</span><span id="line-5466"> NavigableSet<Path> filesUnderWrongRegionWALDir =</span> |
| <span class="source-line-no">5467</span><span id="line-5467"> WALSplitUtil.getSplitEditFilesSorted(walFS, wrongRegionWALDir);</span> |
| <span class="source-line-no">5468</span><span id="line-5468"> seqId = Math.max(seqId, replayRecoveredEditsForPaths(minSeqIdForTheRegion, walFS,</span> |
| <span class="source-line-no">5469</span><span id="line-5469"> filesUnderWrongRegionWALDir, reporter, regionDir));</span> |
| <span class="source-line-no">5470</span><span id="line-5470"> // This is to ensure backwards compatability with HBASE-20723 where recovered edits can appear</span> |
| <span class="source-line-no">5471</span><span id="line-5471"> // under the root dir even if walDir is set.</span> |
| <span class="source-line-no">5472</span><span id="line-5472"> NavigableSet<Path> filesUnderRootDir = Collections.emptyNavigableSet();</span> |
| <span class="source-line-no">5473</span><span id="line-5473"> if (!regionWALDir.equals(regionDir)) {</span> |
| <span class="source-line-no">5474</span><span id="line-5474"> filesUnderRootDir = WALSplitUtil.getSplitEditFilesSorted(rootFS, regionDir);</span> |
| <span class="source-line-no">5475</span><span id="line-5475"> seqId = Math.max(seqId, replayRecoveredEditsForPaths(minSeqIdForTheRegion, rootFS,</span> |
| <span class="source-line-no">5476</span><span id="line-5476"> filesUnderRootDir, reporter, regionDir));</span> |
| <span class="source-line-no">5477</span><span id="line-5477"> }</span> |
| <span class="source-line-no">5478</span><span id="line-5478"></span> |
| <span class="source-line-no">5479</span><span id="line-5479"> NavigableSet<Path> files = WALSplitUtil.getSplitEditFilesSorted(walFS, regionWALDir);</span> |
| <span class="source-line-no">5480</span><span id="line-5480"> seqId = Math.max(seqId,</span> |
| <span class="source-line-no">5481</span><span id="line-5481"> replayRecoveredEditsForPaths(minSeqIdForTheRegion, walFS, files, reporter, regionWALDir));</span> |
| <span class="source-line-no">5482</span><span id="line-5482"> if (seqId > minSeqIdForTheRegion) {</span> |
| <span class="source-line-no">5483</span><span id="line-5483"> // Then we added some edits to memory. Flush and cleanup split edit files.</span> |
| <span class="source-line-no">5484</span><span id="line-5484"> internalFlushcache(null, seqId, stores.values(), status, false,</span> |
| <span class="source-line-no">5485</span><span id="line-5485"> FlushLifeCycleTracker.DUMMY);</span> |
| <span class="source-line-no">5486</span><span id="line-5486"> }</span> |
| <span class="source-line-no">5487</span><span id="line-5487"> // Now delete the content of recovered edits. We're done w/ them.</span> |
| <span class="source-line-no">5488</span><span id="line-5488"> if (files.size() > 0 && this.conf.getBoolean("hbase.region.archive.recovered.edits", false)) {</span> |
| <span class="source-line-no">5489</span><span id="line-5489"> // For debugging data loss issues!</span> |
| <span class="source-line-no">5490</span><span id="line-5490"> // If this flag is set, make use of the hfile archiving by making recovered.edits a fake</span> |
| <span class="source-line-no">5491</span><span id="line-5491"> // column family. Have to fake out file type too by casting our recovered.edits as</span> |
| <span class="source-line-no">5492</span><span id="line-5492"> // storefiles</span> |
| <span class="source-line-no">5493</span><span id="line-5493"> String fakeFamilyName = WALSplitUtil.getRegionDirRecoveredEditsDir(regionWALDir).getName();</span> |
| <span class="source-line-no">5494</span><span id="line-5494"> Set<HStoreFile> fakeStoreFiles = new HashSet<>(files.size());</span> |
| <span class="source-line-no">5495</span><span id="line-5495"> for (Path file : files) {</span> |
| <span class="source-line-no">5496</span><span id="line-5496"> fakeStoreFiles.add(new HStoreFile(walFS, file, this.conf, null, null, true));</span> |
| <span class="source-line-no">5497</span><span id="line-5497"> }</span> |
| <span class="source-line-no">5498</span><span id="line-5498"> getRegionWALFileSystem().archiveRecoveredEdits(fakeFamilyName, fakeStoreFiles);</span> |
| <span class="source-line-no">5499</span><span id="line-5499"> } else {</span> |
| <span class="source-line-no">5500</span><span id="line-5500"> deleteRecoveredEdits(walFS, Iterables.concat(files, filesUnderWrongRegionWALDir));</span> |
| <span class="source-line-no">5501</span><span id="line-5501"> deleteRecoveredEdits(rootFS, filesUnderRootDir);</span> |
| <span class="source-line-no">5502</span><span id="line-5502"> }</span> |
| <span class="source-line-no">5503</span><span id="line-5503"> } else {</span> |
| <span class="source-line-no">5504</span><span id="line-5504"> Path recoveredEditsDir = new Path(specialRecoveredEditsDirStr);</span> |
| <span class="source-line-no">5505</span><span id="line-5505"> FileSystem fs = recoveredEditsDir.getFileSystem(conf);</span> |
| <span class="source-line-no">5506</span><span id="line-5506"> FileStatus[] files = fs.listStatus(recoveredEditsDir);</span> |
| <span class="source-line-no">5507</span><span id="line-5507"> LOG.debug("Found {} recovered edits file(s) under {}", files == null ? 0 : files.length,</span> |
| <span class="source-line-no">5508</span><span id="line-5508"> recoveredEditsDir);</span> |
| <span class="source-line-no">5509</span><span id="line-5509"> if (files != null) {</span> |
| <span class="source-line-no">5510</span><span id="line-5510"> for (FileStatus file : files) {</span> |
| <span class="source-line-no">5511</span><span id="line-5511"> // it is safe to trust the zero-length in this case because we've been through rename and</span> |
| <span class="source-line-no">5512</span><span id="line-5512"> // lease recovery in the above.</span> |
| <span class="source-line-no">5513</span><span id="line-5513"> if (isZeroLengthThenDelete(fs, file, file.getPath())) {</span> |
| <span class="source-line-no">5514</span><span id="line-5514"> continue;</span> |
| <span class="source-line-no">5515</span><span id="line-5515"> }</span> |
| <span class="source-line-no">5516</span><span id="line-5516"> seqId =</span> |
| <span class="source-line-no">5517</span><span id="line-5517"> Math.max(seqId, replayRecoveredEdits(file.getPath(), maxSeqIdInStores, reporter, fs));</span> |
| <span class="source-line-no">5518</span><span id="line-5518"> }</span> |
| <span class="source-line-no">5519</span><span id="line-5519"> }</span> |
| <span class="source-line-no">5520</span><span id="line-5520"> if (seqId > minSeqIdForTheRegion) {</span> |
| <span class="source-line-no">5521</span><span id="line-5521"> // Then we added some edits to memory. Flush and cleanup split edit files.</span> |
| <span class="source-line-no">5522</span><span id="line-5522"> internalFlushcache(null, seqId, stores.values(), status, false,</span> |
| <span class="source-line-no">5523</span><span id="line-5523"> FlushLifeCycleTracker.DUMMY);</span> |
| <span class="source-line-no">5524</span><span id="line-5524"> }</span> |
| <span class="source-line-no">5525</span><span id="line-5525"> deleteRecoveredEdits(fs,</span> |
| <span class="source-line-no">5526</span><span id="line-5526"> Stream.of(files).map(FileStatus::getPath).collect(Collectors.toList()));</span> |
| <span class="source-line-no">5527</span><span id="line-5527"> }</span> |
| <span class="source-line-no">5528</span><span id="line-5528"></span> |
| <span class="source-line-no">5529</span><span id="line-5529"> return seqId;</span> |
| <span class="source-line-no">5530</span><span id="line-5530"> }</span> |
| <span class="source-line-no">5531</span><span id="line-5531"></span> |
| <span class="source-line-no">5532</span><span id="line-5532"> private long replayRecoveredEditsForPaths(long minSeqIdForTheRegion, FileSystem fs,</span> |
| <span class="source-line-no">5533</span><span id="line-5533"> final NavigableSet<Path> files, final CancelableProgressable reporter, final Path regionDir)</span> |
| <span class="source-line-no">5534</span><span id="line-5534"> throws IOException {</span> |
| <span class="source-line-no">5535</span><span id="line-5535"> long seqid = minSeqIdForTheRegion;</span> |
| <span class="source-line-no">5536</span><span id="line-5536"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">5537</span><span id="line-5537"> LOG.debug("Found " + (files == null ? 0 : files.size()) + " recovered edits file(s) under "</span> |
| <span class="source-line-no">5538</span><span id="line-5538"> + regionDir);</span> |
| <span class="source-line-no">5539</span><span id="line-5539"> }</span> |
| <span class="source-line-no">5540</span><span id="line-5540"></span> |
| <span class="source-line-no">5541</span><span id="line-5541"> if (files == null || files.isEmpty()) {</span> |
| <span class="source-line-no">5542</span><span id="line-5542"> return minSeqIdForTheRegion;</span> |
| <span class="source-line-no">5543</span><span id="line-5543"> }</span> |
| <span class="source-line-no">5544</span><span id="line-5544"></span> |
| <span class="source-line-no">5545</span><span id="line-5545"> for (Path edits : files) {</span> |
| <span class="source-line-no">5546</span><span id="line-5546"> if (edits == null || !fs.exists(edits)) {</span> |
| <span class="source-line-no">5547</span><span id="line-5547"> LOG.warn("Null or non-existent edits file: " + edits);</span> |
| <span class="source-line-no">5548</span><span id="line-5548"> continue;</span> |
| <span class="source-line-no">5549</span><span id="line-5549"> }</span> |
| <span class="source-line-no">5550</span><span id="line-5550"> if (isZeroLengthThenDelete(fs, fs.getFileStatus(edits), edits)) {</span> |
| <span class="source-line-no">5551</span><span id="line-5551"> continue;</span> |
| <span class="source-line-no">5552</span><span id="line-5552"> }</span> |
| <span class="source-line-no">5553</span><span id="line-5553"></span> |
| <span class="source-line-no">5554</span><span id="line-5554"> long maxSeqId;</span> |
| <span class="source-line-no">5555</span><span id="line-5555"> String fileName = edits.getName();</span> |
| <span class="source-line-no">5556</span><span id="line-5556"> maxSeqId = Math.abs(Long.parseLong(fileName));</span> |
| <span class="source-line-no">5557</span><span id="line-5557"> if (maxSeqId <= minSeqIdForTheRegion) {</span> |
| <span class="source-line-no">5558</span><span id="line-5558"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">5559</span><span id="line-5559"> String msg = "Maximum sequenceid for this wal is " + maxSeqId</span> |
| <span class="source-line-no">5560</span><span id="line-5560"> + " and minimum sequenceid for the region " + this + " is " + minSeqIdForTheRegion</span> |
| <span class="source-line-no">5561</span><span id="line-5561"> + ", skipped the whole file, path=" + edits;</span> |
| <span class="source-line-no">5562</span><span id="line-5562"> LOG.debug(msg);</span> |
| <span class="source-line-no">5563</span><span id="line-5563"> }</span> |
| <span class="source-line-no">5564</span><span id="line-5564"> continue;</span> |
| <span class="source-line-no">5565</span><span id="line-5565"> }</span> |
| <span class="source-line-no">5566</span><span id="line-5566"></span> |
| <span class="source-line-no">5567</span><span id="line-5567"> try {</span> |
| <span class="source-line-no">5568</span><span id="line-5568"> // replay the edits. Replay can return -1 if everything is skipped, only update</span> |
| <span class="source-line-no">5569</span><span id="line-5569"> // if seqId is greater</span> |
| <span class="source-line-no">5570</span><span id="line-5570"> seqid = Math.max(seqid, replayRecoveredEdits(edits, maxSeqIdInStores, reporter, fs));</span> |
| <span class="source-line-no">5571</span><span id="line-5571"> } catch (IOException e) {</span> |
| <span class="source-line-no">5572</span><span id="line-5572"> handleException(fs, edits, e);</span> |
| <span class="source-line-no">5573</span><span id="line-5573"> }</span> |
| <span class="source-line-no">5574</span><span id="line-5574"> }</span> |
| <span class="source-line-no">5575</span><span id="line-5575"> return seqid;</span> |
| <span class="source-line-no">5576</span><span id="line-5576"> }</span> |
| <span class="source-line-no">5577</span><span id="line-5577"></span> |
| <span class="source-line-no">5578</span><span id="line-5578"> private void handleException(FileSystem fs, Path edits, IOException e) throws IOException {</span> |
| <span class="source-line-no">5579</span><span id="line-5579"> boolean skipErrors = conf.getBoolean(HConstants.HREGION_EDITS_REPLAY_SKIP_ERRORS,</span> |
| <span class="source-line-no">5580</span><span id="line-5580"> conf.getBoolean("hbase.skip.errors", HConstants.DEFAULT_HREGION_EDITS_REPLAY_SKIP_ERRORS));</span> |
| <span class="source-line-no">5581</span><span id="line-5581"> if (conf.get("hbase.skip.errors") != null) {</span> |
| <span class="source-line-no">5582</span><span id="line-5582"> LOG.warn("The property 'hbase.skip.errors' has been deprecated. Please use "</span> |
| <span class="source-line-no">5583</span><span id="line-5583"> + HConstants.HREGION_EDITS_REPLAY_SKIP_ERRORS + " instead.");</span> |
| <span class="source-line-no">5584</span><span id="line-5584"> }</span> |
| <span class="source-line-no">5585</span><span id="line-5585"> if (skipErrors) {</span> |
| <span class="source-line-no">5586</span><span id="line-5586"> Path p = WALSplitUtil.moveAsideBadEditsFile(fs, edits);</span> |
| <span class="source-line-no">5587</span><span id="line-5587"> LOG.error(HConstants.HREGION_EDITS_REPLAY_SKIP_ERRORS + "=true so continuing. Renamed "</span> |
| <span class="source-line-no">5588</span><span id="line-5588"> + edits + " as " + p, e);</span> |
| <span class="source-line-no">5589</span><span id="line-5589"> } else {</span> |
| <span class="source-line-no">5590</span><span id="line-5590"> throw e;</span> |
| <span class="source-line-no">5591</span><span id="line-5591"> }</span> |
| <span class="source-line-no">5592</span><span id="line-5592"> }</span> |
| <span class="source-line-no">5593</span><span id="line-5593"></span> |
| <span class="source-line-no">5594</span><span id="line-5594"> /**</span> |
| <span class="source-line-no">5595</span><span id="line-5595"> * @param edits File of recovered edits.</span> |
| <span class="source-line-no">5596</span><span id="line-5596"> * @param maxSeqIdInStores Maximum sequenceid found in each store. Edits in wal must be larger</span> |
| <span class="source-line-no">5597</span><span id="line-5597"> * than this to be replayed for each store.</span> |
| <span class="source-line-no">5598</span><span id="line-5598"> * @return the sequence id of the last edit added to this region out of the recovered edits log or</span> |
| <span class="source-line-no">5599</span><span id="line-5599"> * <code>minSeqId</code> if nothing added from editlogs.</span> |
| <span class="source-line-no">5600</span><span id="line-5600"> */</span> |
| <span class="source-line-no">5601</span><span id="line-5601"> private long replayRecoveredEdits(final Path edits, Map<byte[], Long> maxSeqIdInStores,</span> |
| <span class="source-line-no">5602</span><span id="line-5602"> final CancelableProgressable reporter, FileSystem fs) throws IOException {</span> |
| <span class="source-line-no">5603</span><span id="line-5603"> String msg = "Replaying edits from " + edits;</span> |
| <span class="source-line-no">5604</span><span id="line-5604"> LOG.info(msg);</span> |
| <span class="source-line-no">5605</span><span id="line-5605"> MonitoredTask status = TaskMonitor.get().createStatus(msg);</span> |
| <span class="source-line-no">5606</span><span id="line-5606"></span> |
| <span class="source-line-no">5607</span><span id="line-5607"> status.setStatus("Opening recovered edits");</span> |
| <span class="source-line-no">5608</span><span id="line-5608"> try (WALStreamReader reader = WALFactory.createStreamReader(fs, edits, conf)) {</span> |
| <span class="source-line-no">5609</span><span id="line-5609"> long currentEditSeqId = -1;</span> |
| <span class="source-line-no">5610</span><span id="line-5610"> long currentReplaySeqId = -1;</span> |
| <span class="source-line-no">5611</span><span id="line-5611"> long firstSeqIdInLog = -1;</span> |
| <span class="source-line-no">5612</span><span id="line-5612"> long skippedEdits = 0;</span> |
| <span class="source-line-no">5613</span><span id="line-5613"> long editsCount = 0;</span> |
| <span class="source-line-no">5614</span><span id="line-5614"> long intervalEdits = 0;</span> |
| <span class="source-line-no">5615</span><span id="line-5615"> WAL.Entry entry;</span> |
| <span class="source-line-no">5616</span><span id="line-5616"> HStore store = null;</span> |
| <span class="source-line-no">5617</span><span id="line-5617"> boolean reported_once = false;</span> |
| <span class="source-line-no">5618</span><span id="line-5618"> ServerNonceManager ng = this.rsServices == null ? null : this.rsServices.getNonceManager();</span> |
| <span class="source-line-no">5619</span><span id="line-5619"></span> |
| <span class="source-line-no">5620</span><span id="line-5620"> try {</span> |
| <span class="source-line-no">5621</span><span id="line-5621"> // How many edits seen before we check elapsed time</span> |
| <span class="source-line-no">5622</span><span id="line-5622"> int interval = this.conf.getInt("hbase.hstore.report.interval.edits", 2000);</span> |
| <span class="source-line-no">5623</span><span id="line-5623"> // How often to send a progress report (default 1/2 master timeout)</span> |
| <span class="source-line-no">5624</span><span id="line-5624"> int period = this.conf.getInt("hbase.hstore.report.period", 300000);</span> |
| <span class="source-line-no">5625</span><span id="line-5625"> long lastReport = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">5626</span><span id="line-5626"></span> |
| <span class="source-line-no">5627</span><span id="line-5627"> if (coprocessorHost != null) {</span> |
| <span class="source-line-no">5628</span><span id="line-5628"> coprocessorHost.preReplayWALs(this.getRegionInfo(), edits);</span> |
| <span class="source-line-no">5629</span><span id="line-5629"> }</span> |
| <span class="source-line-no">5630</span><span id="line-5630"></span> |
| <span class="source-line-no">5631</span><span id="line-5631"> while ((entry = reader.next()) != null) {</span> |
| <span class="source-line-no">5632</span><span id="line-5632"> WALKey key = entry.getKey();</span> |
| <span class="source-line-no">5633</span><span id="line-5633"> WALEdit val = entry.getEdit();</span> |
| <span class="source-line-no">5634</span><span id="line-5634"></span> |
| <span class="source-line-no">5635</span><span id="line-5635"> if (ng != null) { // some test, or nonces disabled</span> |
| <span class="source-line-no">5636</span><span id="line-5636"> ng.reportOperationFromWal(key.getNonceGroup(), key.getNonce(), key.getWriteTime());</span> |
| <span class="source-line-no">5637</span><span id="line-5637"> }</span> |
| <span class="source-line-no">5638</span><span id="line-5638"></span> |
| <span class="source-line-no">5639</span><span id="line-5639"> if (reporter != null) {</span> |
| <span class="source-line-no">5640</span><span id="line-5640"> intervalEdits += val.size();</span> |
| <span class="source-line-no">5641</span><span id="line-5641"> if (intervalEdits >= interval) {</span> |
| <span class="source-line-no">5642</span><span id="line-5642"> // Number of edits interval reached</span> |
| <span class="source-line-no">5643</span><span id="line-5643"> intervalEdits = 0;</span> |
| <span class="source-line-no">5644</span><span id="line-5644"> long cur = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">5645</span><span id="line-5645"> if (lastReport + period <= cur) {</span> |
| <span class="source-line-no">5646</span><span id="line-5646"> status.setStatus(</span> |
| <span class="source-line-no">5647</span><span id="line-5647"> "Replaying edits..." + " skipped=" + skippedEdits + " edits=" + editsCount);</span> |
| <span class="source-line-no">5648</span><span id="line-5648"> // Timeout reached</span> |
| <span class="source-line-no">5649</span><span id="line-5649"> if (!reporter.progress()) {</span> |
| <span class="source-line-no">5650</span><span id="line-5650"> msg = "Progressable reporter failed, stopping replay for region " + this;</span> |
| <span class="source-line-no">5651</span><span id="line-5651"> LOG.warn(msg);</span> |
| <span class="source-line-no">5652</span><span id="line-5652"> status.abort(msg);</span> |
| <span class="source-line-no">5653</span><span id="line-5653"> throw new IOException(msg);</span> |
| <span class="source-line-no">5654</span><span id="line-5654"> }</span> |
| <span class="source-line-no">5655</span><span id="line-5655"> reported_once = true;</span> |
| <span class="source-line-no">5656</span><span id="line-5656"> lastReport = cur;</span> |
| <span class="source-line-no">5657</span><span id="line-5657"> }</span> |
| <span class="source-line-no">5658</span><span id="line-5658"> }</span> |
| <span class="source-line-no">5659</span><span id="line-5659"> }</span> |
| <span class="source-line-no">5660</span><span id="line-5660"></span> |
| <span class="source-line-no">5661</span><span id="line-5661"> if (firstSeqIdInLog == -1) {</span> |
| <span class="source-line-no">5662</span><span id="line-5662"> firstSeqIdInLog = key.getSequenceId();</span> |
| <span class="source-line-no">5663</span><span id="line-5663"> }</span> |
| <span class="source-line-no">5664</span><span id="line-5664"> if (currentEditSeqId > key.getSequenceId()) {</span> |
| <span class="source-line-no">5665</span><span id="line-5665"> // when this condition is true, it means we have a serious defect because we need to</span> |
| <span class="source-line-no">5666</span><span id="line-5666"> // maintain increasing SeqId for WAL edits per region</span> |
| <span class="source-line-no">5667</span><span id="line-5667"> LOG.error(getRegionInfo().getEncodedName() + " : " + "Found decreasing SeqId. PreId="</span> |
| <span class="source-line-no">5668</span><span id="line-5668"> + currentEditSeqId + " key=" + key + "; edit=" + val);</span> |
| <span class="source-line-no">5669</span><span id="line-5669"> } else {</span> |
| <span class="source-line-no">5670</span><span id="line-5670"> currentEditSeqId = key.getSequenceId();</span> |
| <span class="source-line-no">5671</span><span id="line-5671"> }</span> |
| <span class="source-line-no">5672</span><span id="line-5672"> currentReplaySeqId =</span> |
| <span class="source-line-no">5673</span><span id="line-5673"> (key.getOrigLogSeqNum() > 0) ? key.getOrigLogSeqNum() : currentEditSeqId;</span> |
| <span class="source-line-no">5674</span><span id="line-5674"></span> |
| <span class="source-line-no">5675</span><span id="line-5675"> boolean checkRowWithinBoundary = false;</span> |
| <span class="source-line-no">5676</span><span id="line-5676"> // Check this edit is for this region.</span> |
| <span class="source-line-no">5677</span><span id="line-5677"> if (</span> |
| <span class="source-line-no">5678</span><span id="line-5678"> !Bytes.equals(key.getEncodedRegionName(), this.getRegionInfo().getEncodedNameAsBytes())</span> |
| <span class="source-line-no">5679</span><span id="line-5679"> ) {</span> |
| <span class="source-line-no">5680</span><span id="line-5680"> checkRowWithinBoundary = true;</span> |
| <span class="source-line-no">5681</span><span id="line-5681"> }</span> |
| <span class="source-line-no">5682</span><span id="line-5682"></span> |
| <span class="source-line-no">5683</span><span id="line-5683"> boolean flush = false;</span> |
| <span class="source-line-no">5684</span><span id="line-5684"> MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();</span> |
| <span class="source-line-no">5685</span><span id="line-5685"> for (Cell c : val.getCells()) {</span> |
| <span class="source-line-no">5686</span><span id="line-5686"> assert c instanceof ExtendedCell;</span> |
| <span class="source-line-no">5687</span><span id="line-5687"> ExtendedCell cell = (ExtendedCell) c;</span> |
| <span class="source-line-no">5688</span><span id="line-5688"> // Check this edit is for me. Also, guard against writing the special</span> |
| <span class="source-line-no">5689</span><span id="line-5689"> // METACOLUMN info such as HBASE::CACHEFLUSH entries</span> |
| <span class="source-line-no">5690</span><span id="line-5690"> if (WALEdit.isMetaEditFamily(cell)) {</span> |
| <span class="source-line-no">5691</span><span id="line-5691"> // if region names don't match, skipp replaying compaction marker</span> |
| <span class="source-line-no">5692</span><span id="line-5692"> if (!checkRowWithinBoundary) {</span> |
| <span class="source-line-no">5693</span><span id="line-5693"> // this is a special edit, we should handle it</span> |
| <span class="source-line-no">5694</span><span id="line-5694"> CompactionDescriptor compaction = WALEdit.getCompaction(cell);</span> |
| <span class="source-line-no">5695</span><span id="line-5695"> if (compaction != null) {</span> |
| <span class="source-line-no">5696</span><span id="line-5696"> // replay the compaction</span> |
| <span class="source-line-no">5697</span><span id="line-5697"> replayWALCompactionMarker(compaction, false, true, Long.MAX_VALUE);</span> |
| <span class="source-line-no">5698</span><span id="line-5698"> }</span> |
| <span class="source-line-no">5699</span><span id="line-5699"> }</span> |
| <span class="source-line-no">5700</span><span id="line-5700"> skippedEdits++;</span> |
| <span class="source-line-no">5701</span><span id="line-5701"> continue;</span> |
| <span class="source-line-no">5702</span><span id="line-5702"> }</span> |
| <span class="source-line-no">5703</span><span id="line-5703"> // Figure which store the edit is meant for.</span> |
| <span class="source-line-no">5704</span><span id="line-5704"> if (</span> |
| <span class="source-line-no">5705</span><span id="line-5705"> store == null</span> |
| <span class="source-line-no">5706</span><span id="line-5706"> || !CellUtil.matchingFamily(cell, store.getColumnFamilyDescriptor().getName())</span> |
| <span class="source-line-no">5707</span><span id="line-5707"> ) {</span> |
| <span class="source-line-no">5708</span><span id="line-5708"> store = getStore(cell);</span> |
| <span class="source-line-no">5709</span><span id="line-5709"> }</span> |
| <span class="source-line-no">5710</span><span id="line-5710"> if (store == null) {</span> |
| <span class="source-line-no">5711</span><span id="line-5711"> // This should never happen. Perhaps schema was changed between</span> |
| <span class="source-line-no">5712</span><span id="line-5712"> // crash and redeploy?</span> |
| <span class="source-line-no">5713</span><span id="line-5713"> LOG.warn("No family for cell {} in region {}", cell, this);</span> |
| <span class="source-line-no">5714</span><span id="line-5714"> skippedEdits++;</span> |
| <span class="source-line-no">5715</span><span id="line-5715"> continue;</span> |
| <span class="source-line-no">5716</span><span id="line-5716"> }</span> |
| <span class="source-line-no">5717</span><span id="line-5717"> if (</span> |
| <span class="source-line-no">5718</span><span id="line-5718"> checkRowWithinBoundary && !rowIsInRange(this.getRegionInfo(), cell.getRowArray(),</span> |
| <span class="source-line-no">5719</span><span id="line-5719"> cell.getRowOffset(), cell.getRowLength())</span> |
| <span class="source-line-no">5720</span><span id="line-5720"> ) {</span> |
| <span class="source-line-no">5721</span><span id="line-5721"> LOG.warn("Row of {} is not within region boundary for region {}", cell, this);</span> |
| <span class="source-line-no">5722</span><span id="line-5722"> skippedEdits++;</span> |
| <span class="source-line-no">5723</span><span id="line-5723"> continue;</span> |
| <span class="source-line-no">5724</span><span id="line-5724"> }</span> |
| <span class="source-line-no">5725</span><span id="line-5725"> // Now, figure if we should skip this edit.</span> |
| <span class="source-line-no">5726</span><span id="line-5726"> if (</span> |
| <span class="source-line-no">5727</span><span id="line-5727"> key.getSequenceId()</span> |
| <span class="source-line-no">5728</span><span id="line-5728"> <= maxSeqIdInStores.get(store.getColumnFamilyDescriptor().getName())</span> |
| <span class="source-line-no">5729</span><span id="line-5729"> ) {</span> |
| <span class="source-line-no">5730</span><span id="line-5730"> skippedEdits++;</span> |
| <span class="source-line-no">5731</span><span id="line-5731"> continue;</span> |
| <span class="source-line-no">5732</span><span id="line-5732"> }</span> |
| <span class="source-line-no">5733</span><span id="line-5733"> PrivateCellUtil.setSequenceId(cell, currentReplaySeqId);</span> |
| <span class="source-line-no">5734</span><span id="line-5734"></span> |
| <span class="source-line-no">5735</span><span id="line-5735"> restoreEdit(store, cell, memStoreSizing);</span> |
| <span class="source-line-no">5736</span><span id="line-5736"> editsCount++;</span> |
| <span class="source-line-no">5737</span><span id="line-5737"> }</span> |
| <span class="source-line-no">5738</span><span id="line-5738"> MemStoreSize mss = memStoreSizing.getMemStoreSize();</span> |
| <span class="source-line-no">5739</span><span id="line-5739"> incMemStoreSize(mss);</span> |
| <span class="source-line-no">5740</span><span id="line-5740"> flush = isFlushSize(this.memStoreSizing.getMemStoreSize());</span> |
| <span class="source-line-no">5741</span><span id="line-5741"> if (flush) {</span> |
| <span class="source-line-no">5742</span><span id="line-5742"> internalFlushcache(null, currentEditSeqId, stores.values(), status, false,</span> |
| <span class="source-line-no">5743</span><span id="line-5743"> FlushLifeCycleTracker.DUMMY);</span> |
| <span class="source-line-no">5744</span><span id="line-5744"> }</span> |
| <span class="source-line-no">5745</span><span id="line-5745"> }</span> |
| <span class="source-line-no">5746</span><span id="line-5746"></span> |
| <span class="source-line-no">5747</span><span id="line-5747"> if (coprocessorHost != null) {</span> |
| <span class="source-line-no">5748</span><span id="line-5748"> coprocessorHost.postReplayWALs(this.getRegionInfo(), edits);</span> |
| <span class="source-line-no">5749</span><span id="line-5749"> }</span> |
| <span class="source-line-no">5750</span><span id="line-5750"> } catch (EOFException eof) {</span> |
| <span class="source-line-no">5751</span><span id="line-5751"> if (!conf.getBoolean(RECOVERED_EDITS_IGNORE_EOF, false)) {</span> |
| <span class="source-line-no">5752</span><span id="line-5752"> Path p = WALSplitUtil.moveAsideBadEditsFile(walFS, edits);</span> |
| <span class="source-line-no">5753</span><span id="line-5753"> msg = "EnLongAddered EOF. Most likely due to Master failure during "</span> |
| <span class="source-line-no">5754</span><span id="line-5754"> + "wal splitting, so we have this data in another edit. Continuing, but renaming "</span> |
| <span class="source-line-no">5755</span><span id="line-5755"> + edits + " as " + p + " for region " + this;</span> |
| <span class="source-line-no">5756</span><span id="line-5756"> LOG.warn(msg, eof);</span> |
| <span class="source-line-no">5757</span><span id="line-5757"> status.abort(msg);</span> |
| <span class="source-line-no">5758</span><span id="line-5758"> } else {</span> |
| <span class="source-line-no">5759</span><span id="line-5759"> LOG.warn("EOF while replaying recover edits and config '{}' is true so "</span> |
| <span class="source-line-no">5760</span><span id="line-5760"> + "we will ignore it and continue", RECOVERED_EDITS_IGNORE_EOF, eof);</span> |
| <span class="source-line-no">5761</span><span id="line-5761"> }</span> |
| <span class="source-line-no">5762</span><span id="line-5762"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">5763</span><span id="line-5763"> // If the IOE resulted from bad file format,</span> |
| <span class="source-line-no">5764</span><span id="line-5764"> // then this problem is idempotent and retrying won't help</span> |
| <span class="source-line-no">5765</span><span id="line-5765"> if (ioe.getCause() instanceof ParseException) {</span> |
| <span class="source-line-no">5766</span><span id="line-5766"> Path p = WALSplitUtil.moveAsideBadEditsFile(walFS, edits);</span> |
| <span class="source-line-no">5767</span><span id="line-5767"> msg =</span> |
| <span class="source-line-no">5768</span><span id="line-5768"> "File corruption enLongAddered! " + "Continuing, but renaming " + edits + " as " + p;</span> |
| <span class="source-line-no">5769</span><span id="line-5769"> LOG.warn(msg, ioe);</span> |
| <span class="source-line-no">5770</span><span id="line-5770"> status.setStatus(msg);</span> |
| <span class="source-line-no">5771</span><span id="line-5771"> } else {</span> |
| <span class="source-line-no">5772</span><span id="line-5772"> status.abort(StringUtils.stringifyException(ioe));</span> |
| <span class="source-line-no">5773</span><span id="line-5773"> // other IO errors may be transient (bad network connection,</span> |
| <span class="source-line-no">5774</span><span id="line-5774"> // checksum exception on one datanode, etc). throw & retry</span> |
| <span class="source-line-no">5775</span><span id="line-5775"> throw ioe;</span> |
| <span class="source-line-no">5776</span><span id="line-5776"> }</span> |
| <span class="source-line-no">5777</span><span id="line-5777"> }</span> |
| <span class="source-line-no">5778</span><span id="line-5778"> if (reporter != null && !reported_once) {</span> |
| <span class="source-line-no">5779</span><span id="line-5779"> reporter.progress();</span> |
| <span class="source-line-no">5780</span><span id="line-5780"> }</span> |
| <span class="source-line-no">5781</span><span id="line-5781"> msg = "Applied " + editsCount + ", skipped " + skippedEdits + ", firstSequenceIdInLog="</span> |
| <span class="source-line-no">5782</span><span id="line-5782"> + firstSeqIdInLog + ", maxSequenceIdInLog=" + currentEditSeqId + ", path=" + edits;</span> |
| <span class="source-line-no">5783</span><span id="line-5783"> status.markComplete(msg);</span> |
| <span class="source-line-no">5784</span><span id="line-5784"> LOG.debug(msg);</span> |
| <span class="source-line-no">5785</span><span id="line-5785"> return currentEditSeqId;</span> |
| <span class="source-line-no">5786</span><span id="line-5786"> } finally {</span> |
| <span class="source-line-no">5787</span><span id="line-5787"> status.cleanup();</span> |
| <span class="source-line-no">5788</span><span id="line-5788"> }</span> |
| <span class="source-line-no">5789</span><span id="line-5789"> }</span> |
| <span class="source-line-no">5790</span><span id="line-5790"></span> |
| <span class="source-line-no">5791</span><span id="line-5791"> /**</span> |
| <span class="source-line-no">5792</span><span id="line-5792"> * Call to complete a compaction. Its for the case where we find in the WAL a compaction that was</span> |
| <span class="source-line-no">5793</span><span id="line-5793"> * not finished. We could find one recovering a WAL after a regionserver crash. See HBASE-2331.</span> |
| <span class="source-line-no">5794</span><span id="line-5794"> */</span> |
| <span class="source-line-no">5795</span><span id="line-5795"> void replayWALCompactionMarker(CompactionDescriptor compaction, boolean pickCompactionFiles,</span> |
| <span class="source-line-no">5796</span><span id="line-5796"> boolean removeFiles, long replaySeqId) throws IOException {</span> |
| <span class="source-line-no">5797</span><span id="line-5797"> try {</span> |
| <span class="source-line-no">5798</span><span id="line-5798"> checkTargetRegion(compaction.getEncodedRegionName().toByteArray(),</span> |
| <span class="source-line-no">5799</span><span id="line-5799"> "Compaction marker from WAL ", compaction);</span> |
| <span class="source-line-no">5800</span><span id="line-5800"> } catch (WrongRegionException wre) {</span> |
| <span class="source-line-no">5801</span><span id="line-5801"> if (RegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {</span> |
| <span class="source-line-no">5802</span><span id="line-5802"> // skip the compaction marker since it is not for this region</span> |
| <span class="source-line-no">5803</span><span id="line-5803"> return;</span> |
| <span class="source-line-no">5804</span><span id="line-5804"> }</span> |
| <span class="source-line-no">5805</span><span id="line-5805"> throw wre;</span> |
| <span class="source-line-no">5806</span><span id="line-5806"> }</span> |
| <span class="source-line-no">5807</span><span id="line-5807"></span> |
| <span class="source-line-no">5808</span><span id="line-5808"> synchronized (writestate) {</span> |
| <span class="source-line-no">5809</span><span id="line-5809"> if (replaySeqId < lastReplayedOpenRegionSeqId) {</span> |
| <span class="source-line-no">5810</span><span id="line-5810"> LOG.warn(getRegionInfo().getEncodedName() + " : " + "Skipping replaying compaction event :"</span> |
| <span class="source-line-no">5811</span><span id="line-5811"> + TextFormat.shortDebugString(compaction) + " because its sequence id " + replaySeqId</span> |
| <span class="source-line-no">5812</span><span id="line-5812"> + " is smaller than this regions " + "lastReplayedOpenRegionSeqId of "</span> |
| <span class="source-line-no">5813</span><span id="line-5813"> + lastReplayedOpenRegionSeqId);</span> |
| <span class="source-line-no">5814</span><span id="line-5814"> return;</span> |
| <span class="source-line-no">5815</span><span id="line-5815"> }</span> |
| <span class="source-line-no">5816</span><span id="line-5816"> if (replaySeqId < lastReplayedCompactionSeqId) {</span> |
| <span class="source-line-no">5817</span><span id="line-5817"> LOG.warn(getRegionInfo().getEncodedName() + " : " + "Skipping replaying compaction event :"</span> |
| <span class="source-line-no">5818</span><span id="line-5818"> + TextFormat.shortDebugString(compaction) + " because its sequence id " + replaySeqId</span> |
| <span class="source-line-no">5819</span><span id="line-5819"> + " is smaller than this regions " + "lastReplayedCompactionSeqId of "</span> |
| <span class="source-line-no">5820</span><span id="line-5820"> + lastReplayedCompactionSeqId);</span> |
| <span class="source-line-no">5821</span><span id="line-5821"> return;</span> |
| <span class="source-line-no">5822</span><span id="line-5822"> } else {</span> |
| <span class="source-line-no">5823</span><span id="line-5823"> lastReplayedCompactionSeqId = replaySeqId;</span> |
| <span class="source-line-no">5824</span><span id="line-5824"> }</span> |
| <span class="source-line-no">5825</span><span id="line-5825"></span> |
| <span class="source-line-no">5826</span><span id="line-5826"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">5827</span><span id="line-5827"> LOG.debug(getRegionInfo().getEncodedName() + " : " + "Replaying compaction marker "</span> |
| <span class="source-line-no">5828</span><span id="line-5828"> + TextFormat.shortDebugString(compaction) + " with seqId=" + replaySeqId</span> |
| <span class="source-line-no">5829</span><span id="line-5829"> + " and lastReplayedOpenRegionSeqId=" + lastReplayedOpenRegionSeqId);</span> |
| <span class="source-line-no">5830</span><span id="line-5830"> }</span> |
| <span class="source-line-no">5831</span><span id="line-5831"></span> |
| <span class="source-line-no">5832</span><span id="line-5832"> startRegionOperation(Operation.REPLAY_EVENT);</span> |
| <span class="source-line-no">5833</span><span id="line-5833"> try {</span> |
| <span class="source-line-no">5834</span><span id="line-5834"> HStore store = this.getStore(compaction.getFamilyName().toByteArray());</span> |
| <span class="source-line-no">5835</span><span id="line-5835"> if (store == null) {</span> |
| <span class="source-line-no">5836</span><span id="line-5836"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">5837</span><span id="line-5837"> + "Found Compaction WAL edit for deleted family:"</span> |
| <span class="source-line-no">5838</span><span id="line-5838"> + Bytes.toString(compaction.getFamilyName().toByteArray()));</span> |
| <span class="source-line-no">5839</span><span id="line-5839"> return;</span> |
| <span class="source-line-no">5840</span><span id="line-5840"> }</span> |
| <span class="source-line-no">5841</span><span id="line-5841"> store.replayCompactionMarker(compaction, pickCompactionFiles, removeFiles);</span> |
| <span class="source-line-no">5842</span><span id="line-5842"> logRegionFiles();</span> |
| <span class="source-line-no">5843</span><span id="line-5843"> } catch (FileNotFoundException ex) {</span> |
| <span class="source-line-no">5844</span><span id="line-5844"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">5845</span><span id="line-5845"> + "At least one of the store files in compaction: "</span> |
| <span class="source-line-no">5846</span><span id="line-5846"> + TextFormat.shortDebugString(compaction)</span> |
| <span class="source-line-no">5847</span><span id="line-5847"> + " doesn't exist any more. Skip loading the file(s)", ex);</span> |
| <span class="source-line-no">5848</span><span id="line-5848"> } finally {</span> |
| <span class="source-line-no">5849</span><span id="line-5849"> closeRegionOperation(Operation.REPLAY_EVENT);</span> |
| <span class="source-line-no">5850</span><span id="line-5850"> }</span> |
| <span class="source-line-no">5851</span><span id="line-5851"> }</span> |
| <span class="source-line-no">5852</span><span id="line-5852"> }</span> |
| <span class="source-line-no">5853</span><span id="line-5853"></span> |
| <span class="source-line-no">5854</span><span id="line-5854"> /**</span> |
| <span class="source-line-no">5855</span><span id="line-5855"> * @deprecated Since 3.0.0, will be removed in 4.0.0. Only for keep compatibility for old region</span> |
| <span class="source-line-no">5856</span><span id="line-5856"> * replica implementation.</span> |
| <span class="source-line-no">5857</span><span id="line-5857"> */</span> |
| <span class="source-line-no">5858</span><span id="line-5858"> @Deprecated</span> |
| <span class="source-line-no">5859</span><span id="line-5859"> void replayWALFlushMarker(FlushDescriptor flush, long replaySeqId) throws IOException {</span> |
| <span class="source-line-no">5860</span><span id="line-5860"> checkTargetRegion(flush.getEncodedRegionName().toByteArray(), "Flush marker from WAL ", flush);</span> |
| <span class="source-line-no">5861</span><span id="line-5861"></span> |
| <span class="source-line-no">5862</span><span id="line-5862"> if (ServerRegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {</span> |
| <span class="source-line-no">5863</span><span id="line-5863"> return; // if primary nothing to do</span> |
| <span class="source-line-no">5864</span><span id="line-5864"> }</span> |
| <span class="source-line-no">5865</span><span id="line-5865"></span> |
| <span class="source-line-no">5866</span><span id="line-5866"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">5867</span><span id="line-5867"> LOG.debug(getRegionInfo().getEncodedName() + " : " + "Replaying flush marker "</span> |
| <span class="source-line-no">5868</span><span id="line-5868"> + TextFormat.shortDebugString(flush));</span> |
| <span class="source-line-no">5869</span><span id="line-5869"> }</span> |
| <span class="source-line-no">5870</span><span id="line-5870"></span> |
| <span class="source-line-no">5871</span><span id="line-5871"> startRegionOperation(Operation.REPLAY_EVENT); // use region close lock to guard against close</span> |
| <span class="source-line-no">5872</span><span id="line-5872"> try {</span> |
| <span class="source-line-no">5873</span><span id="line-5873"> FlushAction action = flush.getAction();</span> |
| <span class="source-line-no">5874</span><span id="line-5874"> switch (action) {</span> |
| <span class="source-line-no">5875</span><span id="line-5875"> case START_FLUSH:</span> |
| <span class="source-line-no">5876</span><span id="line-5876"> replayWALFlushStartMarker(flush);</span> |
| <span class="source-line-no">5877</span><span id="line-5877"> break;</span> |
| <span class="source-line-no">5878</span><span id="line-5878"> case COMMIT_FLUSH:</span> |
| <span class="source-line-no">5879</span><span id="line-5879"> replayWALFlushCommitMarker(flush);</span> |
| <span class="source-line-no">5880</span><span id="line-5880"> break;</span> |
| <span class="source-line-no">5881</span><span id="line-5881"> case ABORT_FLUSH:</span> |
| <span class="source-line-no">5882</span><span id="line-5882"> replayWALFlushAbortMarker(flush);</span> |
| <span class="source-line-no">5883</span><span id="line-5883"> break;</span> |
| <span class="source-line-no">5884</span><span id="line-5884"> case CANNOT_FLUSH:</span> |
| <span class="source-line-no">5885</span><span id="line-5885"> replayWALFlushCannotFlushMarker(flush, replaySeqId);</span> |
| <span class="source-line-no">5886</span><span id="line-5886"> break;</span> |
| <span class="source-line-no">5887</span><span id="line-5887"> default:</span> |
| <span class="source-line-no">5888</span><span id="line-5888"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">5889</span><span id="line-5889"> + "Received a flush event with unknown action, ignoring. "</span> |
| <span class="source-line-no">5890</span><span id="line-5890"> + TextFormat.shortDebugString(flush));</span> |
| <span class="source-line-no">5891</span><span id="line-5891"> break;</span> |
| <span class="source-line-no">5892</span><span id="line-5892"> }</span> |
| <span class="source-line-no">5893</span><span id="line-5893"></span> |
| <span class="source-line-no">5894</span><span id="line-5894"> logRegionFiles();</span> |
| <span class="source-line-no">5895</span><span id="line-5895"> } finally {</span> |
| <span class="source-line-no">5896</span><span id="line-5896"> closeRegionOperation(Operation.REPLAY_EVENT);</span> |
| <span class="source-line-no">5897</span><span id="line-5897"> }</span> |
| <span class="source-line-no">5898</span><span id="line-5898"> }</span> |
| <span class="source-line-no">5899</span><span id="line-5899"></span> |
| <span class="source-line-no">5900</span><span id="line-5900"> private Collection<HStore> getStoresToFlush(FlushDescriptor flushDesc) {</span> |
| <span class="source-line-no">5901</span><span id="line-5901"> List<HStore> storesToFlush = new ArrayList<>();</span> |
| <span class="source-line-no">5902</span><span id="line-5902"> for (StoreFlushDescriptor storeFlush : flushDesc.getStoreFlushesList()) {</span> |
| <span class="source-line-no">5903</span><span id="line-5903"> byte[] family = storeFlush.getFamilyName().toByteArray();</span> |
| <span class="source-line-no">5904</span><span id="line-5904"> HStore store = getStore(family);</span> |
| <span class="source-line-no">5905</span><span id="line-5905"> if (store == null) {</span> |
| <span class="source-line-no">5906</span><span id="line-5906"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">5907</span><span id="line-5907"> + "Received a flush start marker from primary, but the family is not found. Ignoring"</span> |
| <span class="source-line-no">5908</span><span id="line-5908"> + " StoreFlushDescriptor:" + TextFormat.shortDebugString(storeFlush));</span> |
| <span class="source-line-no">5909</span><span id="line-5909"> continue;</span> |
| <span class="source-line-no">5910</span><span id="line-5910"> }</span> |
| <span class="source-line-no">5911</span><span id="line-5911"> storesToFlush.add(store);</span> |
| <span class="source-line-no">5912</span><span id="line-5912"> }</span> |
| <span class="source-line-no">5913</span><span id="line-5913"> return storesToFlush;</span> |
| <span class="source-line-no">5914</span><span id="line-5914"> }</span> |
| <span class="source-line-no">5915</span><span id="line-5915"></span> |
| <span class="source-line-no">5916</span><span id="line-5916"> /**</span> |
| <span class="source-line-no">5917</span><span id="line-5917"> * Replay the flush marker from primary region by creating a corresponding snapshot of the store</span> |
| <span class="source-line-no">5918</span><span id="line-5918"> * memstores, only if the memstores do not have a higher seqId from an earlier wal edit (because</span> |
| <span class="source-line-no">5919</span><span id="line-5919"> * the events may be coming out of order).</span> |
| <span class="source-line-no">5920</span><span id="line-5920"> * @deprecated Since 3.0.0, will be removed in 4.0.0. Only for keep compatibility for old region</span> |
| <span class="source-line-no">5921</span><span id="line-5921"> * replica implementation.</span> |
| <span class="source-line-no">5922</span><span id="line-5922"> */</span> |
| <span class="source-line-no">5923</span><span id="line-5923"> @Deprecated</span> |
| <span class="source-line-no">5924</span><span id="line-5924"> PrepareFlushResult replayWALFlushStartMarker(FlushDescriptor flush) throws IOException {</span> |
| <span class="source-line-no">5925</span><span id="line-5925"> long flushSeqId = flush.getFlushSequenceNumber();</span> |
| <span class="source-line-no">5926</span><span id="line-5926"></span> |
| <span class="source-line-no">5927</span><span id="line-5927"> Collection<HStore> storesToFlush = getStoresToFlush(flush);</span> |
| <span class="source-line-no">5928</span><span id="line-5928"></span> |
| <span class="source-line-no">5929</span><span id="line-5929"> MonitoredTask status = TaskMonitor.get().createStatus("Preparing flush " + this);</span> |
| <span class="source-line-no">5930</span><span id="line-5930"></span> |
| <span class="source-line-no">5931</span><span id="line-5931"> // we will use writestate as a coarse-grain lock for all the replay events</span> |
| <span class="source-line-no">5932</span><span id="line-5932"> // (flush, compaction, region open etc)</span> |
| <span class="source-line-no">5933</span><span id="line-5933"> synchronized (writestate) {</span> |
| <span class="source-line-no">5934</span><span id="line-5934"> try {</span> |
| <span class="source-line-no">5935</span><span id="line-5935"> if (flush.getFlushSequenceNumber() < lastReplayedOpenRegionSeqId) {</span> |
| <span class="source-line-no">5936</span><span id="line-5936"> LOG.warn(getRegionInfo().getEncodedName() + " : " + "Skipping replaying flush event :"</span> |
| <span class="source-line-no">5937</span><span id="line-5937"> + TextFormat.shortDebugString(flush)</span> |
| <span class="source-line-no">5938</span><span id="line-5938"> + " because its sequence id is smaller than this regions lastReplayedOpenRegionSeqId "</span> |
| <span class="source-line-no">5939</span><span id="line-5939"> + " of " + lastReplayedOpenRegionSeqId);</span> |
| <span class="source-line-no">5940</span><span id="line-5940"> return null;</span> |
| <span class="source-line-no">5941</span><span id="line-5941"> }</span> |
| <span class="source-line-no">5942</span><span id="line-5942"> if (numMutationsWithoutWAL.sum() > 0) {</span> |
| <span class="source-line-no">5943</span><span id="line-5943"> numMutationsWithoutWAL.reset();</span> |
| <span class="source-line-no">5944</span><span id="line-5944"> dataInMemoryWithoutWAL.reset();</span> |
| <span class="source-line-no">5945</span><span id="line-5945"> }</span> |
| <span class="source-line-no">5946</span><span id="line-5946"></span> |
| <span class="source-line-no">5947</span><span id="line-5947"> if (!writestate.flushing) {</span> |
| <span class="source-line-no">5948</span><span id="line-5948"> // we do not have an active snapshot and corresponding this.prepareResult. This means</span> |
| <span class="source-line-no">5949</span><span id="line-5949"> // we can just snapshot our memstores and continue as normal.</span> |
| <span class="source-line-no">5950</span><span id="line-5950"></span> |
| <span class="source-line-no">5951</span><span id="line-5951"> // invoke prepareFlushCache. Send null as wal since we do not want the flush events in wal</span> |
| <span class="source-line-no">5952</span><span id="line-5952"> PrepareFlushResult prepareResult = internalPrepareFlushCache(null, flushSeqId,</span> |
| <span class="source-line-no">5953</span><span id="line-5953"> storesToFlush, status, false, FlushLifeCycleTracker.DUMMY);</span> |
| <span class="source-line-no">5954</span><span id="line-5954"> if (prepareResult.result == null) {</span> |
| <span class="source-line-no">5955</span><span id="line-5955"> // save the PrepareFlushResult so that we can use it later from commit flush</span> |
| <span class="source-line-no">5956</span><span id="line-5956"> this.writestate.flushing = true;</span> |
| <span class="source-line-no">5957</span><span id="line-5957"> this.prepareFlushResult = prepareResult;</span> |
| <span class="source-line-no">5958</span><span id="line-5958"> status.markComplete("Flush prepare successful");</span> |
| <span class="source-line-no">5959</span><span id="line-5959"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">5960</span><span id="line-5960"> LOG.debug(getRegionInfo().getEncodedName() + " : " + " Prepared flush with seqId:"</span> |
| <span class="source-line-no">5961</span><span id="line-5961"> + flush.getFlushSequenceNumber());</span> |
| <span class="source-line-no">5962</span><span id="line-5962"> }</span> |
| <span class="source-line-no">5963</span><span id="line-5963"> } else {</span> |
| <span class="source-line-no">5964</span><span id="line-5964"> // special case empty memstore. We will still save the flush result in this case, since</span> |
| <span class="source-line-no">5965</span><span id="line-5965"> // our memstore ie empty, but the primary is still flushing</span> |
| <span class="source-line-no">5966</span><span id="line-5966"> if (</span> |
| <span class="source-line-no">5967</span><span id="line-5967"> prepareResult.getResult().getResult()</span> |
| <span class="source-line-no">5968</span><span id="line-5968"> == FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY</span> |
| <span class="source-line-no">5969</span><span id="line-5969"> ) {</span> |
| <span class="source-line-no">5970</span><span id="line-5970"> this.writestate.flushing = true;</span> |
| <span class="source-line-no">5971</span><span id="line-5971"> this.prepareFlushResult = prepareResult;</span> |
| <span class="source-line-no">5972</span><span id="line-5972"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">5973</span><span id="line-5973"> LOG.debug(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">5974</span><span id="line-5974"> + " Prepared empty flush with seqId:" + flush.getFlushSequenceNumber());</span> |
| <span class="source-line-no">5975</span><span id="line-5975"> }</span> |
| <span class="source-line-no">5976</span><span id="line-5976"> }</span> |
| <span class="source-line-no">5977</span><span id="line-5977"> status.abort("Flush prepare failed with " + prepareResult.result);</span> |
| <span class="source-line-no">5978</span><span id="line-5978"> // nothing much to do. prepare flush failed because of some reason.</span> |
| <span class="source-line-no">5979</span><span id="line-5979"> }</span> |
| <span class="source-line-no">5980</span><span id="line-5980"> return prepareResult;</span> |
| <span class="source-line-no">5981</span><span id="line-5981"> } else {</span> |
| <span class="source-line-no">5982</span><span id="line-5982"> // we already have an active snapshot.</span> |
| <span class="source-line-no">5983</span><span id="line-5983"> if (flush.getFlushSequenceNumber() == this.prepareFlushResult.flushOpSeqId) {</span> |
| <span class="source-line-no">5984</span><span id="line-5984"> // They define the same flush. Log and continue.</span> |
| <span class="source-line-no">5985</span><span id="line-5985"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">5986</span><span id="line-5986"> + "Received a flush prepare marker with the same seqId: "</span> |
| <span class="source-line-no">5987</span><span id="line-5987"> + +flush.getFlushSequenceNumber() + " before clearing the previous one with seqId: "</span> |
| <span class="source-line-no">5988</span><span id="line-5988"> + prepareFlushResult.flushOpSeqId + ". Ignoring");</span> |
| <span class="source-line-no">5989</span><span id="line-5989"> // ignore</span> |
| <span class="source-line-no">5990</span><span id="line-5990"> } else if (flush.getFlushSequenceNumber() < this.prepareFlushResult.flushOpSeqId) {</span> |
| <span class="source-line-no">5991</span><span id="line-5991"> // We received a flush with a smaller seqNum than what we have prepared. We can only</span> |
| <span class="source-line-no">5992</span><span id="line-5992"> // ignore this prepare flush request.</span> |
| <span class="source-line-no">5993</span><span id="line-5993"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">5994</span><span id="line-5994"> + "Received a flush prepare marker with a smaller seqId: "</span> |
| <span class="source-line-no">5995</span><span id="line-5995"> + +flush.getFlushSequenceNumber() + " before clearing the previous one with seqId: "</span> |
| <span class="source-line-no">5996</span><span id="line-5996"> + prepareFlushResult.flushOpSeqId + ". Ignoring");</span> |
| <span class="source-line-no">5997</span><span id="line-5997"> // ignore</span> |
| <span class="source-line-no">5998</span><span id="line-5998"> } else {</span> |
| <span class="source-line-no">5999</span><span id="line-5999"> // We received a flush with a larger seqNum than what we have prepared</span> |
| <span class="source-line-no">6000</span><span id="line-6000"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6001</span><span id="line-6001"> + "Received a flush prepare marker with a larger seqId: "</span> |
| <span class="source-line-no">6002</span><span id="line-6002"> + +flush.getFlushSequenceNumber() + " before clearing the previous one with seqId: "</span> |
| <span class="source-line-no">6003</span><span id="line-6003"> + prepareFlushResult.flushOpSeqId + ". Ignoring");</span> |
| <span class="source-line-no">6004</span><span id="line-6004"> // We do not have multiple active snapshots in the memstore or a way to merge current</span> |
| <span class="source-line-no">6005</span><span id="line-6005"> // memstore snapshot with the contents and resnapshot for now. We cannot take</span> |
| <span class="source-line-no">6006</span><span id="line-6006"> // another snapshot and drop the previous one because that will cause temporary</span> |
| <span class="source-line-no">6007</span><span id="line-6007"> // data loss in the secondary. So we ignore this for now, deferring the resolution</span> |
| <span class="source-line-no">6008</span><span id="line-6008"> // to happen when we see the corresponding flush commit marker. If we have a memstore</span> |
| <span class="source-line-no">6009</span><span id="line-6009"> // snapshot with x, and later received another prepare snapshot with y (where x < y),</span> |
| <span class="source-line-no">6010</span><span id="line-6010"> // when we see flush commit for y, we will drop snapshot for x, and can also drop all</span> |
| <span class="source-line-no">6011</span><span id="line-6011"> // the memstore edits if everything in memstore is < y. This is the usual case for</span> |
| <span class="source-line-no">6012</span><span id="line-6012"> // RS crash + recovery where we might see consequtive prepare flush wal markers.</span> |
| <span class="source-line-no">6013</span><span id="line-6013"> // Otherwise, this will cause more memory to be used in secondary replica until a</span> |
| <span class="source-line-no">6014</span><span id="line-6014"> // further prapare + commit flush is seen and replayed.</span> |
| <span class="source-line-no">6015</span><span id="line-6015"> }</span> |
| <span class="source-line-no">6016</span><span id="line-6016"> }</span> |
| <span class="source-line-no">6017</span><span id="line-6017"> } finally {</span> |
| <span class="source-line-no">6018</span><span id="line-6018"> status.cleanup();</span> |
| <span class="source-line-no">6019</span><span id="line-6019"> writestate.notifyAll();</span> |
| <span class="source-line-no">6020</span><span id="line-6020"> }</span> |
| <span class="source-line-no">6021</span><span id="line-6021"> }</span> |
| <span class="source-line-no">6022</span><span id="line-6022"> return null;</span> |
| <span class="source-line-no">6023</span><span id="line-6023"> }</span> |
| <span class="source-line-no">6024</span><span id="line-6024"></span> |
| <span class="source-line-no">6025</span><span id="line-6025"> /**</span> |
| <span class="source-line-no">6026</span><span id="line-6026"> * @deprecated Since 3.0.0, will be removed in 4.0.0. Only for keep compatibility for old region</span> |
| <span class="source-line-no">6027</span><span id="line-6027"> * replica implementation.</span> |
| <span class="source-line-no">6028</span><span id="line-6028"> */</span> |
| <span class="source-line-no">6029</span><span id="line-6029"> @Deprecated</span> |
| <span class="source-line-no">6030</span><span id="line-6030"> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NN_NAKED_NOTIFY",</span> |
| <span class="source-line-no">6031</span><span id="line-6031"> justification = "Intentional; post memstore flush")</span> |
| <span class="source-line-no">6032</span><span id="line-6032"> void replayWALFlushCommitMarker(FlushDescriptor flush) throws IOException {</span> |
| <span class="source-line-no">6033</span><span id="line-6033"> MonitoredTask status = TaskMonitor.get().createStatus("Committing flush " + this);</span> |
| <span class="source-line-no">6034</span><span id="line-6034"></span> |
| <span class="source-line-no">6035</span><span id="line-6035"> // check whether we have the memstore snapshot with the corresponding seqId. Replay to</span> |
| <span class="source-line-no">6036</span><span id="line-6036"> // secondary region replicas are in order, except for when the region moves or then the</span> |
| <span class="source-line-no">6037</span><span id="line-6037"> // region server crashes. In those cases, we may receive replay requests out of order from</span> |
| <span class="source-line-no">6038</span><span id="line-6038"> // the original seqIds.</span> |
| <span class="source-line-no">6039</span><span id="line-6039"> synchronized (writestate) {</span> |
| <span class="source-line-no">6040</span><span id="line-6040"> try {</span> |
| <span class="source-line-no">6041</span><span id="line-6041"> if (flush.getFlushSequenceNumber() < lastReplayedOpenRegionSeqId) {</span> |
| <span class="source-line-no">6042</span><span id="line-6042"> LOG.warn(getRegionInfo().getEncodedName() + " : " + "Skipping replaying flush event :"</span> |
| <span class="source-line-no">6043</span><span id="line-6043"> + TextFormat.shortDebugString(flush)</span> |
| <span class="source-line-no">6044</span><span id="line-6044"> + " because its sequence id is smaller than this regions lastReplayedOpenRegionSeqId "</span> |
| <span class="source-line-no">6045</span><span id="line-6045"> + " of " + lastReplayedOpenRegionSeqId);</span> |
| <span class="source-line-no">6046</span><span id="line-6046"> return;</span> |
| <span class="source-line-no">6047</span><span id="line-6047"> }</span> |
| <span class="source-line-no">6048</span><span id="line-6048"></span> |
| <span class="source-line-no">6049</span><span id="line-6049"> if (writestate.flushing) {</span> |
| <span class="source-line-no">6050</span><span id="line-6050"> PrepareFlushResult prepareFlushResult = this.prepareFlushResult;</span> |
| <span class="source-line-no">6051</span><span id="line-6051"> if (flush.getFlushSequenceNumber() == prepareFlushResult.flushOpSeqId) {</span> |
| <span class="source-line-no">6052</span><span id="line-6052"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">6053</span><span id="line-6053"> LOG.debug(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6054</span><span id="line-6054"> + "Received a flush commit marker with seqId:" + flush.getFlushSequenceNumber()</span> |
| <span class="source-line-no">6055</span><span id="line-6055"> + " and a previous prepared snapshot was found");</span> |
| <span class="source-line-no">6056</span><span id="line-6056"> }</span> |
| <span class="source-line-no">6057</span><span id="line-6057"> // This is the regular case where we received commit flush after prepare flush</span> |
| <span class="source-line-no">6058</span><span id="line-6058"> // corresponding to the same seqId.</span> |
| <span class="source-line-no">6059</span><span id="line-6059"> replayFlushInStores(flush, prepareFlushResult, true);</span> |
| <span class="source-line-no">6060</span><span id="line-6060"></span> |
| <span class="source-line-no">6061</span><span id="line-6061"> // Set down the memstore size by amount of flush.</span> |
| <span class="source-line-no">6062</span><span id="line-6062"> this.decrMemStoreSize(prepareFlushResult.totalFlushableSize.getMemStoreSize());</span> |
| <span class="source-line-no">6063</span><span id="line-6063"> this.prepareFlushResult = null;</span> |
| <span class="source-line-no">6064</span><span id="line-6064"> writestate.flushing = false;</span> |
| <span class="source-line-no">6065</span><span id="line-6065"> } else if (flush.getFlushSequenceNumber() < prepareFlushResult.flushOpSeqId) {</span> |
| <span class="source-line-no">6066</span><span id="line-6066"> // This should not happen normally. However, lets be safe and guard against these cases</span> |
| <span class="source-line-no">6067</span><span id="line-6067"> // we received a flush commit with a smaller seqId than what we have prepared</span> |
| <span class="source-line-no">6068</span><span id="line-6068"> // we will pick the flush file up from this commit (if we have not seen it), but we</span> |
| <span class="source-line-no">6069</span><span id="line-6069"> // will not drop the memstore</span> |
| <span class="source-line-no">6070</span><span id="line-6070"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6071</span><span id="line-6071"> + "Received a flush commit marker with smaller seqId: "</span> |
| <span class="source-line-no">6072</span><span id="line-6072"> + flush.getFlushSequenceNumber() + " than what we have prepared with seqId: "</span> |
| <span class="source-line-no">6073</span><span id="line-6073"> + prepareFlushResult.flushOpSeqId + ". Picking up new file, but not dropping"</span> |
| <span class="source-line-no">6074</span><span id="line-6074"> + " prepared memstore snapshot");</span> |
| <span class="source-line-no">6075</span><span id="line-6075"> replayFlushInStores(flush, prepareFlushResult, false);</span> |
| <span class="source-line-no">6076</span><span id="line-6076"></span> |
| <span class="source-line-no">6077</span><span id="line-6077"> // snapshot is not dropped, so memstore sizes should not be decremented</span> |
| <span class="source-line-no">6078</span><span id="line-6078"> // we still have the prepared snapshot, flushing should still be true</span> |
| <span class="source-line-no">6079</span><span id="line-6079"> } else {</span> |
| <span class="source-line-no">6080</span><span id="line-6080"> // This should not happen normally. However, lets be safe and guard against these cases</span> |
| <span class="source-line-no">6081</span><span id="line-6081"> // we received a flush commit with a larger seqId than what we have prepared</span> |
| <span class="source-line-no">6082</span><span id="line-6082"> // we will pick the flush file for this. We will also obtain the updates lock and</span> |
| <span class="source-line-no">6083</span><span id="line-6083"> // look for contents of the memstore to see whether we have edits after this seqId.</span> |
| <span class="source-line-no">6084</span><span id="line-6084"> // If not, we will drop all the memstore edits and the snapshot as well.</span> |
| <span class="source-line-no">6085</span><span id="line-6085"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6086</span><span id="line-6086"> + "Received a flush commit marker with larger seqId: "</span> |
| <span class="source-line-no">6087</span><span id="line-6087"> + flush.getFlushSequenceNumber() + " than what we have prepared with seqId: "</span> |
| <span class="source-line-no">6088</span><span id="line-6088"> + prepareFlushResult.flushOpSeqId + ". Picking up new file and dropping prepared"</span> |
| <span class="source-line-no">6089</span><span id="line-6089"> + " memstore snapshot");</span> |
| <span class="source-line-no">6090</span><span id="line-6090"></span> |
| <span class="source-line-no">6091</span><span id="line-6091"> replayFlushInStores(flush, prepareFlushResult, true);</span> |
| <span class="source-line-no">6092</span><span id="line-6092"></span> |
| <span class="source-line-no">6093</span><span id="line-6093"> // Set down the memstore size by amount of flush.</span> |
| <span class="source-line-no">6094</span><span id="line-6094"> this.decrMemStoreSize(prepareFlushResult.totalFlushableSize.getMemStoreSize());</span> |
| <span class="source-line-no">6095</span><span id="line-6095"></span> |
| <span class="source-line-no">6096</span><span id="line-6096"> // Inspect the memstore contents to see whether the memstore contains only edits</span> |
| <span class="source-line-no">6097</span><span id="line-6097"> // with seqId smaller than the flush seqId. If so, we can discard those edits.</span> |
| <span class="source-line-no">6098</span><span id="line-6098"> dropMemStoreContentsForSeqId(flush.getFlushSequenceNumber(), null);</span> |
| <span class="source-line-no">6099</span><span id="line-6099"></span> |
| <span class="source-line-no">6100</span><span id="line-6100"> this.prepareFlushResult = null;</span> |
| <span class="source-line-no">6101</span><span id="line-6101"> writestate.flushing = false;</span> |
| <span class="source-line-no">6102</span><span id="line-6102"> }</span> |
| <span class="source-line-no">6103</span><span id="line-6103"> // If we were waiting for observing a flush or region opening event for not showing</span> |
| <span class="source-line-no">6104</span><span id="line-6104"> // partial data after a secondary region crash, we can allow reads now. We can only make</span> |
| <span class="source-line-no">6105</span><span id="line-6105"> // sure that we are not showing partial data (for example skipping some previous edits)</span> |
| <span class="source-line-no">6106</span><span id="line-6106"> // until we observe a full flush start and flush commit. So if we were not able to find</span> |
| <span class="source-line-no">6107</span><span id="line-6107"> // a previous flush we will not enable reads now.</span> |
| <span class="source-line-no">6108</span><span id="line-6108"> this.setReadsEnabled(true);</span> |
| <span class="source-line-no">6109</span><span id="line-6109"> } else {</span> |
| <span class="source-line-no">6110</span><span id="line-6110"> LOG.warn(</span> |
| <span class="source-line-no">6111</span><span id="line-6111"> getRegionInfo().getEncodedName() + " : " + "Received a flush commit marker with seqId:"</span> |
| <span class="source-line-no">6112</span><span id="line-6112"> + flush.getFlushSequenceNumber() + ", but no previous prepared snapshot was found");</span> |
| <span class="source-line-no">6113</span><span id="line-6113"> // There is no corresponding prepare snapshot from before.</span> |
| <span class="source-line-no">6114</span><span id="line-6114"> // We will pick up the new flushed file</span> |
| <span class="source-line-no">6115</span><span id="line-6115"> replayFlushInStores(flush, null, false);</span> |
| <span class="source-line-no">6116</span><span id="line-6116"></span> |
| <span class="source-line-no">6117</span><span id="line-6117"> // Inspect the memstore contents to see whether the memstore contains only edits</span> |
| <span class="source-line-no">6118</span><span id="line-6118"> // with seqId smaller than the flush seqId. If so, we can discard those edits.</span> |
| <span class="source-line-no">6119</span><span id="line-6119"> dropMemStoreContentsForSeqId(flush.getFlushSequenceNumber(), null);</span> |
| <span class="source-line-no">6120</span><span id="line-6120"> }</span> |
| <span class="source-line-no">6121</span><span id="line-6121"></span> |
| <span class="source-line-no">6122</span><span id="line-6122"> status.markComplete("Flush commit successful");</span> |
| <span class="source-line-no">6123</span><span id="line-6123"></span> |
| <span class="source-line-no">6124</span><span id="line-6124"> // Update the last flushed sequence id for region.</span> |
| <span class="source-line-no">6125</span><span id="line-6125"> this.maxFlushedSeqId = flush.getFlushSequenceNumber();</span> |
| <span class="source-line-no">6126</span><span id="line-6126"></span> |
| <span class="source-line-no">6127</span><span id="line-6127"> // advance the mvcc read point so that the new flushed file is visible.</span> |
| <span class="source-line-no">6128</span><span id="line-6128"> mvcc.advanceTo(flush.getFlushSequenceNumber());</span> |
| <span class="source-line-no">6129</span><span id="line-6129"></span> |
| <span class="source-line-no">6130</span><span id="line-6130"> } catch (FileNotFoundException ex) {</span> |
| <span class="source-line-no">6131</span><span id="line-6131"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6132</span><span id="line-6132"> + "At least one of the store files in flush: " + TextFormat.shortDebugString(flush)</span> |
| <span class="source-line-no">6133</span><span id="line-6133"> + " doesn't exist any more. Skip loading the file(s)", ex);</span> |
| <span class="source-line-no">6134</span><span id="line-6134"> } finally {</span> |
| <span class="source-line-no">6135</span><span id="line-6135"> status.cleanup();</span> |
| <span class="source-line-no">6136</span><span id="line-6136"> writestate.notifyAll();</span> |
| <span class="source-line-no">6137</span><span id="line-6137"> }</span> |
| <span class="source-line-no">6138</span><span id="line-6138"> }</span> |
| <span class="source-line-no">6139</span><span id="line-6139"></span> |
| <span class="source-line-no">6140</span><span id="line-6140"> // C. Finally notify anyone waiting on memstore to clear:</span> |
| <span class="source-line-no">6141</span><span id="line-6141"> // e.g. checkResources().</span> |
| <span class="source-line-no">6142</span><span id="line-6142"> synchronized (this) {</span> |
| <span class="source-line-no">6143</span><span id="line-6143"> notifyAll(); // FindBugs NN_NAKED_NOTIFY</span> |
| <span class="source-line-no">6144</span><span id="line-6144"> }</span> |
| <span class="source-line-no">6145</span><span id="line-6145"> }</span> |
| <span class="source-line-no">6146</span><span id="line-6146"></span> |
| <span class="source-line-no">6147</span><span id="line-6147"> /**</span> |
| <span class="source-line-no">6148</span><span id="line-6148"> * Replays the given flush descriptor by opening the flush files in stores and dropping the</span> |
| <span class="source-line-no">6149</span><span id="line-6149"> * memstore snapshots if requested.</span> |
| <span class="source-line-no">6150</span><span id="line-6150"> * @deprecated Since 3.0.0, will be removed in 4.0.0. Only for keep compatibility for old region</span> |
| <span class="source-line-no">6151</span><span id="line-6151"> * replica implementation.</span> |
| <span class="source-line-no">6152</span><span id="line-6152"> */</span> |
| <span class="source-line-no">6153</span><span id="line-6153"> @Deprecated</span> |
| <span class="source-line-no">6154</span><span id="line-6154"> private void replayFlushInStores(FlushDescriptor flush, PrepareFlushResult prepareFlushResult,</span> |
| <span class="source-line-no">6155</span><span id="line-6155"> boolean dropMemstoreSnapshot) throws IOException {</span> |
| <span class="source-line-no">6156</span><span id="line-6156"> for (StoreFlushDescriptor storeFlush : flush.getStoreFlushesList()) {</span> |
| <span class="source-line-no">6157</span><span id="line-6157"> byte[] family = storeFlush.getFamilyName().toByteArray();</span> |
| <span class="source-line-no">6158</span><span id="line-6158"> HStore store = getStore(family);</span> |
| <span class="source-line-no">6159</span><span id="line-6159"> if (store == null) {</span> |
| <span class="source-line-no">6160</span><span id="line-6160"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6161</span><span id="line-6161"> + "Received a flush commit marker from primary, but the family is not found."</span> |
| <span class="source-line-no">6162</span><span id="line-6162"> + "Ignoring StoreFlushDescriptor:" + storeFlush);</span> |
| <span class="source-line-no">6163</span><span id="line-6163"> continue;</span> |
| <span class="source-line-no">6164</span><span id="line-6164"> }</span> |
| <span class="source-line-no">6165</span><span id="line-6165"> List<String> flushFiles = storeFlush.getFlushOutputList();</span> |
| <span class="source-line-no">6166</span><span id="line-6166"> StoreFlushContext ctx = null;</span> |
| <span class="source-line-no">6167</span><span id="line-6167"> long startTime = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">6168</span><span id="line-6168"> if (prepareFlushResult == null || prepareFlushResult.storeFlushCtxs == null) {</span> |
| <span class="source-line-no">6169</span><span id="line-6169"> ctx = store.createFlushContext(flush.getFlushSequenceNumber(), FlushLifeCycleTracker.DUMMY);</span> |
| <span class="source-line-no">6170</span><span id="line-6170"> } else {</span> |
| <span class="source-line-no">6171</span><span id="line-6171"> ctx = prepareFlushResult.storeFlushCtxs.get(family);</span> |
| <span class="source-line-no">6172</span><span id="line-6172"> startTime = prepareFlushResult.startTime;</span> |
| <span class="source-line-no">6173</span><span id="line-6173"> }</span> |
| <span class="source-line-no">6174</span><span id="line-6174"></span> |
| <span class="source-line-no">6175</span><span id="line-6175"> if (ctx == null) {</span> |
| <span class="source-line-no">6176</span><span id="line-6176"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6177</span><span id="line-6177"> + "Unexpected: flush commit marker received from store " + Bytes.toString(family)</span> |
| <span class="source-line-no">6178</span><span id="line-6178"> + " but no associated flush context. Ignoring");</span> |
| <span class="source-line-no">6179</span><span id="line-6179"> continue;</span> |
| <span class="source-line-no">6180</span><span id="line-6180"> }</span> |
| <span class="source-line-no">6181</span><span id="line-6181"></span> |
| <span class="source-line-no">6182</span><span id="line-6182"> ctx.replayFlush(flushFiles, dropMemstoreSnapshot); // replay the flush</span> |
| <span class="source-line-no">6183</span><span id="line-6183"></span> |
| <span class="source-line-no">6184</span><span id="line-6184"> // Record latest flush time</span> |
| <span class="source-line-no">6185</span><span id="line-6185"> this.lastStoreFlushTimeMap.put(store, startTime);</span> |
| <span class="source-line-no">6186</span><span id="line-6186"> }</span> |
| <span class="source-line-no">6187</span><span id="line-6187"> }</span> |
| <span class="source-line-no">6188</span><span id="line-6188"></span> |
| <span class="source-line-no">6189</span><span id="line-6189"> private long loadRecoveredHFilesIfAny(Collection<HStore> stores) throws IOException {</span> |
| <span class="source-line-no">6190</span><span id="line-6190"> Path regionDir = fs.getRegionDir();</span> |
| <span class="source-line-no">6191</span><span id="line-6191"> long maxSeqId = -1;</span> |
| <span class="source-line-no">6192</span><span id="line-6192"> for (HStore store : stores) {</span> |
| <span class="source-line-no">6193</span><span id="line-6193"> String familyName = store.getColumnFamilyName();</span> |
| <span class="source-line-no">6194</span><span id="line-6194"> FileStatus[] files =</span> |
| <span class="source-line-no">6195</span><span id="line-6195"> WALSplitUtil.getRecoveredHFiles(fs.getFileSystem(), regionDir, familyName);</span> |
| <span class="source-line-no">6196</span><span id="line-6196"> if (files != null && files.length != 0) {</span> |
| <span class="source-line-no">6197</span><span id="line-6197"> for (FileStatus file : files) {</span> |
| <span class="source-line-no">6198</span><span id="line-6198"> Path filePath = file.getPath();</span> |
| <span class="source-line-no">6199</span><span id="line-6199"> // If file length is zero then delete it</span> |
| <span class="source-line-no">6200</span><span id="line-6200"> if (isZeroLengthThenDelete(fs.getFileSystem(), file, filePath)) {</span> |
| <span class="source-line-no">6201</span><span id="line-6201"> continue;</span> |
| <span class="source-line-no">6202</span><span id="line-6202"> }</span> |
| <span class="source-line-no">6203</span><span id="line-6203"> try {</span> |
| <span class="source-line-no">6204</span><span id="line-6204"> HStoreFile storefile = store.tryCommitRecoveredHFile(file.getPath());</span> |
| <span class="source-line-no">6205</span><span id="line-6205"> maxSeqId = Math.max(maxSeqId, storefile.getReader().getSequenceID());</span> |
| <span class="source-line-no">6206</span><span id="line-6206"> } catch (IOException e) {</span> |
| <span class="source-line-no">6207</span><span id="line-6207"> handleException(fs.getFileSystem(), filePath, e);</span> |
| <span class="source-line-no">6208</span><span id="line-6208"> continue;</span> |
| <span class="source-line-no">6209</span><span id="line-6209"> }</span> |
| <span class="source-line-no">6210</span><span id="line-6210"> }</span> |
| <span class="source-line-no">6211</span><span id="line-6211"> if (this.rsServices != null && store.needsCompaction()) {</span> |
| <span class="source-line-no">6212</span><span id="line-6212"> this.rsServices.getCompactionRequestor().requestCompaction(this, store,</span> |
| <span class="source-line-no">6213</span><span id="line-6213"> "load recovered hfiles request compaction", Store.PRIORITY_USER + 1,</span> |
| <span class="source-line-no">6214</span><span id="line-6214"> CompactionLifeCycleTracker.DUMMY, null);</span> |
| <span class="source-line-no">6215</span><span id="line-6215"> }</span> |
| <span class="source-line-no">6216</span><span id="line-6216"> }</span> |
| <span class="source-line-no">6217</span><span id="line-6217"> }</span> |
| <span class="source-line-no">6218</span><span id="line-6218"> return maxSeqId;</span> |
| <span class="source-line-no">6219</span><span id="line-6219"> }</span> |
| <span class="source-line-no">6220</span><span id="line-6220"></span> |
| <span class="source-line-no">6221</span><span id="line-6221"> /**</span> |
| <span class="source-line-no">6222</span><span id="line-6222"> * Be careful, this method will drop all data in the memstore of this region. Currently, this</span> |
| <span class="source-line-no">6223</span><span id="line-6223"> * method is used to drop memstore to prevent memory leak when replaying recovered.edits while</span> |
| <span class="source-line-no">6224</span><span id="line-6224"> * opening region.</span> |
| <span class="source-line-no">6225</span><span id="line-6225"> */</span> |
| <span class="source-line-no">6226</span><span id="line-6226"> private MemStoreSize dropMemStoreContents() throws IOException {</span> |
| <span class="source-line-no">6227</span><span id="line-6227"> MemStoreSizing totalFreedSize = new NonThreadSafeMemStoreSizing();</span> |
| <span class="source-line-no">6228</span><span id="line-6228"> this.updatesLock.writeLock().lock();</span> |
| <span class="source-line-no">6229</span><span id="line-6229"> try {</span> |
| <span class="source-line-no">6230</span><span id="line-6230"> for (HStore s : stores.values()) {</span> |
| <span class="source-line-no">6231</span><span id="line-6231"> MemStoreSize memStoreSize = doDropStoreMemStoreContentsForSeqId(s, HConstants.NO_SEQNUM);</span> |
| <span class="source-line-no">6232</span><span id="line-6232"> LOG.info("Drop memstore for Store " + s.getColumnFamilyName() + " in region "</span> |
| <span class="source-line-no">6233</span><span id="line-6233"> + this.getRegionInfo().getRegionNameAsString() + " , dropped memstoresize: ["</span> |
| <span class="source-line-no">6234</span><span id="line-6234"> + memStoreSize + " }");</span> |
| <span class="source-line-no">6235</span><span id="line-6235"> totalFreedSize.incMemStoreSize(memStoreSize);</span> |
| <span class="source-line-no">6236</span><span id="line-6236"> }</span> |
| <span class="source-line-no">6237</span><span id="line-6237"> return totalFreedSize.getMemStoreSize();</span> |
| <span class="source-line-no">6238</span><span id="line-6238"> } finally {</span> |
| <span class="source-line-no">6239</span><span id="line-6239"> this.updatesLock.writeLock().unlock();</span> |
| <span class="source-line-no">6240</span><span id="line-6240"> }</span> |
| <span class="source-line-no">6241</span><span id="line-6241"> }</span> |
| <span class="source-line-no">6242</span><span id="line-6242"></span> |
| <span class="source-line-no">6243</span><span id="line-6243"> /**</span> |
| <span class="source-line-no">6244</span><span id="line-6244"> * Drops the memstore contents after replaying a flush descriptor or region open event replay if</span> |
| <span class="source-line-no">6245</span><span id="line-6245"> * the memstore edits have seqNums smaller than the given seq id</span> |
| <span class="source-line-no">6246</span><span id="line-6246"> */</span> |
| <span class="source-line-no">6247</span><span id="line-6247"> private MemStoreSize dropMemStoreContentsForSeqId(long seqId, HStore store) throws IOException {</span> |
| <span class="source-line-no">6248</span><span id="line-6248"> MemStoreSizing totalFreedSize = new NonThreadSafeMemStoreSizing();</span> |
| <span class="source-line-no">6249</span><span id="line-6249"> this.updatesLock.writeLock().lock();</span> |
| <span class="source-line-no">6250</span><span id="line-6250"> try {</span> |
| <span class="source-line-no">6251</span><span id="line-6251"></span> |
| <span class="source-line-no">6252</span><span id="line-6252"> long currentSeqId = mvcc.getReadPoint();</span> |
| <span class="source-line-no">6253</span><span id="line-6253"> if (seqId >= currentSeqId) {</span> |
| <span class="source-line-no">6254</span><span id="line-6254"> // then we can drop the memstore contents since everything is below this seqId</span> |
| <span class="source-line-no">6255</span><span id="line-6255"> LOG.info(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6256</span><span id="line-6256"> + "Dropping memstore contents as well since replayed flush seqId: " + seqId</span> |
| <span class="source-line-no">6257</span><span id="line-6257"> + " is greater than current seqId:" + currentSeqId);</span> |
| <span class="source-line-no">6258</span><span id="line-6258"></span> |
| <span class="source-line-no">6259</span><span id="line-6259"> // Prepare flush (take a snapshot) and then abort (drop the snapshot)</span> |
| <span class="source-line-no">6260</span><span id="line-6260"> if (store == null) {</span> |
| <span class="source-line-no">6261</span><span id="line-6261"> for (HStore s : stores.values()) {</span> |
| <span class="source-line-no">6262</span><span id="line-6262"> totalFreedSize.incMemStoreSize(doDropStoreMemStoreContentsForSeqId(s, currentSeqId));</span> |
| <span class="source-line-no">6263</span><span id="line-6263"> }</span> |
| <span class="source-line-no">6264</span><span id="line-6264"> } else {</span> |
| <span class="source-line-no">6265</span><span id="line-6265"> totalFreedSize.incMemStoreSize(doDropStoreMemStoreContentsForSeqId(store, currentSeqId));</span> |
| <span class="source-line-no">6266</span><span id="line-6266"> }</span> |
| <span class="source-line-no">6267</span><span id="line-6267"> } else {</span> |
| <span class="source-line-no">6268</span><span id="line-6268"> LOG.info(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6269</span><span id="line-6269"> + "Not dropping memstore contents since replayed flush seqId: " + seqId</span> |
| <span class="source-line-no">6270</span><span id="line-6270"> + " is smaller than current seqId:" + currentSeqId);</span> |
| <span class="source-line-no">6271</span><span id="line-6271"> }</span> |
| <span class="source-line-no">6272</span><span id="line-6272"> } finally {</span> |
| <span class="source-line-no">6273</span><span id="line-6273"> this.updatesLock.writeLock().unlock();</span> |
| <span class="source-line-no">6274</span><span id="line-6274"> }</span> |
| <span class="source-line-no">6275</span><span id="line-6275"> return totalFreedSize.getMemStoreSize();</span> |
| <span class="source-line-no">6276</span><span id="line-6276"> }</span> |
| <span class="source-line-no">6277</span><span id="line-6277"></span> |
| <span class="source-line-no">6278</span><span id="line-6278"> private MemStoreSize doDropStoreMemStoreContentsForSeqId(HStore s, long currentSeqId)</span> |
| <span class="source-line-no">6279</span><span id="line-6279"> throws IOException {</span> |
| <span class="source-line-no">6280</span><span id="line-6280"> MemStoreSize flushableSize = s.getFlushableSize();</span> |
| <span class="source-line-no">6281</span><span id="line-6281"> this.decrMemStoreSize(flushableSize);</span> |
| <span class="source-line-no">6282</span><span id="line-6282"> StoreFlushContext ctx = s.createFlushContext(currentSeqId, FlushLifeCycleTracker.DUMMY);</span> |
| <span class="source-line-no">6283</span><span id="line-6283"> ctx.prepare();</span> |
| <span class="source-line-no">6284</span><span id="line-6284"> ctx.abort();</span> |
| <span class="source-line-no">6285</span><span id="line-6285"> return flushableSize;</span> |
| <span class="source-line-no">6286</span><span id="line-6286"> }</span> |
| <span class="source-line-no">6287</span><span id="line-6287"></span> |
| <span class="source-line-no">6288</span><span id="line-6288"> private void replayWALFlushAbortMarker(FlushDescriptor flush) {</span> |
| <span class="source-line-no">6289</span><span id="line-6289"> // nothing to do for now. A flush abort will cause a RS abort which means that the region</span> |
| <span class="source-line-no">6290</span><span id="line-6290"> // will be opened somewhere else later. We will see the region open event soon, and replaying</span> |
| <span class="source-line-no">6291</span><span id="line-6291"> // that will drop the snapshot</span> |
| <span class="source-line-no">6292</span><span id="line-6292"> }</span> |
| <span class="source-line-no">6293</span><span id="line-6293"></span> |
| <span class="source-line-no">6294</span><span id="line-6294"> private void replayWALFlushCannotFlushMarker(FlushDescriptor flush, long replaySeqId) {</span> |
| <span class="source-line-no">6295</span><span id="line-6295"> synchronized (writestate) {</span> |
| <span class="source-line-no">6296</span><span id="line-6296"> if (this.lastReplayedOpenRegionSeqId > replaySeqId) {</span> |
| <span class="source-line-no">6297</span><span id="line-6297"> LOG.warn(getRegionInfo().getEncodedName() + " : " + "Skipping replaying flush event :"</span> |
| <span class="source-line-no">6298</span><span id="line-6298"> + TextFormat.shortDebugString(flush) + " because its sequence id " + replaySeqId</span> |
| <span class="source-line-no">6299</span><span id="line-6299"> + " is smaller than this regions " + "lastReplayedOpenRegionSeqId of "</span> |
| <span class="source-line-no">6300</span><span id="line-6300"> + lastReplayedOpenRegionSeqId);</span> |
| <span class="source-line-no">6301</span><span id="line-6301"> return;</span> |
| <span class="source-line-no">6302</span><span id="line-6302"> }</span> |
| <span class="source-line-no">6303</span><span id="line-6303"></span> |
| <span class="source-line-no">6304</span><span id="line-6304"> // If we were waiting for observing a flush or region opening event for not showing partial</span> |
| <span class="source-line-no">6305</span><span id="line-6305"> // data after a secondary region crash, we can allow reads now. This event means that the</span> |
| <span class="source-line-no">6306</span><span id="line-6306"> // primary was not able to flush because memstore is empty when we requested flush. By the</span> |
| <span class="source-line-no">6307</span><span id="line-6307"> // time we observe this, we are guaranteed to have up to date seqId with our previous</span> |
| <span class="source-line-no">6308</span><span id="line-6308"> // assignment.</span> |
| <span class="source-line-no">6309</span><span id="line-6309"> this.setReadsEnabled(true);</span> |
| <span class="source-line-no">6310</span><span id="line-6310"> }</span> |
| <span class="source-line-no">6311</span><span id="line-6311"> }</span> |
| <span class="source-line-no">6312</span><span id="line-6312"></span> |
| <span class="source-line-no">6313</span><span id="line-6313"> PrepareFlushResult getPrepareFlushResult() {</span> |
| <span class="source-line-no">6314</span><span id="line-6314"> return prepareFlushResult;</span> |
| <span class="source-line-no">6315</span><span id="line-6315"> }</span> |
| <span class="source-line-no">6316</span><span id="line-6316"></span> |
| <span class="source-line-no">6317</span><span id="line-6317"> /**</span> |
| <span class="source-line-no">6318</span><span id="line-6318"> * @deprecated Since 3.0.0, will be removed in 4.0.0. Only for keep compatibility for old region</span> |
| <span class="source-line-no">6319</span><span id="line-6319"> * replica implementation.</span> |
| <span class="source-line-no">6320</span><span id="line-6320"> */</span> |
| <span class="source-line-no">6321</span><span id="line-6321"> @Deprecated</span> |
| <span class="source-line-no">6322</span><span id="line-6322"> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NN_NAKED_NOTIFY",</span> |
| <span class="source-line-no">6323</span><span id="line-6323"> justification = "Intentional; cleared the memstore")</span> |
| <span class="source-line-no">6324</span><span id="line-6324"> void replayWALRegionEventMarker(RegionEventDescriptor regionEvent) throws IOException {</span> |
| <span class="source-line-no">6325</span><span id="line-6325"> checkTargetRegion(regionEvent.getEncodedRegionName().toByteArray(),</span> |
| <span class="source-line-no">6326</span><span id="line-6326"> "RegionEvent marker from WAL ", regionEvent);</span> |
| <span class="source-line-no">6327</span><span id="line-6327"></span> |
| <span class="source-line-no">6328</span><span id="line-6328"> startRegionOperation(Operation.REPLAY_EVENT);</span> |
| <span class="source-line-no">6329</span><span id="line-6329"> try {</span> |
| <span class="source-line-no">6330</span><span id="line-6330"> if (ServerRegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {</span> |
| <span class="source-line-no">6331</span><span id="line-6331"> return; // if primary nothing to do</span> |
| <span class="source-line-no">6332</span><span id="line-6332"> }</span> |
| <span class="source-line-no">6333</span><span id="line-6333"></span> |
| <span class="source-line-no">6334</span><span id="line-6334"> if (regionEvent.getEventType() == EventType.REGION_CLOSE) {</span> |
| <span class="source-line-no">6335</span><span id="line-6335"> // nothing to do on REGION_CLOSE for now.</span> |
| <span class="source-line-no">6336</span><span id="line-6336"> return;</span> |
| <span class="source-line-no">6337</span><span id="line-6337"> }</span> |
| <span class="source-line-no">6338</span><span id="line-6338"> if (regionEvent.getEventType() != EventType.REGION_OPEN) {</span> |
| <span class="source-line-no">6339</span><span id="line-6339"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6340</span><span id="line-6340"> + "Unknown region event received, ignoring :" + TextFormat.shortDebugString(regionEvent));</span> |
| <span class="source-line-no">6341</span><span id="line-6341"> return;</span> |
| <span class="source-line-no">6342</span><span id="line-6342"> }</span> |
| <span class="source-line-no">6343</span><span id="line-6343"></span> |
| <span class="source-line-no">6344</span><span id="line-6344"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">6345</span><span id="line-6345"> LOG.debug(getRegionInfo().getEncodedName() + " : " + "Replaying region open event marker "</span> |
| <span class="source-line-no">6346</span><span id="line-6346"> + TextFormat.shortDebugString(regionEvent));</span> |
| <span class="source-line-no">6347</span><span id="line-6347"> }</span> |
| <span class="source-line-no">6348</span><span id="line-6348"></span> |
| <span class="source-line-no">6349</span><span id="line-6349"> // we will use writestate as a coarse-grain lock for all the replay events</span> |
| <span class="source-line-no">6350</span><span id="line-6350"> synchronized (writestate) {</span> |
| <span class="source-line-no">6351</span><span id="line-6351"> // Replication can deliver events out of order when primary region moves or the region</span> |
| <span class="source-line-no">6352</span><span id="line-6352"> // server crashes, since there is no coordination between replication of different wal files</span> |
| <span class="source-line-no">6353</span><span id="line-6353"> // belonging to different region servers. We have to safe guard against this case by using</span> |
| <span class="source-line-no">6354</span><span id="line-6354"> // region open event's seqid. Since this is the first event that the region puts (after</span> |
| <span class="source-line-no">6355</span><span id="line-6355"> // possibly flushing recovered.edits), after seeing this event, we can ignore every edit</span> |
| <span class="source-line-no">6356</span><span id="line-6356"> // smaller than this seqId</span> |
| <span class="source-line-no">6357</span><span id="line-6357"> if (this.lastReplayedOpenRegionSeqId <= regionEvent.getLogSequenceNumber()) {</span> |
| <span class="source-line-no">6358</span><span id="line-6358"> this.lastReplayedOpenRegionSeqId = regionEvent.getLogSequenceNumber();</span> |
| <span class="source-line-no">6359</span><span id="line-6359"> } else {</span> |
| <span class="source-line-no">6360</span><span id="line-6360"> LOG.warn(getRegionInfo().getEncodedName() + " : " + "Skipping replaying region event :"</span> |
| <span class="source-line-no">6361</span><span id="line-6361"> + TextFormat.shortDebugString(regionEvent)</span> |
| <span class="source-line-no">6362</span><span id="line-6362"> + " because its sequence id is smaller than this regions lastReplayedOpenRegionSeqId "</span> |
| <span class="source-line-no">6363</span><span id="line-6363"> + " of " + lastReplayedOpenRegionSeqId);</span> |
| <span class="source-line-no">6364</span><span id="line-6364"> return;</span> |
| <span class="source-line-no">6365</span><span id="line-6365"> }</span> |
| <span class="source-line-no">6366</span><span id="line-6366"></span> |
| <span class="source-line-no">6367</span><span id="line-6367"> // region open lists all the files that the region has at the time of the opening. Just pick</span> |
| <span class="source-line-no">6368</span><span id="line-6368"> // all the files and drop prepared flushes and empty memstores</span> |
| <span class="source-line-no">6369</span><span id="line-6369"> for (StoreDescriptor storeDescriptor : regionEvent.getStoresList()) {</span> |
| <span class="source-line-no">6370</span><span id="line-6370"> // stores of primary may be different now</span> |
| <span class="source-line-no">6371</span><span id="line-6371"> byte[] family = storeDescriptor.getFamilyName().toByteArray();</span> |
| <span class="source-line-no">6372</span><span id="line-6372"> HStore store = getStore(family);</span> |
| <span class="source-line-no">6373</span><span id="line-6373"> if (store == null) {</span> |
| <span class="source-line-no">6374</span><span id="line-6374"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6375</span><span id="line-6375"> + "Received a region open marker from primary, but the family is not found. "</span> |
| <span class="source-line-no">6376</span><span id="line-6376"> + "Ignoring. StoreDescriptor:" + storeDescriptor);</span> |
| <span class="source-line-no">6377</span><span id="line-6377"> continue;</span> |
| <span class="source-line-no">6378</span><span id="line-6378"> }</span> |
| <span class="source-line-no">6379</span><span id="line-6379"></span> |
| <span class="source-line-no">6380</span><span id="line-6380"> long storeSeqId = store.getMaxSequenceId().orElse(0L);</span> |
| <span class="source-line-no">6381</span><span id="line-6381"> List<String> storeFiles = storeDescriptor.getStoreFileList();</span> |
| <span class="source-line-no">6382</span><span id="line-6382"> try {</span> |
| <span class="source-line-no">6383</span><span id="line-6383"> store.refreshStoreFiles(storeFiles); // replace the files with the new ones</span> |
| <span class="source-line-no">6384</span><span id="line-6384"> } catch (FileNotFoundException ex) {</span> |
| <span class="source-line-no">6385</span><span id="line-6385"> LOG.warn(getRegionInfo().getEncodedName() + " : " + "At least one of the store files: "</span> |
| <span class="source-line-no">6386</span><span id="line-6386"> + storeFiles + " doesn't exist any more. Skip loading the file(s)", ex);</span> |
| <span class="source-line-no">6387</span><span id="line-6387"> continue;</span> |
| <span class="source-line-no">6388</span><span id="line-6388"> }</span> |
| <span class="source-line-no">6389</span><span id="line-6389"> if (store.getMaxSequenceId().orElse(0L) != storeSeqId) {</span> |
| <span class="source-line-no">6390</span><span id="line-6390"> // Record latest flush time if we picked up new files</span> |
| <span class="source-line-no">6391</span><span id="line-6391"> lastStoreFlushTimeMap.put(store, EnvironmentEdgeManager.currentTime());</span> |
| <span class="source-line-no">6392</span><span id="line-6392"> }</span> |
| <span class="source-line-no">6393</span><span id="line-6393"></span> |
| <span class="source-line-no">6394</span><span id="line-6394"> if (writestate.flushing) {</span> |
| <span class="source-line-no">6395</span><span id="line-6395"> // only drop memstore snapshots if they are smaller than last flush for the store</span> |
| <span class="source-line-no">6396</span><span id="line-6396"> if (this.prepareFlushResult.flushOpSeqId <= regionEvent.getLogSequenceNumber()) {</span> |
| <span class="source-line-no">6397</span><span id="line-6397"> StoreFlushContext ctx = this.prepareFlushResult.storeFlushCtxs == null</span> |
| <span class="source-line-no">6398</span><span id="line-6398"> ? null</span> |
| <span class="source-line-no">6399</span><span id="line-6399"> : this.prepareFlushResult.storeFlushCtxs.get(family);</span> |
| <span class="source-line-no">6400</span><span id="line-6400"> if (ctx != null) {</span> |
| <span class="source-line-no">6401</span><span id="line-6401"> MemStoreSize mss = store.getFlushableSize();</span> |
| <span class="source-line-no">6402</span><span id="line-6402"> ctx.abort();</span> |
| <span class="source-line-no">6403</span><span id="line-6403"> this.decrMemStoreSize(mss);</span> |
| <span class="source-line-no">6404</span><span id="line-6404"> this.prepareFlushResult.storeFlushCtxs.remove(family);</span> |
| <span class="source-line-no">6405</span><span id="line-6405"> }</span> |
| <span class="source-line-no">6406</span><span id="line-6406"> }</span> |
| <span class="source-line-no">6407</span><span id="line-6407"> }</span> |
| <span class="source-line-no">6408</span><span id="line-6408"></span> |
| <span class="source-line-no">6409</span><span id="line-6409"> // Drop the memstore contents if they are now smaller than the latest seen flushed file</span> |
| <span class="source-line-no">6410</span><span id="line-6410"> dropMemStoreContentsForSeqId(regionEvent.getLogSequenceNumber(), store);</span> |
| <span class="source-line-no">6411</span><span id="line-6411"> if (storeSeqId > this.maxFlushedSeqId) {</span> |
| <span class="source-line-no">6412</span><span id="line-6412"> this.maxFlushedSeqId = storeSeqId;</span> |
| <span class="source-line-no">6413</span><span id="line-6413"> }</span> |
| <span class="source-line-no">6414</span><span id="line-6414"> }</span> |
| <span class="source-line-no">6415</span><span id="line-6415"></span> |
| <span class="source-line-no">6416</span><span id="line-6416"> // if all stores ended up dropping their snapshots, we can safely drop the</span> |
| <span class="source-line-no">6417</span><span id="line-6417"> // prepareFlushResult</span> |
| <span class="source-line-no">6418</span><span id="line-6418"> dropPrepareFlushIfPossible();</span> |
| <span class="source-line-no">6419</span><span id="line-6419"></span> |
| <span class="source-line-no">6420</span><span id="line-6420"> // advance the mvcc read point so that the new flushed file is visible.</span> |
| <span class="source-line-no">6421</span><span id="line-6421"> mvcc.await();</span> |
| <span class="source-line-no">6422</span><span id="line-6422"></span> |
| <span class="source-line-no">6423</span><span id="line-6423"> // If we were waiting for observing a flush or region opening event for not showing partial</span> |
| <span class="source-line-no">6424</span><span id="line-6424"> // data after a secondary region crash, we can allow reads now.</span> |
| <span class="source-line-no">6425</span><span id="line-6425"> this.setReadsEnabled(true);</span> |
| <span class="source-line-no">6426</span><span id="line-6426"></span> |
| <span class="source-line-no">6427</span><span id="line-6427"> // C. Finally notify anyone waiting on memstore to clear:</span> |
| <span class="source-line-no">6428</span><span id="line-6428"> // e.g. checkResources().</span> |
| <span class="source-line-no">6429</span><span id="line-6429"> synchronized (this) {</span> |
| <span class="source-line-no">6430</span><span id="line-6430"> notifyAll(); // FindBugs NN_NAKED_NOTIFY</span> |
| <span class="source-line-no">6431</span><span id="line-6431"> }</span> |
| <span class="source-line-no">6432</span><span id="line-6432"> }</span> |
| <span class="source-line-no">6433</span><span id="line-6433"> logRegionFiles();</span> |
| <span class="source-line-no">6434</span><span id="line-6434"> } finally {</span> |
| <span class="source-line-no">6435</span><span id="line-6435"> closeRegionOperation(Operation.REPLAY_EVENT);</span> |
| <span class="source-line-no">6436</span><span id="line-6436"> }</span> |
| <span class="source-line-no">6437</span><span id="line-6437"> }</span> |
| <span class="source-line-no">6438</span><span id="line-6438"></span> |
| <span class="source-line-no">6439</span><span id="line-6439"> /**</span> |
| <span class="source-line-no">6440</span><span id="line-6440"> * @deprecated Since 3.0.0, will be removed in 4.0.0. Only for keep compatibility for old region</span> |
| <span class="source-line-no">6441</span><span id="line-6441"> * replica implementation.</span> |
| <span class="source-line-no">6442</span><span id="line-6442"> */</span> |
| <span class="source-line-no">6443</span><span id="line-6443"> @Deprecated</span> |
| <span class="source-line-no">6444</span><span id="line-6444"> void replayWALBulkLoadEventMarker(WALProtos.BulkLoadDescriptor bulkLoadEvent) throws IOException {</span> |
| <span class="source-line-no">6445</span><span id="line-6445"> checkTargetRegion(bulkLoadEvent.getEncodedRegionName().toByteArray(),</span> |
| <span class="source-line-no">6446</span><span id="line-6446"> "BulkLoad marker from WAL ", bulkLoadEvent);</span> |
| <span class="source-line-no">6447</span><span id="line-6447"></span> |
| <span class="source-line-no">6448</span><span id="line-6448"> if (ServerRegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {</span> |
| <span class="source-line-no">6449</span><span id="line-6449"> return; // if primary nothing to do</span> |
| <span class="source-line-no">6450</span><span id="line-6450"> }</span> |
| <span class="source-line-no">6451</span><span id="line-6451"></span> |
| <span class="source-line-no">6452</span><span id="line-6452"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">6453</span><span id="line-6453"> LOG.debug(getRegionInfo().getEncodedName() + " : " + "Replaying bulkload event marker "</span> |
| <span class="source-line-no">6454</span><span id="line-6454"> + TextFormat.shortDebugString(bulkLoadEvent));</span> |
| <span class="source-line-no">6455</span><span id="line-6455"> }</span> |
| <span class="source-line-no">6456</span><span id="line-6456"> // check if multiple families involved</span> |
| <span class="source-line-no">6457</span><span id="line-6457"> boolean multipleFamilies = false;</span> |
| <span class="source-line-no">6458</span><span id="line-6458"> byte[] family = null;</span> |
| <span class="source-line-no">6459</span><span id="line-6459"> for (StoreDescriptor storeDescriptor : bulkLoadEvent.getStoresList()) {</span> |
| <span class="source-line-no">6460</span><span id="line-6460"> byte[] fam = storeDescriptor.getFamilyName().toByteArray();</span> |
| <span class="source-line-no">6461</span><span id="line-6461"> if (family == null) {</span> |
| <span class="source-line-no">6462</span><span id="line-6462"> family = fam;</span> |
| <span class="source-line-no">6463</span><span id="line-6463"> } else if (!Bytes.equals(family, fam)) {</span> |
| <span class="source-line-no">6464</span><span id="line-6464"> multipleFamilies = true;</span> |
| <span class="source-line-no">6465</span><span id="line-6465"> break;</span> |
| <span class="source-line-no">6466</span><span id="line-6466"> }</span> |
| <span class="source-line-no">6467</span><span id="line-6467"> }</span> |
| <span class="source-line-no">6468</span><span id="line-6468"></span> |
| <span class="source-line-no">6469</span><span id="line-6469"> startBulkRegionOperation(multipleFamilies);</span> |
| <span class="source-line-no">6470</span><span id="line-6470"> try {</span> |
| <span class="source-line-no">6471</span><span id="line-6471"> // we will use writestate as a coarse-grain lock for all the replay events</span> |
| <span class="source-line-no">6472</span><span id="line-6472"> synchronized (writestate) {</span> |
| <span class="source-line-no">6473</span><span id="line-6473"> // Replication can deliver events out of order when primary region moves or the region</span> |
| <span class="source-line-no">6474</span><span id="line-6474"> // server crashes, since there is no coordination between replication of different wal files</span> |
| <span class="source-line-no">6475</span><span id="line-6475"> // belonging to different region servers. We have to safe guard against this case by using</span> |
| <span class="source-line-no">6476</span><span id="line-6476"> // region open event's seqid. Since this is the first event that the region puts (after</span> |
| <span class="source-line-no">6477</span><span id="line-6477"> // possibly flushing recovered.edits), after seeing this event, we can ignore every edit</span> |
| <span class="source-line-no">6478</span><span id="line-6478"> // smaller than this seqId</span> |
| <span class="source-line-no">6479</span><span id="line-6479"> if (</span> |
| <span class="source-line-no">6480</span><span id="line-6480"> bulkLoadEvent.getBulkloadSeqNum() >= 0</span> |
| <span class="source-line-no">6481</span><span id="line-6481"> && this.lastReplayedOpenRegionSeqId >= bulkLoadEvent.getBulkloadSeqNum()</span> |
| <span class="source-line-no">6482</span><span id="line-6482"> ) {</span> |
| <span class="source-line-no">6483</span><span id="line-6483"> LOG.warn(getRegionInfo().getEncodedName() + " : " + "Skipping replaying bulkload event :"</span> |
| <span class="source-line-no">6484</span><span id="line-6484"> + TextFormat.shortDebugString(bulkLoadEvent)</span> |
| <span class="source-line-no">6485</span><span id="line-6485"> + " because its sequence id is smaller than this region's lastReplayedOpenRegionSeqId"</span> |
| <span class="source-line-no">6486</span><span id="line-6486"> + " =" + lastReplayedOpenRegionSeqId);</span> |
| <span class="source-line-no">6487</span><span id="line-6487"></span> |
| <span class="source-line-no">6488</span><span id="line-6488"> return;</span> |
| <span class="source-line-no">6489</span><span id="line-6489"> }</span> |
| <span class="source-line-no">6490</span><span id="line-6490"></span> |
| <span class="source-line-no">6491</span><span id="line-6491"> for (StoreDescriptor storeDescriptor : bulkLoadEvent.getStoresList()) {</span> |
| <span class="source-line-no">6492</span><span id="line-6492"> // stores of primary may be different now</span> |
| <span class="source-line-no">6493</span><span id="line-6493"> family = storeDescriptor.getFamilyName().toByteArray();</span> |
| <span class="source-line-no">6494</span><span id="line-6494"> HStore store = getStore(family);</span> |
| <span class="source-line-no">6495</span><span id="line-6495"> if (store == null) {</span> |
| <span class="source-line-no">6496</span><span id="line-6496"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6497</span><span id="line-6497"> + "Received a bulk load marker from primary, but the family is not found. "</span> |
| <span class="source-line-no">6498</span><span id="line-6498"> + "Ignoring. StoreDescriptor:" + storeDescriptor);</span> |
| <span class="source-line-no">6499</span><span id="line-6499"> continue;</span> |
| <span class="source-line-no">6500</span><span id="line-6500"> }</span> |
| <span class="source-line-no">6501</span><span id="line-6501"></span> |
| <span class="source-line-no">6502</span><span id="line-6502"> List<String> storeFiles = storeDescriptor.getStoreFileList();</span> |
| <span class="source-line-no">6503</span><span id="line-6503"> for (String storeFile : storeFiles) {</span> |
| <span class="source-line-no">6504</span><span id="line-6504"> StoreFileInfo storeFileInfo = null;</span> |
| <span class="source-line-no">6505</span><span id="line-6505"> try {</span> |
| <span class="source-line-no">6506</span><span id="line-6506"> storeFileInfo = fs.getStoreFileInfo(Bytes.toString(family), storeFile);</span> |
| <span class="source-line-no">6507</span><span id="line-6507"> store.bulkLoadHFile(storeFileInfo);</span> |
| <span class="source-line-no">6508</span><span id="line-6508"> } catch (FileNotFoundException ex) {</span> |
| <span class="source-line-no">6509</span><span id="line-6509"> LOG.warn(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6510</span><span id="line-6510"> + ((storeFileInfo != null)</span> |
| <span class="source-line-no">6511</span><span id="line-6511"> ? storeFileInfo.toString()</span> |
| <span class="source-line-no">6512</span><span id="line-6512"> : (new Path(Bytes.toString(family), storeFile)).toString())</span> |
| <span class="source-line-no">6513</span><span id="line-6513"> + " doesn't exist any more. Skip loading the file");</span> |
| <span class="source-line-no">6514</span><span id="line-6514"> }</span> |
| <span class="source-line-no">6515</span><span id="line-6515"> }</span> |
| <span class="source-line-no">6516</span><span id="line-6516"> }</span> |
| <span class="source-line-no">6517</span><span id="line-6517"> }</span> |
| <span class="source-line-no">6518</span><span id="line-6518"> if (bulkLoadEvent.getBulkloadSeqNum() > 0) {</span> |
| <span class="source-line-no">6519</span><span id="line-6519"> mvcc.advanceTo(bulkLoadEvent.getBulkloadSeqNum());</span> |
| <span class="source-line-no">6520</span><span id="line-6520"> }</span> |
| <span class="source-line-no">6521</span><span id="line-6521"> } finally {</span> |
| <span class="source-line-no">6522</span><span id="line-6522"> closeBulkRegionOperation();</span> |
| <span class="source-line-no">6523</span><span id="line-6523"> }</span> |
| <span class="source-line-no">6524</span><span id="line-6524"> }</span> |
| <span class="source-line-no">6525</span><span id="line-6525"></span> |
| <span class="source-line-no">6526</span><span id="line-6526"> /**</span> |
| <span class="source-line-no">6527</span><span id="line-6527"> * Replay the batch mutate for secondary replica.</span> |
| <span class="source-line-no">6528</span><span id="line-6528"> * <p/></span> |
| <span class="source-line-no">6529</span><span id="line-6529"> * We will directly apply the cells to the memstore. This is because:</span> |
| <span class="source-line-no">6530</span><span id="line-6530"> * <ol></span> |
| <span class="source-line-no">6531</span><span id="line-6531"> * <li>All the cells are gotten from {@link WALEdit}, so we only have {@link Put} and</span> |
| <span class="source-line-no">6532</span><span id="line-6532"> * {@link Delete} here</li></span> |
| <span class="source-line-no">6533</span><span id="line-6533"> * <li>The replay is single threaded, we do not need to acquire row lock, as the region is read</span> |
| <span class="source-line-no">6534</span><span id="line-6534"> * only so no one else can write it.</li></span> |
| <span class="source-line-no">6535</span><span id="line-6535"> * <li>We do not need to write WAL.</li></span> |
| <span class="source-line-no">6536</span><span id="line-6536"> * <li>We will advance MVCC in the caller directly.</li></span> |
| <span class="source-line-no">6537</span><span id="line-6537"> * </ol></span> |
| <span class="source-line-no">6538</span><span id="line-6538"> */</span> |
| <span class="source-line-no">6539</span><span id="line-6539"> private void replayWALBatchMutate(Map<byte[], List<ExtendedCell>> family2Cells)</span> |
| <span class="source-line-no">6540</span><span id="line-6540"> throws IOException {</span> |
| <span class="source-line-no">6541</span><span id="line-6541"> startRegionOperation(Operation.REPLAY_BATCH_MUTATE);</span> |
| <span class="source-line-no">6542</span><span id="line-6542"> try {</span> |
| <span class="source-line-no">6543</span><span id="line-6543"> for (Map.Entry<byte[], List<ExtendedCell>> entry : family2Cells.entrySet()) {</span> |
| <span class="source-line-no">6544</span><span id="line-6544"> applyToMemStore(getStore(entry.getKey()), entry.getValue(), false, memStoreSizing);</span> |
| <span class="source-line-no">6545</span><span id="line-6545"> }</span> |
| <span class="source-line-no">6546</span><span id="line-6546"> } finally {</span> |
| <span class="source-line-no">6547</span><span id="line-6547"> closeRegionOperation(Operation.REPLAY_BATCH_MUTATE);</span> |
| <span class="source-line-no">6548</span><span id="line-6548"> }</span> |
| <span class="source-line-no">6549</span><span id="line-6549"> }</span> |
| <span class="source-line-no">6550</span><span id="line-6550"></span> |
| <span class="source-line-no">6551</span><span id="line-6551"> /**</span> |
| <span class="source-line-no">6552</span><span id="line-6552"> * Replay the meta edits, i.e, flush marker, compaction marker, bulk load marker, region event</span> |
| <span class="source-line-no">6553</span><span id="line-6553"> * marker, etc.</span> |
| <span class="source-line-no">6554</span><span id="line-6554"> * <p/></span> |
| <span class="source-line-no">6555</span><span id="line-6555"> * For all events other than start flush, we will just call {@link #refreshStoreFiles()} as the</span> |
| <span class="source-line-no">6556</span><span id="line-6556"> * logic is straight-forward and robust. For start flush, we need to snapshot the memstore, so</span> |
| <span class="source-line-no">6557</span><span id="line-6557"> * later {@link #refreshStoreFiles()} call could drop the snapshot, otherwise we may run out of</span> |
| <span class="source-line-no">6558</span><span id="line-6558"> * memory.</span> |
| <span class="source-line-no">6559</span><span id="line-6559"> */</span> |
| <span class="source-line-no">6560</span><span id="line-6560"> private void replayWALMetaEdit(Cell cell) throws IOException {</span> |
| <span class="source-line-no">6561</span><span id="line-6561"> startRegionOperation(Operation.REPLAY_EVENT);</span> |
| <span class="source-line-no">6562</span><span id="line-6562"> try {</span> |
| <span class="source-line-no">6563</span><span id="line-6563"> FlushDescriptor flushDesc = WALEdit.getFlushDescriptor(cell);</span> |
| <span class="source-line-no">6564</span><span id="line-6564"> if (flushDesc != null) {</span> |
| <span class="source-line-no">6565</span><span id="line-6565"> switch (flushDesc.getAction()) {</span> |
| <span class="source-line-no">6566</span><span id="line-6566"> case START_FLUSH:</span> |
| <span class="source-line-no">6567</span><span id="line-6567"> // for start flush, we need to take a snapshot of the current memstore</span> |
| <span class="source-line-no">6568</span><span id="line-6568"> synchronized (writestate) {</span> |
| <span class="source-line-no">6569</span><span id="line-6569"> if (!writestate.flushing) {</span> |
| <span class="source-line-no">6570</span><span id="line-6570"> this.writestate.flushing = true;</span> |
| <span class="source-line-no">6571</span><span id="line-6571"> } else {</span> |
| <span class="source-line-no">6572</span><span id="line-6572"> // usually this should not happen but let's make the code more robust, it is not a</span> |
| <span class="source-line-no">6573</span><span id="line-6573"> // big deal to just ignore it, the refreshStoreFiles call should have the ability to</span> |
| <span class="source-line-no">6574</span><span id="line-6574"> // clean up the inconsistent state.</span> |
| <span class="source-line-no">6575</span><span id="line-6575"> LOG.debug("NOT flushing {} as already flushing", getRegionInfo());</span> |
| <span class="source-line-no">6576</span><span id="line-6576"> break;</span> |
| <span class="source-line-no">6577</span><span id="line-6577"> }</span> |
| <span class="source-line-no">6578</span><span id="line-6578"> }</span> |
| <span class="source-line-no">6579</span><span id="line-6579"> MonitoredTask status =</span> |
| <span class="source-line-no">6580</span><span id="line-6580"> TaskMonitor.get().createStatus("Preparing flush " + getRegionInfo());</span> |
| <span class="source-line-no">6581</span><span id="line-6581"> Collection<HStore> storesToFlush = getStoresToFlush(flushDesc);</span> |
| <span class="source-line-no">6582</span><span id="line-6582"> try {</span> |
| <span class="source-line-no">6583</span><span id="line-6583"> PrepareFlushResult prepareResult =</span> |
| <span class="source-line-no">6584</span><span id="line-6584"> internalPrepareFlushCache(null, flushDesc.getFlushSequenceNumber(), storesToFlush,</span> |
| <span class="source-line-no">6585</span><span id="line-6585"> status, false, FlushLifeCycleTracker.DUMMY);</span> |
| <span class="source-line-no">6586</span><span id="line-6586"> if (prepareResult.result == null) {</span> |
| <span class="source-line-no">6587</span><span id="line-6587"> // save the PrepareFlushResult so that we can use it later from commit flush</span> |
| <span class="source-line-no">6588</span><span id="line-6588"> this.prepareFlushResult = prepareResult;</span> |
| <span class="source-line-no">6589</span><span id="line-6589"> status.markComplete("Flush prepare successful");</span> |
| <span class="source-line-no">6590</span><span id="line-6590"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">6591</span><span id="line-6591"> LOG.debug("{} prepared flush with seqId: {}", getRegionInfo(),</span> |
| <span class="source-line-no">6592</span><span id="line-6592"> flushDesc.getFlushSequenceNumber());</span> |
| <span class="source-line-no">6593</span><span id="line-6593"> }</span> |
| <span class="source-line-no">6594</span><span id="line-6594"> } else {</span> |
| <span class="source-line-no">6595</span><span id="line-6595"> // special case empty memstore. We will still save the flush result in this case,</span> |
| <span class="source-line-no">6596</span><span id="line-6596"> // since our memstore is empty, but the primary is still flushing</span> |
| <span class="source-line-no">6597</span><span id="line-6597"> if (</span> |
| <span class="source-line-no">6598</span><span id="line-6598"> prepareResult.getResult().getResult()</span> |
| <span class="source-line-no">6599</span><span id="line-6599"> == FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY</span> |
| <span class="source-line-no">6600</span><span id="line-6600"> ) {</span> |
| <span class="source-line-no">6601</span><span id="line-6601"> this.prepareFlushResult = prepareResult;</span> |
| <span class="source-line-no">6602</span><span id="line-6602"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">6603</span><span id="line-6603"> LOG.debug("{} prepared empty flush with seqId: {}", getRegionInfo(),</span> |
| <span class="source-line-no">6604</span><span id="line-6604"> flushDesc.getFlushSequenceNumber());</span> |
| <span class="source-line-no">6605</span><span id="line-6605"> }</span> |
| <span class="source-line-no">6606</span><span id="line-6606"> }</span> |
| <span class="source-line-no">6607</span><span id="line-6607"> status.abort("Flush prepare failed with " + prepareResult.result);</span> |
| <span class="source-line-no">6608</span><span id="line-6608"> // nothing much to do. prepare flush failed because of some reason.</span> |
| <span class="source-line-no">6609</span><span id="line-6609"> }</span> |
| <span class="source-line-no">6610</span><span id="line-6610"> } finally {</span> |
| <span class="source-line-no">6611</span><span id="line-6611"> status.cleanup();</span> |
| <span class="source-line-no">6612</span><span id="line-6612"> }</span> |
| <span class="source-line-no">6613</span><span id="line-6613"> break;</span> |
| <span class="source-line-no">6614</span><span id="line-6614"> case ABORT_FLUSH:</span> |
| <span class="source-line-no">6615</span><span id="line-6615"> // do nothing, an abort flush means the source region server will crash itself, after</span> |
| <span class="source-line-no">6616</span><span id="line-6616"> // the primary region online, it will send us an open region marker, then we can clean</span> |
| <span class="source-line-no">6617</span><span id="line-6617"> // up the memstore.</span> |
| <span class="source-line-no">6618</span><span id="line-6618"> synchronized (writestate) {</span> |
| <span class="source-line-no">6619</span><span id="line-6619"> writestate.flushing = false;</span> |
| <span class="source-line-no">6620</span><span id="line-6620"> }</span> |
| <span class="source-line-no">6621</span><span id="line-6621"> break;</span> |
| <span class="source-line-no">6622</span><span id="line-6622"> case COMMIT_FLUSH:</span> |
| <span class="source-line-no">6623</span><span id="line-6623"> case CANNOT_FLUSH:</span> |
| <span class="source-line-no">6624</span><span id="line-6624"> // just call refreshStoreFiles</span> |
| <span class="source-line-no">6625</span><span id="line-6625"> refreshStoreFiles();</span> |
| <span class="source-line-no">6626</span><span id="line-6626"> logRegionFiles();</span> |
| <span class="source-line-no">6627</span><span id="line-6627"> synchronized (writestate) {</span> |
| <span class="source-line-no">6628</span><span id="line-6628"> writestate.flushing = false;</span> |
| <span class="source-line-no">6629</span><span id="line-6629"> }</span> |
| <span class="source-line-no">6630</span><span id="line-6630"> break;</span> |
| <span class="source-line-no">6631</span><span id="line-6631"> default:</span> |
| <span class="source-line-no">6632</span><span id="line-6632"> LOG.warn("{} received a flush event with unknown action: {}", getRegionInfo(),</span> |
| <span class="source-line-no">6633</span><span id="line-6633"> TextFormat.shortDebugString(flushDesc));</span> |
| <span class="source-line-no">6634</span><span id="line-6634"> }</span> |
| <span class="source-line-no">6635</span><span id="line-6635"> } else {</span> |
| <span class="source-line-no">6636</span><span id="line-6636"> // for all other region events, we will do a refreshStoreFiles</span> |
| <span class="source-line-no">6637</span><span id="line-6637"> refreshStoreFiles();</span> |
| <span class="source-line-no">6638</span><span id="line-6638"> logRegionFiles();</span> |
| <span class="source-line-no">6639</span><span id="line-6639"> }</span> |
| <span class="source-line-no">6640</span><span id="line-6640"> } finally {</span> |
| <span class="source-line-no">6641</span><span id="line-6641"> closeRegionOperation(Operation.REPLAY_EVENT);</span> |
| <span class="source-line-no">6642</span><span id="line-6642"> }</span> |
| <span class="source-line-no">6643</span><span id="line-6643"> }</span> |
| <span class="source-line-no">6644</span><span id="line-6644"></span> |
| <span class="source-line-no">6645</span><span id="line-6645"> /**</span> |
| <span class="source-line-no">6646</span><span id="line-6646"> * Replay remote wal entry sent by primary replica.</span> |
| <span class="source-line-no">6647</span><span id="line-6647"> * <p/></span> |
| <span class="source-line-no">6648</span><span id="line-6648"> * Should only call this method on secondary replicas.</span> |
| <span class="source-line-no">6649</span><span id="line-6649"> */</span> |
| <span class="source-line-no">6650</span><span id="line-6650"> void replayWALEntry(WALEntry entry, CellScanner cells) throws IOException {</span> |
| <span class="source-line-no">6651</span><span id="line-6651"> long timeout = -1L;</span> |
| <span class="source-line-no">6652</span><span id="line-6652"> Optional<RpcCall> call = RpcServer.getCurrentCall();</span> |
| <span class="source-line-no">6653</span><span id="line-6653"> if (call.isPresent()) {</span> |
| <span class="source-line-no">6654</span><span id="line-6654"> long deadline = call.get().getDeadline();</span> |
| <span class="source-line-no">6655</span><span id="line-6655"> if (deadline < Long.MAX_VALUE) {</span> |
| <span class="source-line-no">6656</span><span id="line-6656"> timeout = deadline - EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">6657</span><span id="line-6657"> if (timeout <= 0) {</span> |
| <span class="source-line-no">6658</span><span id="line-6658"> throw new TimeoutIOException("Timeout while replaying edits for " + getRegionInfo());</span> |
| <span class="source-line-no">6659</span><span id="line-6659"> }</span> |
| <span class="source-line-no">6660</span><span id="line-6660"> }</span> |
| <span class="source-line-no">6661</span><span id="line-6661"> }</span> |
| <span class="source-line-no">6662</span><span id="line-6662"> if (timeout > 0) {</span> |
| <span class="source-line-no">6663</span><span id="line-6663"> try {</span> |
| <span class="source-line-no">6664</span><span id="line-6664"> if (!replayLock.tryLock(timeout, TimeUnit.MILLISECONDS)) {</span> |
| <span class="source-line-no">6665</span><span id="line-6665"> throw new TimeoutIOException(</span> |
| <span class="source-line-no">6666</span><span id="line-6666"> "Timeout while waiting for lock when replaying edits for " + getRegionInfo());</span> |
| <span class="source-line-no">6667</span><span id="line-6667"> }</span> |
| <span class="source-line-no">6668</span><span id="line-6668"> } catch (InterruptedException e) {</span> |
| <span class="source-line-no">6669</span><span id="line-6669"> throw throwOnInterrupt(e);</span> |
| <span class="source-line-no">6670</span><span id="line-6670"> }</span> |
| <span class="source-line-no">6671</span><span id="line-6671"> } else {</span> |
| <span class="source-line-no">6672</span><span id="line-6672"> replayLock.lock();</span> |
| <span class="source-line-no">6673</span><span id="line-6673"> }</span> |
| <span class="source-line-no">6674</span><span id="line-6674"> try {</span> |
| <span class="source-line-no">6675</span><span id="line-6675"> int count = entry.getAssociatedCellCount();</span> |
| <span class="source-line-no">6676</span><span id="line-6676"> long sequenceId = entry.getKey().getLogSequenceNumber();</span> |
| <span class="source-line-no">6677</span><span id="line-6677"> if (lastReplayedSequenceId >= sequenceId) {</span> |
| <span class="source-line-no">6678</span><span id="line-6678"> // we have already replayed this edit, skip</span> |
| <span class="source-line-no">6679</span><span id="line-6679"> // remember to advance the CellScanner, as we may have multiple WALEntries, we may still</span> |
| <span class="source-line-no">6680</span><span id="line-6680"> // need apply later WALEntries</span> |
| <span class="source-line-no">6681</span><span id="line-6681"> for (int i = 0; i < count; i++) {</span> |
| <span class="source-line-no">6682</span><span id="line-6682"> // Throw index out of bounds if our cell count is off</span> |
| <span class="source-line-no">6683</span><span id="line-6683"> if (!cells.advance()) {</span> |
| <span class="source-line-no">6684</span><span id="line-6684"> throw new ArrayIndexOutOfBoundsException("Expected=" + count + ", index=" + i);</span> |
| <span class="source-line-no">6685</span><span id="line-6685"> }</span> |
| <span class="source-line-no">6686</span><span id="line-6686"> }</span> |
| <span class="source-line-no">6687</span><span id="line-6687"> return;</span> |
| <span class="source-line-no">6688</span><span id="line-6688"> }</span> |
| <span class="source-line-no">6689</span><span id="line-6689"> Map<byte[], List<ExtendedCell>> family2Cells = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">6690</span><span id="line-6690"> for (int i = 0; i < count; i++) {</span> |
| <span class="source-line-no">6691</span><span id="line-6691"> // Throw index out of bounds if our cell count is off</span> |
| <span class="source-line-no">6692</span><span id="line-6692"> if (!cells.advance()) {</span> |
| <span class="source-line-no">6693</span><span id="line-6693"> throw new ArrayIndexOutOfBoundsException("Expected=" + count + ", index=" + i);</span> |
| <span class="source-line-no">6694</span><span id="line-6694"> }</span> |
| <span class="source-line-no">6695</span><span id="line-6695"> Cell c = cells.current();</span> |
| <span class="source-line-no">6696</span><span id="line-6696"> assert c instanceof ExtendedCell;</span> |
| <span class="source-line-no">6697</span><span id="line-6697"> ExtendedCell cell = (ExtendedCell) c;</span> |
| <span class="source-line-no">6698</span><span id="line-6698"> if (WALEdit.isMetaEditFamily(cell)) {</span> |
| <span class="source-line-no">6699</span><span id="line-6699"> // If there is meta edit, i.e, we have done flush/compaction/open, then we need to apply</span> |
| <span class="source-line-no">6700</span><span id="line-6700"> // the previous cells first, and then replay the special meta edit. The meta edit is like</span> |
| <span class="source-line-no">6701</span><span id="line-6701"> // a barrier, We need to keep the order. For example, the flush marker will contain a</span> |
| <span class="source-line-no">6702</span><span id="line-6702"> // flush sequence number, which makes us possible to drop memstore content, but if we</span> |
| <span class="source-line-no">6703</span><span id="line-6703"> // apply some edits which have greater sequence id first, then we can not drop the</span> |
| <span class="source-line-no">6704</span><span id="line-6704"> // memstore content when replaying the flush marker, which is not good as we could run out</span> |
| <span class="source-line-no">6705</span><span id="line-6705"> // of memory.</span> |
| <span class="source-line-no">6706</span><span id="line-6706"> // And usually, a meta edit will have a special WALEntry for it, so this is just a safe</span> |
| <span class="source-line-no">6707</span><span id="line-6707"> // guard logic to make sure we do not break things in the worst case.</span> |
| <span class="source-line-no">6708</span><span id="line-6708"> if (!family2Cells.isEmpty()) {</span> |
| <span class="source-line-no">6709</span><span id="line-6709"> replayWALBatchMutate(family2Cells);</span> |
| <span class="source-line-no">6710</span><span id="line-6710"> family2Cells.clear();</span> |
| <span class="source-line-no">6711</span><span id="line-6711"> }</span> |
| <span class="source-line-no">6712</span><span id="line-6712"> replayWALMetaEdit(cell);</span> |
| <span class="source-line-no">6713</span><span id="line-6713"> } else {</span> |
| <span class="source-line-no">6714</span><span id="line-6714"> family2Cells.computeIfAbsent(CellUtil.cloneFamily(cell), k -> new ArrayList<>())</span> |
| <span class="source-line-no">6715</span><span id="line-6715"> .add(cell);</span> |
| <span class="source-line-no">6716</span><span id="line-6716"> }</span> |
| <span class="source-line-no">6717</span><span id="line-6717"> }</span> |
| <span class="source-line-no">6718</span><span id="line-6718"> // do not forget to apply the remaining cells</span> |
| <span class="source-line-no">6719</span><span id="line-6719"> if (!family2Cells.isEmpty()) {</span> |
| <span class="source-line-no">6720</span><span id="line-6720"> replayWALBatchMutate(family2Cells);</span> |
| <span class="source-line-no">6721</span><span id="line-6721"> }</span> |
| <span class="source-line-no">6722</span><span id="line-6722"> mvcc.advanceTo(sequenceId);</span> |
| <span class="source-line-no">6723</span><span id="line-6723"> lastReplayedSequenceId = sequenceId;</span> |
| <span class="source-line-no">6724</span><span id="line-6724"> } finally {</span> |
| <span class="source-line-no">6725</span><span id="line-6725"> replayLock.unlock();</span> |
| <span class="source-line-no">6726</span><span id="line-6726"> }</span> |
| <span class="source-line-no">6727</span><span id="line-6727"> }</span> |
| <span class="source-line-no">6728</span><span id="line-6728"></span> |
| <span class="source-line-no">6729</span><span id="line-6729"> /**</span> |
| <span class="source-line-no">6730</span><span id="line-6730"> * If all stores ended up dropping their snapshots, we can safely drop the prepareFlushResult</span> |
| <span class="source-line-no">6731</span><span id="line-6731"> */</span> |
| <span class="source-line-no">6732</span><span id="line-6732"> private void dropPrepareFlushIfPossible() {</span> |
| <span class="source-line-no">6733</span><span id="line-6733"> if (writestate.flushing) {</span> |
| <span class="source-line-no">6734</span><span id="line-6734"> boolean canDrop = true;</span> |
| <span class="source-line-no">6735</span><span id="line-6735"> if (prepareFlushResult.storeFlushCtxs != null) {</span> |
| <span class="source-line-no">6736</span><span id="line-6736"> for (Entry<byte[], StoreFlushContext> entry : prepareFlushResult.storeFlushCtxs</span> |
| <span class="source-line-no">6737</span><span id="line-6737"> .entrySet()) {</span> |
| <span class="source-line-no">6738</span><span id="line-6738"> HStore store = getStore(entry.getKey());</span> |
| <span class="source-line-no">6739</span><span id="line-6739"> if (store == null) {</span> |
| <span class="source-line-no">6740</span><span id="line-6740"> continue;</span> |
| <span class="source-line-no">6741</span><span id="line-6741"> }</span> |
| <span class="source-line-no">6742</span><span id="line-6742"> if (store.getSnapshotSize().getDataSize() > 0) {</span> |
| <span class="source-line-no">6743</span><span id="line-6743"> canDrop = false;</span> |
| <span class="source-line-no">6744</span><span id="line-6744"> break;</span> |
| <span class="source-line-no">6745</span><span id="line-6745"> }</span> |
| <span class="source-line-no">6746</span><span id="line-6746"> }</span> |
| <span class="source-line-no">6747</span><span id="line-6747"> }</span> |
| <span class="source-line-no">6748</span><span id="line-6748"></span> |
| <span class="source-line-no">6749</span><span id="line-6749"> // this means that all the stores in the region has finished flushing, but the WAL marker</span> |
| <span class="source-line-no">6750</span><span id="line-6750"> // may not have been written or we did not receive it yet.</span> |
| <span class="source-line-no">6751</span><span id="line-6751"> if (canDrop) {</span> |
| <span class="source-line-no">6752</span><span id="line-6752"> writestate.flushing = false;</span> |
| <span class="source-line-no">6753</span><span id="line-6753"> this.prepareFlushResult = null;</span> |
| <span class="source-line-no">6754</span><span id="line-6754"> }</span> |
| <span class="source-line-no">6755</span><span id="line-6755"> }</span> |
| <span class="source-line-no">6756</span><span id="line-6756"> }</span> |
| <span class="source-line-no">6757</span><span id="line-6757"></span> |
| <span class="source-line-no">6758</span><span id="line-6758"> @Override</span> |
| <span class="source-line-no">6759</span><span id="line-6759"> public boolean refreshStoreFiles() throws IOException {</span> |
| <span class="source-line-no">6760</span><span id="line-6760"> return refreshStoreFiles(false);</span> |
| <span class="source-line-no">6761</span><span id="line-6761"> }</span> |
| <span class="source-line-no">6762</span><span id="line-6762"></span> |
| <span class="source-line-no">6763</span><span id="line-6763"> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NN_NAKED_NOTIFY",</span> |
| <span class="source-line-no">6764</span><span id="line-6764"> justification = "Notify is about post replay. Intentional")</span> |
| <span class="source-line-no">6765</span><span id="line-6765"> protected boolean refreshStoreFiles(boolean force) throws IOException {</span> |
| <span class="source-line-no">6766</span><span id="line-6766"> if (!force && ServerRegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {</span> |
| <span class="source-line-no">6767</span><span id="line-6767"> return false; // if primary nothing to do</span> |
| <span class="source-line-no">6768</span><span id="line-6768"> }</span> |
| <span class="source-line-no">6769</span><span id="line-6769"></span> |
| <span class="source-line-no">6770</span><span id="line-6770"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">6771</span><span id="line-6771"> LOG.debug(getRegionInfo().getEncodedName() + " : "</span> |
| <span class="source-line-no">6772</span><span id="line-6772"> + "Refreshing store files to see whether we can free up memstore");</span> |
| <span class="source-line-no">6773</span><span id="line-6773"> }</span> |
| <span class="source-line-no">6774</span><span id="line-6774"></span> |
| <span class="source-line-no">6775</span><span id="line-6775"> long totalFreedDataSize = 0;</span> |
| <span class="source-line-no">6776</span><span id="line-6776"></span> |
| <span class="source-line-no">6777</span><span id="line-6777"> long smallestSeqIdInStores = Long.MAX_VALUE;</span> |
| <span class="source-line-no">6778</span><span id="line-6778"></span> |
| <span class="source-line-no">6779</span><span id="line-6779"> startRegionOperation(); // obtain region close lock</span> |
| <span class="source-line-no">6780</span><span id="line-6780"> try {</span> |
| <span class="source-line-no">6781</span><span id="line-6781"> Map<HStore, Long> map = new HashMap<>();</span> |
| <span class="source-line-no">6782</span><span id="line-6782"> synchronized (writestate) {</span> |
| <span class="source-line-no">6783</span><span id="line-6783"> for (HStore store : stores.values()) {</span> |
| <span class="source-line-no">6784</span><span id="line-6784"> // TODO: some stores might see new data from flush, while others do not which</span> |
| <span class="source-line-no">6785</span><span id="line-6785"> // MIGHT break atomic edits across column families.</span> |
| <span class="source-line-no">6786</span><span id="line-6786"> long maxSeqIdBefore = store.getMaxSequenceId().orElse(0L);</span> |
| <span class="source-line-no">6787</span><span id="line-6787"></span> |
| <span class="source-line-no">6788</span><span id="line-6788"> // refresh the store files. This is similar to observing a region open wal marker.</span> |
| <span class="source-line-no">6789</span><span id="line-6789"> store.refreshStoreFiles();</span> |
| <span class="source-line-no">6790</span><span id="line-6790"></span> |
| <span class="source-line-no">6791</span><span id="line-6791"> long storeSeqId = store.getMaxSequenceId().orElse(0L);</span> |
| <span class="source-line-no">6792</span><span id="line-6792"> if (storeSeqId < smallestSeqIdInStores) {</span> |
| <span class="source-line-no">6793</span><span id="line-6793"> smallestSeqIdInStores = storeSeqId;</span> |
| <span class="source-line-no">6794</span><span id="line-6794"> }</span> |
| <span class="source-line-no">6795</span><span id="line-6795"></span> |
| <span class="source-line-no">6796</span><span id="line-6796"> // see whether we can drop the memstore or the snapshot</span> |
| <span class="source-line-no">6797</span><span id="line-6797"> if (storeSeqId > maxSeqIdBefore) {</span> |
| <span class="source-line-no">6798</span><span id="line-6798"> if (writestate.flushing) {</span> |
| <span class="source-line-no">6799</span><span id="line-6799"> // only drop memstore snapshots if they are smaller than last flush for the store</span> |
| <span class="source-line-no">6800</span><span id="line-6800"> if (this.prepareFlushResult.flushOpSeqId <= storeSeqId) {</span> |
| <span class="source-line-no">6801</span><span id="line-6801"> StoreFlushContext ctx = this.prepareFlushResult.storeFlushCtxs == null</span> |
| <span class="source-line-no">6802</span><span id="line-6802"> ? null</span> |
| <span class="source-line-no">6803</span><span id="line-6803"> : this.prepareFlushResult.storeFlushCtxs</span> |
| <span class="source-line-no">6804</span><span id="line-6804"> .get(store.getColumnFamilyDescriptor().getName());</span> |
| <span class="source-line-no">6805</span><span id="line-6805"> if (ctx != null) {</span> |
| <span class="source-line-no">6806</span><span id="line-6806"> MemStoreSize mss = store.getFlushableSize();</span> |
| <span class="source-line-no">6807</span><span id="line-6807"> ctx.abort();</span> |
| <span class="source-line-no">6808</span><span id="line-6808"> this.decrMemStoreSize(mss);</span> |
| <span class="source-line-no">6809</span><span id="line-6809"> this.prepareFlushResult.storeFlushCtxs</span> |
| <span class="source-line-no">6810</span><span id="line-6810"> .remove(store.getColumnFamilyDescriptor().getName());</span> |
| <span class="source-line-no">6811</span><span id="line-6811"> totalFreedDataSize += mss.getDataSize();</span> |
| <span class="source-line-no">6812</span><span id="line-6812"> }</span> |
| <span class="source-line-no">6813</span><span id="line-6813"> }</span> |
| <span class="source-line-no">6814</span><span id="line-6814"> }</span> |
| <span class="source-line-no">6815</span><span id="line-6815"></span> |
| <span class="source-line-no">6816</span><span id="line-6816"> map.put(store, storeSeqId);</span> |
| <span class="source-line-no">6817</span><span id="line-6817"> }</span> |
| <span class="source-line-no">6818</span><span id="line-6818"> }</span> |
| <span class="source-line-no">6819</span><span id="line-6819"></span> |
| <span class="source-line-no">6820</span><span id="line-6820"> // if all stores ended up dropping their snapshots, we can safely drop the</span> |
| <span class="source-line-no">6821</span><span id="line-6821"> // prepareFlushResult</span> |
| <span class="source-line-no">6822</span><span id="line-6822"> dropPrepareFlushIfPossible();</span> |
| <span class="source-line-no">6823</span><span id="line-6823"></span> |
| <span class="source-line-no">6824</span><span id="line-6824"> // advance the mvcc read point so that the new flushed files are visible.</span> |
| <span class="source-line-no">6825</span><span id="line-6825"> // either greater than flush seq number or they were already picked up via flush.</span> |
| <span class="source-line-no">6826</span><span id="line-6826"> for (HStore s : stores.values()) {</span> |
| <span class="source-line-no">6827</span><span id="line-6827"> mvcc.advanceTo(s.getMaxMemStoreTS().orElse(0L));</span> |
| <span class="source-line-no">6828</span><span id="line-6828"> }</span> |
| <span class="source-line-no">6829</span><span id="line-6829"></span> |
| <span class="source-line-no">6830</span><span id="line-6830"> // smallestSeqIdInStores is the seqId that we have a corresponding hfile for. We can safely</span> |
| <span class="source-line-no">6831</span><span id="line-6831"> // skip all edits that are to be replayed in the future with that has a smaller seqId</span> |
| <span class="source-line-no">6832</span><span id="line-6832"> // than this. We are updating lastReplayedOpenRegionSeqId so that we can skip all edits</span> |
| <span class="source-line-no">6833</span><span id="line-6833"> // that we have picked the flush files for</span> |
| <span class="source-line-no">6834</span><span id="line-6834"> if (this.lastReplayedOpenRegionSeqId < smallestSeqIdInStores) {</span> |
| <span class="source-line-no">6835</span><span id="line-6835"> this.lastReplayedOpenRegionSeqId = smallestSeqIdInStores;</span> |
| <span class="source-line-no">6836</span><span id="line-6836"> }</span> |
| <span class="source-line-no">6837</span><span id="line-6837"> }</span> |
| <span class="source-line-no">6838</span><span id="line-6838"> if (!map.isEmpty()) {</span> |
| <span class="source-line-no">6839</span><span id="line-6839"> for (Map.Entry<HStore, Long> entry : map.entrySet()) {</span> |
| <span class="source-line-no">6840</span><span id="line-6840"> // Drop the memstore contents if they are now smaller than the latest seen flushed file</span> |
| <span class="source-line-no">6841</span><span id="line-6841"> totalFreedDataSize +=</span> |
| <span class="source-line-no">6842</span><span id="line-6842"> dropMemStoreContentsForSeqId(entry.getValue(), entry.getKey()).getDataSize();</span> |
| <span class="source-line-no">6843</span><span id="line-6843"> }</span> |
| <span class="source-line-no">6844</span><span id="line-6844"> }</span> |
| <span class="source-line-no">6845</span><span id="line-6845"> // C. Finally notify anyone waiting on memstore to clear:</span> |
| <span class="source-line-no">6846</span><span id="line-6846"> // e.g. checkResources().</span> |
| <span class="source-line-no">6847</span><span id="line-6847"> synchronized (this) {</span> |
| <span class="source-line-no">6848</span><span id="line-6848"> notifyAll(); // FindBugs NN_NAKED_NOTIFY</span> |
| <span class="source-line-no">6849</span><span id="line-6849"> }</span> |
| <span class="source-line-no">6850</span><span id="line-6850"> return totalFreedDataSize > 0;</span> |
| <span class="source-line-no">6851</span><span id="line-6851"> } finally {</span> |
| <span class="source-line-no">6852</span><span id="line-6852"> closeRegionOperation();</span> |
| <span class="source-line-no">6853</span><span id="line-6853"> }</span> |
| <span class="source-line-no">6854</span><span id="line-6854"> }</span> |
| <span class="source-line-no">6855</span><span id="line-6855"></span> |
| <span class="source-line-no">6856</span><span id="line-6856"> private void logRegionFiles() {</span> |
| <span class="source-line-no">6857</span><span id="line-6857"> if (LOG.isTraceEnabled()) {</span> |
| <span class="source-line-no">6858</span><span id="line-6858"> LOG.trace(getRegionInfo().getEncodedName() + " : Store files for region: ");</span> |
| <span class="source-line-no">6859</span><span id="line-6859"> stores.values().stream().filter(s -> s.getStorefiles() != null)</span> |
| <span class="source-line-no">6860</span><span id="line-6860"> .flatMap(s -> s.getStorefiles().stream())</span> |
| <span class="source-line-no">6861</span><span id="line-6861"> .forEachOrdered(sf -> LOG.trace(getRegionInfo().getEncodedName() + " : " + sf));</span> |
| <span class="source-line-no">6862</span><span id="line-6862"> }</span> |
| <span class="source-line-no">6863</span><span id="line-6863"> }</span> |
| <span class="source-line-no">6864</span><span id="line-6864"></span> |
| <span class="source-line-no">6865</span><span id="line-6865"> /**</span> |
| <span class="source-line-no">6866</span><span id="line-6866"> * Checks whether the given regionName is either equal to our region, or that the regionName is</span> |
| <span class="source-line-no">6867</span><span id="line-6867"> * the primary region to our corresponding range for the secondary replica.</span> |
| <span class="source-line-no">6868</span><span id="line-6868"> */</span> |
| <span class="source-line-no">6869</span><span id="line-6869"> private void checkTargetRegion(byte[] encodedRegionName, String exceptionMsg, Object payload)</span> |
| <span class="source-line-no">6870</span><span id="line-6870"> throws WrongRegionException {</span> |
| <span class="source-line-no">6871</span><span id="line-6871"> if (Bytes.equals(this.getRegionInfo().getEncodedNameAsBytes(), encodedRegionName)) {</span> |
| <span class="source-line-no">6872</span><span id="line-6872"> return;</span> |
| <span class="source-line-no">6873</span><span id="line-6873"> }</span> |
| <span class="source-line-no">6874</span><span id="line-6874"></span> |
| <span class="source-line-no">6875</span><span id="line-6875"> if (</span> |
| <span class="source-line-no">6876</span><span id="line-6876"> !RegionReplicaUtil.isDefaultReplica(this.getRegionInfo())</span> |
| <span class="source-line-no">6877</span><span id="line-6877"> && Bytes.equals(encodedRegionName, this.fs.getRegionInfoForFS().getEncodedNameAsBytes())</span> |
| <span class="source-line-no">6878</span><span id="line-6878"> ) {</span> |
| <span class="source-line-no">6879</span><span id="line-6879"> return;</span> |
| <span class="source-line-no">6880</span><span id="line-6880"> }</span> |
| <span class="source-line-no">6881</span><span id="line-6881"></span> |
| <span class="source-line-no">6882</span><span id="line-6882"> throw new WrongRegionException(</span> |
| <span class="source-line-no">6883</span><span id="line-6883"> exceptionMsg + payload + " targetted for region " + Bytes.toStringBinary(encodedRegionName)</span> |
| <span class="source-line-no">6884</span><span id="line-6884"> + " does not match this region: " + this.getRegionInfo());</span> |
| <span class="source-line-no">6885</span><span id="line-6885"> }</span> |
| <span class="source-line-no">6886</span><span id="line-6886"></span> |
| <span class="source-line-no">6887</span><span id="line-6887"> /**</span> |
| <span class="source-line-no">6888</span><span id="line-6888"> * Used by tests</span> |
| <span class="source-line-no">6889</span><span id="line-6889"> * @param s Store to add edit too.</span> |
| <span class="source-line-no">6890</span><span id="line-6890"> * @param cell Cell to add.</span> |
| <span class="source-line-no">6891</span><span id="line-6891"> */</span> |
| <span class="source-line-no">6892</span><span id="line-6892"> protected void restoreEdit(HStore s, ExtendedCell cell, MemStoreSizing memstoreAccounting) {</span> |
| <span class="source-line-no">6893</span><span id="line-6893"> s.add(cell, memstoreAccounting);</span> |
| <span class="source-line-no">6894</span><span id="line-6894"> }</span> |
| <span class="source-line-no">6895</span><span id="line-6895"></span> |
| <span class="source-line-no">6896</span><span id="line-6896"> /**</span> |
| <span class="source-line-no">6897</span><span id="line-6897"> * make sure have been through lease recovery before get file status, so the file length can be</span> |
| <span class="source-line-no">6898</span><span id="line-6898"> * trusted.</span> |
| <span class="source-line-no">6899</span><span id="line-6899"> * @param p File to check.</span> |
| <span class="source-line-no">6900</span><span id="line-6900"> * @return True if file was zero-length (and if so, we'll delete it in here).</span> |
| <span class="source-line-no">6901</span><span id="line-6901"> */</span> |
| <span class="source-line-no">6902</span><span id="line-6902"> private static boolean isZeroLengthThenDelete(final FileSystem fs, final FileStatus stat,</span> |
| <span class="source-line-no">6903</span><span id="line-6903"> final Path p) throws IOException {</span> |
| <span class="source-line-no">6904</span><span id="line-6904"> if (stat.getLen() > 0) {</span> |
| <span class="source-line-no">6905</span><span id="line-6905"> return false;</span> |
| <span class="source-line-no">6906</span><span id="line-6906"> }</span> |
| <span class="source-line-no">6907</span><span id="line-6907"> LOG.warn("File " + p + " is zero-length, deleting.");</span> |
| <span class="source-line-no">6908</span><span id="line-6908"> fs.delete(p, false);</span> |
| <span class="source-line-no">6909</span><span id="line-6909"> return true;</span> |
| <span class="source-line-no">6910</span><span id="line-6910"> }</span> |
| <span class="source-line-no">6911</span><span id="line-6911"></span> |
| <span class="source-line-no">6912</span><span id="line-6912"> protected HStore instantiateHStore(final ColumnFamilyDescriptor family, boolean warmup)</span> |
| <span class="source-line-no">6913</span><span id="line-6913"> throws IOException {</span> |
| <span class="source-line-no">6914</span><span id="line-6914"> if (family.isMobEnabled()) {</span> |
| <span class="source-line-no">6915</span><span id="line-6915"> if (HFile.getFormatVersion(this.conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) {</span> |
| <span class="source-line-no">6916</span><span id="line-6916"> throw new IOException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS</span> |
| <span class="source-line-no">6917</span><span id="line-6917"> + " is required for MOB feature. Consider setting " + HFile.FORMAT_VERSION_KEY</span> |
| <span class="source-line-no">6918</span><span id="line-6918"> + " accordingly.");</span> |
| <span class="source-line-no">6919</span><span id="line-6919"> }</span> |
| <span class="source-line-no">6920</span><span id="line-6920"> return new HMobStore(this, family, this.conf, warmup);</span> |
| <span class="source-line-no">6921</span><span id="line-6921"> }</span> |
| <span class="source-line-no">6922</span><span id="line-6922"> return new HStore(this, family, this.conf, warmup);</span> |
| <span class="source-line-no">6923</span><span id="line-6923"> }</span> |
| <span class="source-line-no">6924</span><span id="line-6924"></span> |
| <span class="source-line-no">6925</span><span id="line-6925"> @Override</span> |
| <span class="source-line-no">6926</span><span id="line-6926"> public HStore getStore(byte[] column) {</span> |
| <span class="source-line-no">6927</span><span id="line-6927"> return this.stores.get(column);</span> |
| <span class="source-line-no">6928</span><span id="line-6928"> }</span> |
| <span class="source-line-no">6929</span><span id="line-6929"></span> |
| <span class="source-line-no">6930</span><span id="line-6930"> /**</span> |
| <span class="source-line-no">6931</span><span id="line-6931"> * Return HStore instance. Does not do any copy: as the number of store is limited, we iterate on</span> |
| <span class="source-line-no">6932</span><span id="line-6932"> * the list.</span> |
| <span class="source-line-no">6933</span><span id="line-6933"> */</span> |
| <span class="source-line-no">6934</span><span id="line-6934"> private HStore getStore(Cell cell) {</span> |
| <span class="source-line-no">6935</span><span id="line-6935"> return stores.entrySet().stream().filter(e -> CellUtil.matchingFamily(cell, e.getKey()))</span> |
| <span class="source-line-no">6936</span><span id="line-6936"> .map(e -> e.getValue()).findFirst().orElse(null);</span> |
| <span class="source-line-no">6937</span><span id="line-6937"> }</span> |
| <span class="source-line-no">6938</span><span id="line-6938"></span> |
| <span class="source-line-no">6939</span><span id="line-6939"> @Override</span> |
| <span class="source-line-no">6940</span><span id="line-6940"> public List<HStore> getStores() {</span> |
| <span class="source-line-no">6941</span><span id="line-6941"> return new ArrayList<>(stores.values());</span> |
| <span class="source-line-no">6942</span><span id="line-6942"> }</span> |
| <span class="source-line-no">6943</span><span id="line-6943"></span> |
| <span class="source-line-no">6944</span><span id="line-6944"> @Override</span> |
| <span class="source-line-no">6945</span><span id="line-6945"> public List<String> getStoreFileList(byte[][] columns) throws IllegalArgumentException {</span> |
| <span class="source-line-no">6946</span><span id="line-6946"> List<String> storeFileNames = new ArrayList<>();</span> |
| <span class="source-line-no">6947</span><span id="line-6947"> synchronized (closeLock) {</span> |
| <span class="source-line-no">6948</span><span id="line-6948"> for (byte[] column : columns) {</span> |
| <span class="source-line-no">6949</span><span id="line-6949"> HStore store = this.stores.get(column);</span> |
| <span class="source-line-no">6950</span><span id="line-6950"> if (store == null) {</span> |
| <span class="source-line-no">6951</span><span id="line-6951"> throw new IllegalArgumentException(</span> |
| <span class="source-line-no">6952</span><span id="line-6952"> "No column family : " + new String(column, StandardCharsets.UTF_8) + " available");</span> |
| <span class="source-line-no">6953</span><span id="line-6953"> }</span> |
| <span class="source-line-no">6954</span><span id="line-6954"> Collection<HStoreFile> storeFiles = store.getStorefiles();</span> |
| <span class="source-line-no">6955</span><span id="line-6955"> if (storeFiles == null) {</span> |
| <span class="source-line-no">6956</span><span id="line-6956"> continue;</span> |
| <span class="source-line-no">6957</span><span id="line-6957"> }</span> |
| <span class="source-line-no">6958</span><span id="line-6958"> for (HStoreFile storeFile : storeFiles) {</span> |
| <span class="source-line-no">6959</span><span id="line-6959"> storeFileNames.add(storeFile.getPath().toString());</span> |
| <span class="source-line-no">6960</span><span id="line-6960"> }</span> |
| <span class="source-line-no">6961</span><span id="line-6961"></span> |
| <span class="source-line-no">6962</span><span id="line-6962"> logRegionFiles();</span> |
| <span class="source-line-no">6963</span><span id="line-6963"> }</span> |
| <span class="source-line-no">6964</span><span id="line-6964"> }</span> |
| <span class="source-line-no">6965</span><span id="line-6965"> return storeFileNames;</span> |
| <span class="source-line-no">6966</span><span id="line-6966"> }</span> |
| <span class="source-line-no">6967</span><span id="line-6967"></span> |
| <span class="source-line-no">6968</span><span id="line-6968"> //////////////////////////////////////////////////////////////////////////////</span> |
| <span class="source-line-no">6969</span><span id="line-6969"> // Support code</span> |
| <span class="source-line-no">6970</span><span id="line-6970"> //////////////////////////////////////////////////////////////////////////////</span> |
| <span class="source-line-no">6971</span><span id="line-6971"></span> |
| <span class="source-line-no">6972</span><span id="line-6972"> /** Make sure this is a valid row for the HRegion */</span> |
| <span class="source-line-no">6973</span><span id="line-6973"> void checkRow(byte[] row, String op) throws IOException {</span> |
| <span class="source-line-no">6974</span><span id="line-6974"> if (!rowIsInRange(getRegionInfo(), row)) {</span> |
| <span class="source-line-no">6975</span><span id="line-6975"> throw new WrongRegionException("Requested row out of range for " + op + " on HRegion " + this</span> |
| <span class="source-line-no">6976</span><span id="line-6976"> + ", startKey='" + Bytes.toStringBinary(getRegionInfo().getStartKey()) + "', getEndKey()='"</span> |
| <span class="source-line-no">6977</span><span id="line-6977"> + Bytes.toStringBinary(getRegionInfo().getEndKey()) + "', row='" + Bytes.toStringBinary(row)</span> |
| <span class="source-line-no">6978</span><span id="line-6978"> + "'");</span> |
| <span class="source-line-no">6979</span><span id="line-6979"> }</span> |
| <span class="source-line-no">6980</span><span id="line-6980"> }</span> |
| <span class="source-line-no">6981</span><span id="line-6981"></span> |
| <span class="source-line-no">6982</span><span id="line-6982"> /**</span> |
| <span class="source-line-no">6983</span><span id="line-6983"> * Get an exclusive ( write lock ) lock on a given row.</span> |
| <span class="source-line-no">6984</span><span id="line-6984"> * @param row Which row to lock.</span> |
| <span class="source-line-no">6985</span><span id="line-6985"> * @return A locked RowLock. The lock is exclusive and already aqquired.</span> |
| <span class="source-line-no">6986</span><span id="line-6986"> */</span> |
| <span class="source-line-no">6987</span><span id="line-6987"> public RowLock getRowLock(byte[] row) throws IOException {</span> |
| <span class="source-line-no">6988</span><span id="line-6988"> return getRowLock(row, false);</span> |
| <span class="source-line-no">6989</span><span id="line-6989"> }</span> |
| <span class="source-line-no">6990</span><span id="line-6990"></span> |
| <span class="source-line-no">6991</span><span id="line-6991"> @Override</span> |
| <span class="source-line-no">6992</span><span id="line-6992"> public RowLock getRowLock(byte[] row, boolean readLock) throws IOException {</span> |
| <span class="source-line-no">6993</span><span id="line-6993"> checkRow(row, "row lock");</span> |
| <span class="source-line-no">6994</span><span id="line-6994"> return getRowLock(row, readLock, null);</span> |
| <span class="source-line-no">6995</span><span id="line-6995"> }</span> |
| <span class="source-line-no">6996</span><span id="line-6996"></span> |
| <span class="source-line-no">6997</span><span id="line-6997"> Span createRegionSpan(String name) {</span> |
| <span class="source-line-no">6998</span><span id="line-6998"> return TraceUtil.createSpan(name).setAttribute(REGION_NAMES_KEY,</span> |
| <span class="source-line-no">6999</span><span id="line-6999"> Collections.singletonList(getRegionInfo().getRegionNameAsString()));</span> |
| <span class="source-line-no">7000</span><span id="line-7000"> }</span> |
| <span class="source-line-no">7001</span><span id="line-7001"></span> |
| <span class="source-line-no">7002</span><span id="line-7002"> // will be override in tests</span> |
| <span class="source-line-no">7003</span><span id="line-7003"> protected RowLock getRowLockInternal(byte[] row, boolean readLock, RowLock prevRowLock)</span> |
| <span class="source-line-no">7004</span><span id="line-7004"> throws IOException {</span> |
| <span class="source-line-no">7005</span><span id="line-7005"> // create an object to use a a key in the row lock map</span> |
| <span class="source-line-no">7006</span><span id="line-7006"> HashedBytes rowKey = new HashedBytes(row);</span> |
| <span class="source-line-no">7007</span><span id="line-7007"></span> |
| <span class="source-line-no">7008</span><span id="line-7008"> RowLockContext rowLockContext = null;</span> |
| <span class="source-line-no">7009</span><span id="line-7009"> RowLockImpl result = null;</span> |
| <span class="source-line-no">7010</span><span id="line-7010"></span> |
| <span class="source-line-no">7011</span><span id="line-7011"> boolean success = false;</span> |
| <span class="source-line-no">7012</span><span id="line-7012"> try {</span> |
| <span class="source-line-no">7013</span><span id="line-7013"> // Keep trying until we have a lock or error out.</span> |
| <span class="source-line-no">7014</span><span id="line-7014"> // TODO: do we need to add a time component here?</span> |
| <span class="source-line-no">7015</span><span id="line-7015"> while (result == null) {</span> |
| <span class="source-line-no">7016</span><span id="line-7016"> rowLockContext = computeIfAbsent(lockedRows, rowKey, () -> new RowLockContext(rowKey));</span> |
| <span class="source-line-no">7017</span><span id="line-7017"> // Now try an get the lock.</span> |
| <span class="source-line-no">7018</span><span id="line-7018"> // This can fail as</span> |
| <span class="source-line-no">7019</span><span id="line-7019"> if (readLock) {</span> |
| <span class="source-line-no">7020</span><span id="line-7020"> // For read lock, if the caller has locked the same row previously, it will not try</span> |
| <span class="source-line-no">7021</span><span id="line-7021"> // to acquire the same read lock. It simply returns the previous row lock.</span> |
| <span class="source-line-no">7022</span><span id="line-7022"> RowLockImpl prevRowLockImpl = (RowLockImpl) prevRowLock;</span> |
| <span class="source-line-no">7023</span><span id="line-7023"> if (</span> |
| <span class="source-line-no">7024</span><span id="line-7024"> (prevRowLockImpl != null)</span> |
| <span class="source-line-no">7025</span><span id="line-7025"> && (prevRowLockImpl.getLock() == rowLockContext.readWriteLock.readLock())</span> |
| <span class="source-line-no">7026</span><span id="line-7026"> ) {</span> |
| <span class="source-line-no">7027</span><span id="line-7027"> success = true;</span> |
| <span class="source-line-no">7028</span><span id="line-7028"> return prevRowLock;</span> |
| <span class="source-line-no">7029</span><span id="line-7029"> }</span> |
| <span class="source-line-no">7030</span><span id="line-7030"> result = rowLockContext.newReadLock();</span> |
| <span class="source-line-no">7031</span><span id="line-7031"> } else {</span> |
| <span class="source-line-no">7032</span><span id="line-7032"> result = rowLockContext.newWriteLock();</span> |
| <span class="source-line-no">7033</span><span id="line-7033"> }</span> |
| <span class="source-line-no">7034</span><span id="line-7034"> }</span> |
| <span class="source-line-no">7035</span><span id="line-7035"></span> |
| <span class="source-line-no">7036</span><span id="line-7036"> int timeout = rowLockWaitDuration;</span> |
| <span class="source-line-no">7037</span><span id="line-7037"> boolean reachDeadlineFirst = false;</span> |
| <span class="source-line-no">7038</span><span id="line-7038"> Optional<RpcCall> call = RpcServer.getCurrentCall();</span> |
| <span class="source-line-no">7039</span><span id="line-7039"> if (call.isPresent()) {</span> |
| <span class="source-line-no">7040</span><span id="line-7040"> long deadline = call.get().getDeadline();</span> |
| <span class="source-line-no">7041</span><span id="line-7041"> if (deadline < Long.MAX_VALUE) {</span> |
| <span class="source-line-no">7042</span><span id="line-7042"> int timeToDeadline = (int) (deadline - EnvironmentEdgeManager.currentTime());</span> |
| <span class="source-line-no">7043</span><span id="line-7043"> if (timeToDeadline <= this.rowLockWaitDuration) {</span> |
| <span class="source-line-no">7044</span><span id="line-7044"> reachDeadlineFirst = true;</span> |
| <span class="source-line-no">7045</span><span id="line-7045"> timeout = timeToDeadline;</span> |
| <span class="source-line-no">7046</span><span id="line-7046"> }</span> |
| <span class="source-line-no">7047</span><span id="line-7047"> }</span> |
| <span class="source-line-no">7048</span><span id="line-7048"> }</span> |
| <span class="source-line-no">7049</span><span id="line-7049"></span> |
| <span class="source-line-no">7050</span><span id="line-7050"> if (timeout <= 0 || !result.getLock().tryLock(timeout, TimeUnit.MILLISECONDS)) {</span> |
| <span class="source-line-no">7051</span><span id="line-7051"> String message = "Timed out waiting for lock for row: " + rowKey + " in region "</span> |
| <span class="source-line-no">7052</span><span id="line-7052"> + getRegionInfo().getEncodedName();</span> |
| <span class="source-line-no">7053</span><span id="line-7053"> if (reachDeadlineFirst) {</span> |
| <span class="source-line-no">7054</span><span id="line-7054"> throw new TimeoutIOException(message);</span> |
| <span class="source-line-no">7055</span><span id="line-7055"> } else {</span> |
| <span class="source-line-no">7056</span><span id="line-7056"> // If timeToDeadline is larger than rowLockWaitDuration, we can not drop the request.</span> |
| <span class="source-line-no">7057</span><span id="line-7057"> throw new IOException(message);</span> |
| <span class="source-line-no">7058</span><span id="line-7058"> }</span> |
| <span class="source-line-no">7059</span><span id="line-7059"> }</span> |
| <span class="source-line-no">7060</span><span id="line-7060"> rowLockContext.setThreadName(Thread.currentThread().getName());</span> |
| <span class="source-line-no">7061</span><span id="line-7061"> success = true;</span> |
| <span class="source-line-no">7062</span><span id="line-7062"> return result;</span> |
| <span class="source-line-no">7063</span><span id="line-7063"> } catch (InterruptedException ie) {</span> |
| <span class="source-line-no">7064</span><span id="line-7064"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">7065</span><span id="line-7065"> LOG.debug("Thread interrupted waiting for lock on row: {}, in region {}", rowKey,</span> |
| <span class="source-line-no">7066</span><span id="line-7066"> getRegionInfo().getRegionNameAsString());</span> |
| <span class="source-line-no">7067</span><span id="line-7067"> }</span> |
| <span class="source-line-no">7068</span><span id="line-7068"> throw throwOnInterrupt(ie);</span> |
| <span class="source-line-no">7069</span><span id="line-7069"> } catch (Error error) {</span> |
| <span class="source-line-no">7070</span><span id="line-7070"> // The maximum lock count for read lock is 64K (hardcoded), when this maximum count</span> |
| <span class="source-line-no">7071</span><span id="line-7071"> // is reached, it will throw out an Error. This Error needs to be caught so it can</span> |
| <span class="source-line-no">7072</span><span id="line-7072"> // go ahead to process the minibatch with lock acquired.</span> |
| <span class="source-line-no">7073</span><span id="line-7073"> LOG.warn("Error to get row lock for {}, in region {}, cause: {}", Bytes.toStringBinary(row),</span> |
| <span class="source-line-no">7074</span><span id="line-7074"> getRegionInfo().getRegionNameAsString(), error);</span> |
| <span class="source-line-no">7075</span><span id="line-7075"> IOException ioe = new IOException(error);</span> |
| <span class="source-line-no">7076</span><span id="line-7076"> throw ioe;</span> |
| <span class="source-line-no">7077</span><span id="line-7077"> } finally {</span> |
| <span class="source-line-no">7078</span><span id="line-7078"> // Clean up the counts just in case this was the thing keeping the context alive.</span> |
| <span class="source-line-no">7079</span><span id="line-7079"> if (!success && rowLockContext != null) {</span> |
| <span class="source-line-no">7080</span><span id="line-7080"> rowLockContext.cleanUp();</span> |
| <span class="source-line-no">7081</span><span id="line-7081"> }</span> |
| <span class="source-line-no">7082</span><span id="line-7082"> }</span> |
| <span class="source-line-no">7083</span><span id="line-7083"> }</span> |
| <span class="source-line-no">7084</span><span id="line-7084"></span> |
| <span class="source-line-no">7085</span><span id="line-7085"> private RowLock getRowLock(byte[] row, boolean readLock, final RowLock prevRowLock)</span> |
| <span class="source-line-no">7086</span><span id="line-7086"> throws IOException {</span> |
| <span class="source-line-no">7087</span><span id="line-7087"> return TraceUtil.trace(() -> getRowLockInternal(row, readLock, prevRowLock),</span> |
| <span class="source-line-no">7088</span><span id="line-7088"> () -> createRegionSpan("Region.getRowLock").setAttribute(ROW_LOCK_READ_LOCK_KEY, readLock));</span> |
| <span class="source-line-no">7089</span><span id="line-7089"> }</span> |
| <span class="source-line-no">7090</span><span id="line-7090"></span> |
| <span class="source-line-no">7091</span><span id="line-7091"> private void releaseRowLocks(List<RowLock> rowLocks) {</span> |
| <span class="source-line-no">7092</span><span id="line-7092"> if (rowLocks != null) {</span> |
| <span class="source-line-no">7093</span><span id="line-7093"> for (RowLock rowLock : rowLocks) {</span> |
| <span class="source-line-no">7094</span><span id="line-7094"> rowLock.release();</span> |
| <span class="source-line-no">7095</span><span id="line-7095"> }</span> |
| <span class="source-line-no">7096</span><span id="line-7096"> rowLocks.clear();</span> |
| <span class="source-line-no">7097</span><span id="line-7097"> }</span> |
| <span class="source-line-no">7098</span><span id="line-7098"> }</span> |
| <span class="source-line-no">7099</span><span id="line-7099"></span> |
| <span class="source-line-no">7100</span><span id="line-7100"> public int getReadLockCount() {</span> |
| <span class="source-line-no">7101</span><span id="line-7101"> return lock.getReadLockCount();</span> |
| <span class="source-line-no">7102</span><span id="line-7102"> }</span> |
| <span class="source-line-no">7103</span><span id="line-7103"></span> |
| <span class="source-line-no">7104</span><span id="line-7104"> public ConcurrentHashMap<HashedBytes, RowLockContext> getLockedRows() {</span> |
| <span class="source-line-no">7105</span><span id="line-7105"> return lockedRows;</span> |
| <span class="source-line-no">7106</span><span id="line-7106"> }</span> |
| <span class="source-line-no">7107</span><span id="line-7107"></span> |
| <span class="source-line-no">7108</span><span id="line-7108"> class RowLockContext {</span> |
| <span class="source-line-no">7109</span><span id="line-7109"> private final HashedBytes row;</span> |
| <span class="source-line-no">7110</span><span id="line-7110"> final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(true);</span> |
| <span class="source-line-no">7111</span><span id="line-7111"> final AtomicBoolean usable = new AtomicBoolean(true);</span> |
| <span class="source-line-no">7112</span><span id="line-7112"> final AtomicInteger count = new AtomicInteger(0);</span> |
| <span class="source-line-no">7113</span><span id="line-7113"> final Object lock = new Object();</span> |
| <span class="source-line-no">7114</span><span id="line-7114"> private String threadName;</span> |
| <span class="source-line-no">7115</span><span id="line-7115"></span> |
| <span class="source-line-no">7116</span><span id="line-7116"> RowLockContext(HashedBytes row) {</span> |
| <span class="source-line-no">7117</span><span id="line-7117"> this.row = row;</span> |
| <span class="source-line-no">7118</span><span id="line-7118"> }</span> |
| <span class="source-line-no">7119</span><span id="line-7119"></span> |
| <span class="source-line-no">7120</span><span id="line-7120"> RowLockImpl newWriteLock() {</span> |
| <span class="source-line-no">7121</span><span id="line-7121"> Lock l = readWriteLock.writeLock();</span> |
| <span class="source-line-no">7122</span><span id="line-7122"> return getRowLock(l);</span> |
| <span class="source-line-no">7123</span><span id="line-7123"> }</span> |
| <span class="source-line-no">7124</span><span id="line-7124"></span> |
| <span class="source-line-no">7125</span><span id="line-7125"> RowLockImpl newReadLock() {</span> |
| <span class="source-line-no">7126</span><span id="line-7126"> Lock l = readWriteLock.readLock();</span> |
| <span class="source-line-no">7127</span><span id="line-7127"> return getRowLock(l);</span> |
| <span class="source-line-no">7128</span><span id="line-7128"> }</span> |
| <span class="source-line-no">7129</span><span id="line-7129"></span> |
| <span class="source-line-no">7130</span><span id="line-7130"> private RowLockImpl getRowLock(Lock l) {</span> |
| <span class="source-line-no">7131</span><span id="line-7131"> count.incrementAndGet();</span> |
| <span class="source-line-no">7132</span><span id="line-7132"> synchronized (lock) {</span> |
| <span class="source-line-no">7133</span><span id="line-7133"> if (usable.get()) {</span> |
| <span class="source-line-no">7134</span><span id="line-7134"> return new RowLockImpl(this, l);</span> |
| <span class="source-line-no">7135</span><span id="line-7135"> } else {</span> |
| <span class="source-line-no">7136</span><span id="line-7136"> return null;</span> |
| <span class="source-line-no">7137</span><span id="line-7137"> }</span> |
| <span class="source-line-no">7138</span><span id="line-7138"> }</span> |
| <span class="source-line-no">7139</span><span id="line-7139"> }</span> |
| <span class="source-line-no">7140</span><span id="line-7140"></span> |
| <span class="source-line-no">7141</span><span id="line-7141"> void cleanUp() {</span> |
| <span class="source-line-no">7142</span><span id="line-7142"> long c = count.decrementAndGet();</span> |
| <span class="source-line-no">7143</span><span id="line-7143"> if (c <= 0) {</span> |
| <span class="source-line-no">7144</span><span id="line-7144"> synchronized (lock) {</span> |
| <span class="source-line-no">7145</span><span id="line-7145"> if (count.get() <= 0 && usable.get()) { // Don't attempt to remove row if already removed</span> |
| <span class="source-line-no">7146</span><span id="line-7146"> usable.set(false);</span> |
| <span class="source-line-no">7147</span><span id="line-7147"> RowLockContext removed = lockedRows.remove(row);</span> |
| <span class="source-line-no">7148</span><span id="line-7148"> assert removed == this : "we should never remove a different context";</span> |
| <span class="source-line-no">7149</span><span id="line-7149"> }</span> |
| <span class="source-line-no">7150</span><span id="line-7150"> }</span> |
| <span class="source-line-no">7151</span><span id="line-7151"> }</span> |
| <span class="source-line-no">7152</span><span id="line-7152"> }</span> |
| <span class="source-line-no">7153</span><span id="line-7153"></span> |
| <span class="source-line-no">7154</span><span id="line-7154"> public void setThreadName(String threadName) {</span> |
| <span class="source-line-no">7155</span><span id="line-7155"> this.threadName = threadName;</span> |
| <span class="source-line-no">7156</span><span id="line-7156"> }</span> |
| <span class="source-line-no">7157</span><span id="line-7157"></span> |
| <span class="source-line-no">7158</span><span id="line-7158"> @Override</span> |
| <span class="source-line-no">7159</span><span id="line-7159"> public String toString() {</span> |
| <span class="source-line-no">7160</span><span id="line-7160"> return "RowLockContext{" + "row=" + row + ", readWriteLock=" + readWriteLock + ", count="</span> |
| <span class="source-line-no">7161</span><span id="line-7161"> + count + ", threadName=" + threadName + '}';</span> |
| <span class="source-line-no">7162</span><span id="line-7162"> }</span> |
| <span class="source-line-no">7163</span><span id="line-7163"> }</span> |
| <span class="source-line-no">7164</span><span id="line-7164"></span> |
| <span class="source-line-no">7165</span><span id="line-7165"> /**</span> |
| <span class="source-line-no">7166</span><span id="line-7166"> * Class used to represent a lock on a row.</span> |
| <span class="source-line-no">7167</span><span id="line-7167"> */</span> |
| <span class="source-line-no">7168</span><span id="line-7168"> public static class RowLockImpl implements RowLock {</span> |
| <span class="source-line-no">7169</span><span id="line-7169"> private final RowLockContext context;</span> |
| <span class="source-line-no">7170</span><span id="line-7170"> private final Lock lock;</span> |
| <span class="source-line-no">7171</span><span id="line-7171"></span> |
| <span class="source-line-no">7172</span><span id="line-7172"> public RowLockImpl(RowLockContext context, Lock lock) {</span> |
| <span class="source-line-no">7173</span><span id="line-7173"> this.context = context;</span> |
| <span class="source-line-no">7174</span><span id="line-7174"> this.lock = lock;</span> |
| <span class="source-line-no">7175</span><span id="line-7175"> }</span> |
| <span class="source-line-no">7176</span><span id="line-7176"></span> |
| <span class="source-line-no">7177</span><span id="line-7177"> public Lock getLock() {</span> |
| <span class="source-line-no">7178</span><span id="line-7178"> return lock;</span> |
| <span class="source-line-no">7179</span><span id="line-7179"> }</span> |
| <span class="source-line-no">7180</span><span id="line-7180"></span> |
| <span class="source-line-no">7181</span><span id="line-7181"> public RowLockContext getContext() {</span> |
| <span class="source-line-no">7182</span><span id="line-7182"> return context;</span> |
| <span class="source-line-no">7183</span><span id="line-7183"> }</span> |
| <span class="source-line-no">7184</span><span id="line-7184"></span> |
| <span class="source-line-no">7185</span><span id="line-7185"> @Override</span> |
| <span class="source-line-no">7186</span><span id="line-7186"> public void release() {</span> |
| <span class="source-line-no">7187</span><span id="line-7187"> lock.unlock();</span> |
| <span class="source-line-no">7188</span><span id="line-7188"> context.cleanUp();</span> |
| <span class="source-line-no">7189</span><span id="line-7189"> }</span> |
| <span class="source-line-no">7190</span><span id="line-7190"></span> |
| <span class="source-line-no">7191</span><span id="line-7191"> @Override</span> |
| <span class="source-line-no">7192</span><span id="line-7192"> public String toString() {</span> |
| <span class="source-line-no">7193</span><span id="line-7193"> return "RowLockImpl{" + "context=" + context + ", lock=" + lock + '}';</span> |
| <span class="source-line-no">7194</span><span id="line-7194"> }</span> |
| <span class="source-line-no">7195</span><span id="line-7195"> }</span> |
| <span class="source-line-no">7196</span><span id="line-7196"></span> |
| <span class="source-line-no">7197</span><span id="line-7197"> /**</span> |
| <span class="source-line-no">7198</span><span id="line-7198"> * Determines whether multiple column families are present Precondition: familyPaths is not null</span> |
| <span class="source-line-no">7199</span><span id="line-7199"> * @param familyPaths List of (column family, hfilePath)</span> |
| <span class="source-line-no">7200</span><span id="line-7200"> */</span> |
| <span class="source-line-no">7201</span><span id="line-7201"> private static boolean hasMultipleColumnFamilies(Collection<Pair<byte[], String>> familyPaths) {</span> |
| <span class="source-line-no">7202</span><span id="line-7202"> boolean multipleFamilies = false;</span> |
| <span class="source-line-no">7203</span><span id="line-7203"> byte[] family = null;</span> |
| <span class="source-line-no">7204</span><span id="line-7204"> for (Pair<byte[], String> pair : familyPaths) {</span> |
| <span class="source-line-no">7205</span><span id="line-7205"> byte[] fam = pair.getFirst();</span> |
| <span class="source-line-no">7206</span><span id="line-7206"> if (family == null) {</span> |
| <span class="source-line-no">7207</span><span id="line-7207"> family = fam;</span> |
| <span class="source-line-no">7208</span><span id="line-7208"> } else if (!Bytes.equals(family, fam)) {</span> |
| <span class="source-line-no">7209</span><span id="line-7209"> multipleFamilies = true;</span> |
| <span class="source-line-no">7210</span><span id="line-7210"> break;</span> |
| <span class="source-line-no">7211</span><span id="line-7211"> }</span> |
| <span class="source-line-no">7212</span><span id="line-7212"> }</span> |
| <span class="source-line-no">7213</span><span id="line-7213"> return multipleFamilies;</span> |
| <span class="source-line-no">7214</span><span id="line-7214"> }</span> |
| <span class="source-line-no">7215</span><span id="line-7215"></span> |
| <span class="source-line-no">7216</span><span id="line-7216"> /**</span> |
| <span class="source-line-no">7217</span><span id="line-7217"> * Attempts to atomically load a group of hfiles. This is critical for loading rows with multiple</span> |
| <span class="source-line-no">7218</span><span id="line-7218"> * column families atomically.</span> |
| <span class="source-line-no">7219</span><span id="line-7219"> * @param familyPaths List of Pair&lt;byte[] column family, String hfilePath&gt;</span> |
| <span class="source-line-no">7220</span><span id="line-7220"> * @param bulkLoadListener Internal hooks enabling massaging/preparation of a file about to be</span> |
| <span class="source-line-no">7221</span><span id="line-7221"> * bulk loaded</span> |
| <span class="source-line-no">7222</span><span id="line-7222"> * @return Map from family to List of store file paths if successful, null if failed recoverably</span> |
| <span class="source-line-no">7223</span><span id="line-7223"> * @throws IOException if failed unrecoverably.</span> |
| <span class="source-line-no">7224</span><span id="line-7224"> */</span> |
| <span class="source-line-no">7225</span><span id="line-7225"> public Map<byte[], List<Path>> bulkLoadHFiles(Collection<Pair<byte[], String>> familyPaths,</span> |
| <span class="source-line-no">7226</span><span id="line-7226"> boolean assignSeqId, BulkLoadListener bulkLoadListener) throws IOException {</span> |
| <span class="source-line-no">7227</span><span id="line-7227"> return bulkLoadHFiles(familyPaths, assignSeqId, bulkLoadListener, false, null, true);</span> |
| <span class="source-line-no">7228</span><span id="line-7228"> }</span> |
| <span class="source-line-no">7229</span><span id="line-7229"></span> |
| <span class="source-line-no">7230</span><span id="line-7230"> /**</span> |
| <span class="source-line-no">7231</span><span id="line-7231"> * Listener class to enable callers of bulkLoadHFile() to perform any necessary pre/post</span> |
| <span class="source-line-no">7232</span><span id="line-7232"> * processing of a given bulkload call</span> |
| <span class="source-line-no">7233</span><span id="line-7233"> */</span> |
| <span class="source-line-no">7234</span><span id="line-7234"> public interface BulkLoadListener {</span> |
| <span class="source-line-no">7235</span><span id="line-7235"> /**</span> |
| <span class="source-line-no">7236</span><span id="line-7236"> * Called before an HFile is actually loaded</span> |
| <span class="source-line-no">7237</span><span id="line-7237"> * @param family family being loaded to</span> |
| <span class="source-line-no">7238</span><span id="line-7238"> * @param srcPath path of HFile</span> |
| <span class="source-line-no">7239</span><span id="line-7239"> * @return final path to be used for actual loading</span> |
| <span class="source-line-no">7240</span><span id="line-7240"> */</span> |
| <span class="source-line-no">7241</span><span id="line-7241"> String prepareBulkLoad(byte[] family, String srcPath, boolean copyFile, String customStaging)</span> |
| <span class="source-line-no">7242</span><span id="line-7242"> throws IOException;</span> |
| <span class="source-line-no">7243</span><span id="line-7243"></span> |
| <span class="source-line-no">7244</span><span id="line-7244"> /**</span> |
| <span class="source-line-no">7245</span><span id="line-7245"> * Called after a successful HFile load</span> |
| <span class="source-line-no">7246</span><span id="line-7246"> * @param family family being loaded to</span> |
| <span class="source-line-no">7247</span><span id="line-7247"> * @param srcPath path of HFile</span> |
| <span class="source-line-no">7248</span><span id="line-7248"> */</span> |
| <span class="source-line-no">7249</span><span id="line-7249"> void doneBulkLoad(byte[] family, String srcPath) throws IOException;</span> |
| <span class="source-line-no">7250</span><span id="line-7250"></span> |
| <span class="source-line-no">7251</span><span id="line-7251"> /**</span> |
| <span class="source-line-no">7252</span><span id="line-7252"> * Called after a failed HFile load</span> |
| <span class="source-line-no">7253</span><span id="line-7253"> * @param family family being loaded to</span> |
| <span class="source-line-no">7254</span><span id="line-7254"> * @param srcPath path of HFile</span> |
| <span class="source-line-no">7255</span><span id="line-7255"> */</span> |
| <span class="source-line-no">7256</span><span id="line-7256"> void failedBulkLoad(byte[] family, String srcPath) throws IOException;</span> |
| <span class="source-line-no">7257</span><span id="line-7257"> }</span> |
| <span class="source-line-no">7258</span><span id="line-7258"></span> |
| <span class="source-line-no">7259</span><span id="line-7259"> /**</span> |
| <span class="source-line-no">7260</span><span id="line-7260"> * Attempts to atomically load a group of hfiles. This is critical for loading rows with multiple</span> |
| <span class="source-line-no">7261</span><span id="line-7261"> * column families atomically.</span> |
| <span class="source-line-no">7262</span><span id="line-7262"> * @param familyPaths List of Pair&lt;byte[] column family, String hfilePath&gt;</span> |
| <span class="source-line-no">7263</span><span id="line-7263"> * @param bulkLoadListener Internal hooks enabling massaging/preparation of a file about to be</span> |
| <span class="source-line-no">7264</span><span id="line-7264"> * bulk loaded</span> |
| <span class="source-line-no">7265</span><span id="line-7265"> * @param copyFile always copy hfiles if true</span> |
| <span class="source-line-no">7266</span><span id="line-7266"> * @param clusterIds ids from clusters that had already handled the given bulkload event.</span> |
| <span class="source-line-no">7267</span><span id="line-7267"> * @return Map from family to List of store file paths if successful, null if failed recoverably</span> |
| <span class="source-line-no">7268</span><span id="line-7268"> * @throws IOException if failed unrecoverably.</span> |
| <span class="source-line-no">7269</span><span id="line-7269"> */</span> |
| <span class="source-line-no">7270</span><span id="line-7270"> public Map<byte[], List<Path>> bulkLoadHFiles(Collection<Pair<byte[], String>> familyPaths,</span> |
| <span class="source-line-no">7271</span><span id="line-7271"> boolean assignSeqId, BulkLoadListener bulkLoadListener, boolean copyFile,</span> |
| <span class="source-line-no">7272</span><span id="line-7272"> List<String> clusterIds, boolean replicate) throws IOException {</span> |
| <span class="source-line-no">7273</span><span id="line-7273"> long seqId = -1;</span> |
| <span class="source-line-no">7274</span><span id="line-7274"> Map<byte[], List<Path>> storeFiles = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">7275</span><span id="line-7275"> Map<String, Long> storeFilesSizes = new HashMap<>();</span> |
| <span class="source-line-no">7276</span><span id="line-7276"> Preconditions.checkNotNull(familyPaths);</span> |
| <span class="source-line-no">7277</span><span id="line-7277"> // we need writeLock for multi-family bulk load</span> |
| <span class="source-line-no">7278</span><span id="line-7278"> startBulkRegionOperation(hasMultipleColumnFamilies(familyPaths));</span> |
| <span class="source-line-no">7279</span><span id="line-7279"> boolean isSuccessful = false;</span> |
| <span class="source-line-no">7280</span><span id="line-7280"> try {</span> |
| <span class="source-line-no">7281</span><span id="line-7281"> this.writeRequestsCount.increment();</span> |
| <span class="source-line-no">7282</span><span id="line-7282"></span> |
| <span class="source-line-no">7283</span><span id="line-7283"> // There possibly was a split that happened between when the split keys</span> |
| <span class="source-line-no">7284</span><span id="line-7284"> // were gathered and before the HRegion's write lock was taken. We need</span> |
| <span class="source-line-no">7285</span><span id="line-7285"> // to validate the HFile region before attempting to bulk load all of them</span> |
| <span class="source-line-no">7286</span><span id="line-7286"> IOException ioException = null;</span> |
| <span class="source-line-no">7287</span><span id="line-7287"> List<Pair<byte[], String>> failures = new ArrayList<>();</span> |
| <span class="source-line-no">7288</span><span id="line-7288"> for (Pair<byte[], String> p : familyPaths) {</span> |
| <span class="source-line-no">7289</span><span id="line-7289"> byte[] familyName = p.getFirst();</span> |
| <span class="source-line-no">7290</span><span id="line-7290"> String path = p.getSecond();</span> |
| <span class="source-line-no">7291</span><span id="line-7291"></span> |
| <span class="source-line-no">7292</span><span id="line-7292"> HStore store = getStore(familyName);</span> |
| <span class="source-line-no">7293</span><span id="line-7293"> if (store == null) {</span> |
| <span class="source-line-no">7294</span><span id="line-7294"> ioException = new org.apache.hadoop.hbase.DoNotRetryIOException(</span> |
| <span class="source-line-no">7295</span><span id="line-7295"> "No such column family " + Bytes.toStringBinary(familyName));</span> |
| <span class="source-line-no">7296</span><span id="line-7296"> } else {</span> |
| <span class="source-line-no">7297</span><span id="line-7297"> try {</span> |
| <span class="source-line-no">7298</span><span id="line-7298"> store.assertBulkLoadHFileOk(new Path(path));</span> |
| <span class="source-line-no">7299</span><span id="line-7299"> } catch (WrongRegionException wre) {</span> |
| <span class="source-line-no">7300</span><span id="line-7300"> // recoverable (file doesn't fit in region)</span> |
| <span class="source-line-no">7301</span><span id="line-7301"> failures.add(p);</span> |
| <span class="source-line-no">7302</span><span id="line-7302"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">7303</span><span id="line-7303"> // unrecoverable (hdfs problem)</span> |
| <span class="source-line-no">7304</span><span id="line-7304"> ioException = ioe;</span> |
| <span class="source-line-no">7305</span><span id="line-7305"> }</span> |
| <span class="source-line-no">7306</span><span id="line-7306"> }</span> |
| <span class="source-line-no">7307</span><span id="line-7307"></span> |
| <span class="source-line-no">7308</span><span id="line-7308"> // validation failed because of some sort of IO problem.</span> |
| <span class="source-line-no">7309</span><span id="line-7309"> if (ioException != null) {</span> |
| <span class="source-line-no">7310</span><span id="line-7310"> LOG.error("There was IO error when checking if the bulk load is ok in region {}.", this,</span> |
| <span class="source-line-no">7311</span><span id="line-7311"> ioException);</span> |
| <span class="source-line-no">7312</span><span id="line-7312"> throw ioException;</span> |
| <span class="source-line-no">7313</span><span id="line-7313"> }</span> |
| <span class="source-line-no">7314</span><span id="line-7314"> }</span> |
| <span class="source-line-no">7315</span><span id="line-7315"> // validation failed, bail out before doing anything permanent.</span> |
| <span class="source-line-no">7316</span><span id="line-7316"> if (failures.size() != 0) {</span> |
| <span class="source-line-no">7317</span><span id="line-7317"> StringBuilder list = new StringBuilder();</span> |
| <span class="source-line-no">7318</span><span id="line-7318"> for (Pair<byte[], String> p : failures) {</span> |
| <span class="source-line-no">7319</span><span id="line-7319"> list.append("\n").append(Bytes.toString(p.getFirst())).append(" : ")</span> |
| <span class="source-line-no">7320</span><span id="line-7320"> .append(p.getSecond());</span> |
| <span class="source-line-no">7321</span><span id="line-7321"> }</span> |
| <span class="source-line-no">7322</span><span id="line-7322"> // problem when validating</span> |
| <span class="source-line-no">7323</span><span id="line-7323"> LOG.warn("There was a recoverable bulk load failure likely due to a split. These (family,"</span> |
| <span class="source-line-no">7324</span><span id="line-7324"> + " HFile) pairs were not loaded: {}, in region {}", list.toString(), this);</span> |
| <span class="source-line-no">7325</span><span id="line-7325"> return null;</span> |
| <span class="source-line-no">7326</span><span id="line-7326"> }</span> |
| <span class="source-line-no">7327</span><span id="line-7327"></span> |
| <span class="source-line-no">7328</span><span id="line-7328"> // We need to assign a sequential ID that's in between two memstores in order to preserve</span> |
| <span class="source-line-no">7329</span><span id="line-7329"> // the guarantee that all the edits lower than the highest sequential ID from all the</span> |
| <span class="source-line-no">7330</span><span id="line-7330"> // HFiles are flushed on disk. See HBASE-10958. The sequence id returned when we flush is</span> |
| <span class="source-line-no">7331</span><span id="line-7331"> // guaranteed to be one beyond the file made when we flushed (or if nothing to flush, it is</span> |
| <span class="source-line-no">7332</span><span id="line-7332"> // a sequence id that we can be sure is beyond the last hfile written).</span> |
| <span class="source-line-no">7333</span><span id="line-7333"> if (assignSeqId) {</span> |
| <span class="source-line-no">7334</span><span id="line-7334"> FlushResult fs = flushcache(true, false, FlushLifeCycleTracker.DUMMY);</span> |
| <span class="source-line-no">7335</span><span id="line-7335"> if (fs.isFlushSucceeded()) {</span> |
| <span class="source-line-no">7336</span><span id="line-7336"> seqId = ((FlushResultImpl) fs).flushSequenceId;</span> |
| <span class="source-line-no">7337</span><span id="line-7337"> } else if (fs.getResult() == FlushResult.Result.CANNOT_FLUSH_MEMSTORE_EMPTY) {</span> |
| <span class="source-line-no">7338</span><span id="line-7338"> seqId = ((FlushResultImpl) fs).flushSequenceId;</span> |
| <span class="source-line-no">7339</span><span id="line-7339"> } else if (fs.getResult() == FlushResult.Result.CANNOT_FLUSH) {</span> |
| <span class="source-line-no">7340</span><span id="line-7340"> // CANNOT_FLUSH may mean that a flush is already on-going</span> |
| <span class="source-line-no">7341</span><span id="line-7341"> // we need to wait for that flush to complete</span> |
| <span class="source-line-no">7342</span><span id="line-7342"> waitForFlushes();</span> |
| <span class="source-line-no">7343</span><span id="line-7343"> } else {</span> |
| <span class="source-line-no">7344</span><span id="line-7344"> throw new IOException("Could not bulk load with an assigned sequential ID because the "</span> |
| <span class="source-line-no">7345</span><span id="line-7345"> + "flush didn't run. Reason for not flushing: " + ((FlushResultImpl) fs).failureReason);</span> |
| <span class="source-line-no">7346</span><span id="line-7346"> }</span> |
| <span class="source-line-no">7347</span><span id="line-7347"> }</span> |
| <span class="source-line-no">7348</span><span id="line-7348"></span> |
| <span class="source-line-no">7349</span><span id="line-7349"> Map<byte[], List<Pair<Path, Path>>> familyWithFinalPath =</span> |
| <span class="source-line-no">7350</span><span id="line-7350"> new TreeMap<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">7351</span><span id="line-7351"> for (Pair<byte[], String> p : familyPaths) {</span> |
| <span class="source-line-no">7352</span><span id="line-7352"> byte[] familyName = p.getFirst();</span> |
| <span class="source-line-no">7353</span><span id="line-7353"> String path = p.getSecond();</span> |
| <span class="source-line-no">7354</span><span id="line-7354"> HStore store = getStore(familyName);</span> |
| <span class="source-line-no">7355</span><span id="line-7355"> if (!familyWithFinalPath.containsKey(familyName)) {</span> |
| <span class="source-line-no">7356</span><span id="line-7356"> familyWithFinalPath.put(familyName, new ArrayList<>());</span> |
| <span class="source-line-no">7357</span><span id="line-7357"> }</span> |
| <span class="source-line-no">7358</span><span id="line-7358"> List<Pair<Path, Path>> lst = familyWithFinalPath.get(familyName);</span> |
| <span class="source-line-no">7359</span><span id="line-7359"> String finalPath = path;</span> |
| <span class="source-line-no">7360</span><span id="line-7360"> try {</span> |
| <span class="source-line-no">7361</span><span id="line-7361"> boolean reqTmp = store.storeEngine.requireWritingToTmpDirFirst();</span> |
| <span class="source-line-no">7362</span><span id="line-7362"> if (bulkLoadListener != null) {</span> |
| <span class="source-line-no">7363</span><span id="line-7363"> finalPath = bulkLoadListener.prepareBulkLoad(familyName, path, copyFile,</span> |
| <span class="source-line-no">7364</span><span id="line-7364"> reqTmp ? null : fs.getRegionDir().toString());</span> |
| <span class="source-line-no">7365</span><span id="line-7365"> }</span> |
| <span class="source-line-no">7366</span><span id="line-7366"> Pair<Path, Path> pair = null;</span> |
| <span class="source-line-no">7367</span><span id="line-7367"> if (reqTmp || !StoreFileInfo.isHFile(finalPath)) {</span> |
| <span class="source-line-no">7368</span><span id="line-7368"> pair = store.preBulkLoadHFile(finalPath, seqId);</span> |
| <span class="source-line-no">7369</span><span id="line-7369"> } else {</span> |
| <span class="source-line-no">7370</span><span id="line-7370"> Path livePath = new Path(finalPath);</span> |
| <span class="source-line-no">7371</span><span id="line-7371"> pair = new Pair<>(livePath, livePath);</span> |
| <span class="source-line-no">7372</span><span id="line-7372"> }</span> |
| <span class="source-line-no">7373</span><span id="line-7373"> lst.add(pair);</span> |
| <span class="source-line-no">7374</span><span id="line-7374"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">7375</span><span id="line-7375"> // A failure here can cause an atomicity violation that we currently</span> |
| <span class="source-line-no">7376</span><span id="line-7376"> // cannot recover from since it is likely a failed HDFS operation.</span> |
| <span class="source-line-no">7377</span><span id="line-7377"></span> |
| <span class="source-line-no">7378</span><span id="line-7378"> LOG.error("There was a partial failure due to IO when attempting to" + " load "</span> |
| <span class="source-line-no">7379</span><span id="line-7379"> + Bytes.toString(p.getFirst()) + " : " + p.getSecond(), ioe);</span> |
| <span class="source-line-no">7380</span><span id="line-7380"> if (bulkLoadListener != null) {</span> |
| <span class="source-line-no">7381</span><span id="line-7381"> try {</span> |
| <span class="source-line-no">7382</span><span id="line-7382"> bulkLoadListener.failedBulkLoad(familyName, finalPath);</span> |
| <span class="source-line-no">7383</span><span id="line-7383"> } catch (Exception ex) {</span> |
| <span class="source-line-no">7384</span><span id="line-7384"> LOG.error("Error while calling failedBulkLoad for family "</span> |
| <span class="source-line-no">7385</span><span id="line-7385"> + Bytes.toString(familyName) + " with path " + path, ex);</span> |
| <span class="source-line-no">7386</span><span id="line-7386"> }</span> |
| <span class="source-line-no">7387</span><span id="line-7387"> }</span> |
| <span class="source-line-no">7388</span><span id="line-7388"> throw ioe;</span> |
| <span class="source-line-no">7389</span><span id="line-7389"> }</span> |
| <span class="source-line-no">7390</span><span id="line-7390"> }</span> |
| <span class="source-line-no">7391</span><span id="line-7391"></span> |
| <span class="source-line-no">7392</span><span id="line-7392"> if (this.getCoprocessorHost() != null) {</span> |
| <span class="source-line-no">7393</span><span id="line-7393"> for (Map.Entry<byte[], List<Pair<Path, Path>>> entry : familyWithFinalPath.entrySet()) {</span> |
| <span class="source-line-no">7394</span><span id="line-7394"> this.getCoprocessorHost().preCommitStoreFile(entry.getKey(), entry.getValue());</span> |
| <span class="source-line-no">7395</span><span id="line-7395"> }</span> |
| <span class="source-line-no">7396</span><span id="line-7396"> }</span> |
| <span class="source-line-no">7397</span><span id="line-7397"> for (Map.Entry<byte[], List<Pair<Path, Path>>> entry : familyWithFinalPath.entrySet()) {</span> |
| <span class="source-line-no">7398</span><span id="line-7398"> byte[] familyName = entry.getKey();</span> |
| <span class="source-line-no">7399</span><span id="line-7399"> for (Pair<Path, Path> p : entry.getValue()) {</span> |
| <span class="source-line-no">7400</span><span id="line-7400"> String path = p.getFirst().toString();</span> |
| <span class="source-line-no">7401</span><span id="line-7401"> Path commitedStoreFile = p.getSecond();</span> |
| <span class="source-line-no">7402</span><span id="line-7402"> HStore store = getStore(familyName);</span> |
| <span class="source-line-no">7403</span><span id="line-7403"> try {</span> |
| <span class="source-line-no">7404</span><span id="line-7404"> store.bulkLoadHFile(familyName, path, commitedStoreFile);</span> |
| <span class="source-line-no">7405</span><span id="line-7405"> // Note the size of the store file</span> |
| <span class="source-line-no">7406</span><span id="line-7406"> try {</span> |
| <span class="source-line-no">7407</span><span id="line-7407"> FileSystem fs = commitedStoreFile.getFileSystem(baseConf);</span> |
| <span class="source-line-no">7408</span><span id="line-7408"> storeFilesSizes.put(commitedStoreFile.getName(),</span> |
| <span class="source-line-no">7409</span><span id="line-7409"> fs.getFileStatus(commitedStoreFile).getLen());</span> |
| <span class="source-line-no">7410</span><span id="line-7410"> } catch (IOException e) {</span> |
| <span class="source-line-no">7411</span><span id="line-7411"> LOG.warn("Failed to find the size of hfile " + commitedStoreFile, e);</span> |
| <span class="source-line-no">7412</span><span id="line-7412"> storeFilesSizes.put(commitedStoreFile.getName(), 0L);</span> |
| <span class="source-line-no">7413</span><span id="line-7413"> }</span> |
| <span class="source-line-no">7414</span><span id="line-7414"></span> |
| <span class="source-line-no">7415</span><span id="line-7415"> if (storeFiles.containsKey(familyName)) {</span> |
| <span class="source-line-no">7416</span><span id="line-7416"> storeFiles.get(familyName).add(commitedStoreFile);</span> |
| <span class="source-line-no">7417</span><span id="line-7417"> } else {</span> |
| <span class="source-line-no">7418</span><span id="line-7418"> List<Path> storeFileNames = new ArrayList<>();</span> |
| <span class="source-line-no">7419</span><span id="line-7419"> storeFileNames.add(commitedStoreFile);</span> |
| <span class="source-line-no">7420</span><span id="line-7420"> storeFiles.put(familyName, storeFileNames);</span> |
| <span class="source-line-no">7421</span><span id="line-7421"> }</span> |
| <span class="source-line-no">7422</span><span id="line-7422"> if (bulkLoadListener != null) {</span> |
| <span class="source-line-no">7423</span><span id="line-7423"> bulkLoadListener.doneBulkLoad(familyName, path);</span> |
| <span class="source-line-no">7424</span><span id="line-7424"> }</span> |
| <span class="source-line-no">7425</span><span id="line-7425"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">7426</span><span id="line-7426"> // A failure here can cause an atomicity violation that we currently</span> |
| <span class="source-line-no">7427</span><span id="line-7427"> // cannot recover from since it is likely a failed HDFS operation.</span> |
| <span class="source-line-no">7428</span><span id="line-7428"></span> |
| <span class="source-line-no">7429</span><span id="line-7429"> // TODO Need a better story for reverting partial failures due to HDFS.</span> |
| <span class="source-line-no">7430</span><span id="line-7430"> LOG.error("There was a partial failure due to IO when attempting to" + " load "</span> |
| <span class="source-line-no">7431</span><span id="line-7431"> + Bytes.toString(familyName) + " : " + p.getSecond(), ioe);</span> |
| <span class="source-line-no">7432</span><span id="line-7432"> if (bulkLoadListener != null) {</span> |
| <span class="source-line-no">7433</span><span id="line-7433"> try {</span> |
| <span class="source-line-no">7434</span><span id="line-7434"> bulkLoadListener.failedBulkLoad(familyName, path);</span> |
| <span class="source-line-no">7435</span><span id="line-7435"> } catch (Exception ex) {</span> |
| <span class="source-line-no">7436</span><span id="line-7436"> LOG.error("Error while calling failedBulkLoad for family "</span> |
| <span class="source-line-no">7437</span><span id="line-7437"> + Bytes.toString(familyName) + " with path " + path, ex);</span> |
| <span class="source-line-no">7438</span><span id="line-7438"> }</span> |
| <span class="source-line-no">7439</span><span id="line-7439"> }</span> |
| <span class="source-line-no">7440</span><span id="line-7440"> throw ioe;</span> |
| <span class="source-line-no">7441</span><span id="line-7441"> }</span> |
| <span class="source-line-no">7442</span><span id="line-7442"> }</span> |
| <span class="source-line-no">7443</span><span id="line-7443"> }</span> |
| <span class="source-line-no">7444</span><span id="line-7444"></span> |
| <span class="source-line-no">7445</span><span id="line-7445"> isSuccessful = true;</span> |
| <span class="source-line-no">7446</span><span id="line-7446"> if (conf.getBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, false)) {</span> |
| <span class="source-line-no">7447</span><span id="line-7447"> // request compaction</span> |
| <span class="source-line-no">7448</span><span id="line-7448"> familyWithFinalPath.keySet().forEach(family -> {</span> |
| <span class="source-line-no">7449</span><span id="line-7449"> HStore store = getStore(family);</span> |
| <span class="source-line-no">7450</span><span id="line-7450"> try {</span> |
| <span class="source-line-no">7451</span><span id="line-7451"> if (this.rsServices != null && store.needsCompaction()) {</span> |
| <span class="source-line-no">7452</span><span id="line-7452"> this.rsServices.getCompactionRequestor().requestSystemCompaction(this, store,</span> |
| <span class="source-line-no">7453</span><span id="line-7453"> "bulkload hfiles request compaction", true);</span> |
| <span class="source-line-no">7454</span><span id="line-7454"> LOG.info("Request compaction for region {} family {} after bulk load",</span> |
| <span class="source-line-no">7455</span><span id="line-7455"> this.getRegionInfo().getEncodedName(), store.getColumnFamilyName());</span> |
| <span class="source-line-no">7456</span><span id="line-7456"> }</span> |
| <span class="source-line-no">7457</span><span id="line-7457"> } catch (IOException e) {</span> |
| <span class="source-line-no">7458</span><span id="line-7458"> LOG.error("bulkload hfiles request compaction error ", e);</span> |
| <span class="source-line-no">7459</span><span id="line-7459"> }</span> |
| <span class="source-line-no">7460</span><span id="line-7460"> });</span> |
| <span class="source-line-no">7461</span><span id="line-7461"> }</span> |
| <span class="source-line-no">7462</span><span id="line-7462"> } finally {</span> |
| <span class="source-line-no">7463</span><span id="line-7463"> if (wal != null && !storeFiles.isEmpty()) {</span> |
| <span class="source-line-no">7464</span><span id="line-7464"> // Write a bulk load event for hfiles that are loaded</span> |
| <span class="source-line-no">7465</span><span id="line-7465"> try {</span> |
| <span class="source-line-no">7466</span><span id="line-7466"> WALProtos.BulkLoadDescriptor loadDescriptor =</span> |
| <span class="source-line-no">7467</span><span id="line-7467"> ProtobufUtil.toBulkLoadDescriptor(this.getRegionInfo().getTable(),</span> |
| <span class="source-line-no">7468</span><span id="line-7468"> UnsafeByteOperations.unsafeWrap(this.getRegionInfo().getEncodedNameAsBytes()),</span> |
| <span class="source-line-no">7469</span><span id="line-7469"> storeFiles, storeFilesSizes, seqId, clusterIds, replicate);</span> |
| <span class="source-line-no">7470</span><span id="line-7470"> WALUtil.writeBulkLoadMarkerAndSync(this.wal, this.getReplicationScope(), getRegionInfo(),</span> |
| <span class="source-line-no">7471</span><span id="line-7471"> loadDescriptor, mvcc, regionReplicationSink.orElse(null));</span> |
| <span class="source-line-no">7472</span><span id="line-7472"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">7473</span><span id="line-7473"> if (this.rsServices != null) {</span> |
| <span class="source-line-no">7474</span><span id="line-7474"> // Have to abort region server because some hfiles has been loaded but we can't write</span> |
| <span class="source-line-no">7475</span><span id="line-7475"> // the event into WAL</span> |
| <span class="source-line-no">7476</span><span id="line-7476"> isSuccessful = false;</span> |
| <span class="source-line-no">7477</span><span id="line-7477"> this.rsServices.abort("Failed to write bulk load event into WAL.", ioe);</span> |
| <span class="source-line-no">7478</span><span id="line-7478"> }</span> |
| <span class="source-line-no">7479</span><span id="line-7479"> }</span> |
| <span class="source-line-no">7480</span><span id="line-7480"> }</span> |
| <span class="source-line-no">7481</span><span id="line-7481"></span> |
| <span class="source-line-no">7482</span><span id="line-7482"> closeBulkRegionOperation();</span> |
| <span class="source-line-no">7483</span><span id="line-7483"> }</span> |
| <span class="source-line-no">7484</span><span id="line-7484"> return isSuccessful ? storeFiles : null;</span> |
| <span class="source-line-no">7485</span><span id="line-7485"> }</span> |
| <span class="source-line-no">7486</span><span id="line-7486"></span> |
| <span class="source-line-no">7487</span><span id="line-7487"> @Override</span> |
| <span class="source-line-no">7488</span><span id="line-7488"> public boolean equals(Object o) {</span> |
| <span class="source-line-no">7489</span><span id="line-7489"> return o instanceof HRegion && Bytes.equals(getRegionInfo().getRegionName(),</span> |
| <span class="source-line-no">7490</span><span id="line-7490"> ((HRegion) o).getRegionInfo().getRegionName());</span> |
| <span class="source-line-no">7491</span><span id="line-7491"> }</span> |
| <span class="source-line-no">7492</span><span id="line-7492"></span> |
| <span class="source-line-no">7493</span><span id="line-7493"> @Override</span> |
| <span class="source-line-no">7494</span><span id="line-7494"> public int hashCode() {</span> |
| <span class="source-line-no">7495</span><span id="line-7495"> return Bytes.hashCode(getRegionInfo().getRegionName());</span> |
| <span class="source-line-no">7496</span><span id="line-7496"> }</span> |
| <span class="source-line-no">7497</span><span id="line-7497"></span> |
| <span class="source-line-no">7498</span><span id="line-7498"> @Override</span> |
| <span class="source-line-no">7499</span><span id="line-7499"> public String toString() {</span> |
| <span class="source-line-no">7500</span><span id="line-7500"> return getRegionInfo().getRegionNameAsString();</span> |
| <span class="source-line-no">7501</span><span id="line-7501"> }</span> |
| <span class="source-line-no">7502</span><span id="line-7502"></span> |
| <span class="source-line-no">7503</span><span id="line-7503"> // Utility methods</span> |
| <span class="source-line-no">7504</span><span id="line-7504"> /**</span> |
| <span class="source-line-no">7505</span><span id="line-7505"> * A utility method to create new instances of HRegion based on the {@link HConstants#REGION_IMPL}</span> |
| <span class="source-line-no">7506</span><span id="line-7506"> * configuration property.</span> |
| <span class="source-line-no">7507</span><span id="line-7507"> * @param tableDir qualified path of directory where region should be located, usually the table</span> |
| <span class="source-line-no">7508</span><span id="line-7508"> * directory.</span> |
| <span class="source-line-no">7509</span><span id="line-7509"> * @param wal The WAL is the outbound log for any updates to the HRegion The wal file is a</span> |
| <span class="source-line-no">7510</span><span id="line-7510"> * logfile from the previous execution that's custom-computed for this HRegion.</span> |
| <span class="source-line-no">7511</span><span id="line-7511"> * The HRegionServer computes and sorts the appropriate wal info for this</span> |
| <span class="source-line-no">7512</span><span id="line-7512"> * HRegion. If there is a previous file (implying that the HRegion has been</span> |
| <span class="source-line-no">7513</span><span id="line-7513"> * written-to before), then read it from the supplied path.</span> |
| <span class="source-line-no">7514</span><span id="line-7514"> * @param fs is the filesystem.</span> |
| <span class="source-line-no">7515</span><span id="line-7515"> * @param conf is global configuration settings.</span> |
| <span class="source-line-no">7516</span><span id="line-7516"> * @param regionInfo - RegionInfo that describes the region is new), then read them from the</span> |
| <span class="source-line-no">7517</span><span id="line-7517"> * supplied path.</span> |
| <span class="source-line-no">7518</span><span id="line-7518"> * @param htd the table descriptor</span> |
| <span class="source-line-no">7519</span><span id="line-7519"> * @return the new instance</span> |
| <span class="source-line-no">7520</span><span id="line-7520"> */</span> |
| <span class="source-line-no">7521</span><span id="line-7521"> public static HRegion newHRegion(Path tableDir, WAL wal, FileSystem fs, Configuration conf,</span> |
| <span class="source-line-no">7522</span><span id="line-7522"> RegionInfo regionInfo, final TableDescriptor htd, RegionServerServices rsServices) {</span> |
| <span class="source-line-no">7523</span><span id="line-7523"> try {</span> |
| <span class="source-line-no">7524</span><span id="line-7524"> @SuppressWarnings("unchecked")</span> |
| <span class="source-line-no">7525</span><span id="line-7525"> Class<? extends HRegion> regionClass =</span> |
| <span class="source-line-no">7526</span><span id="line-7526"> (Class<? extends HRegion>) conf.getClass(HConstants.REGION_IMPL, HRegion.class);</span> |
| <span class="source-line-no">7527</span><span id="line-7527"></span> |
| <span class="source-line-no">7528</span><span id="line-7528"> Constructor<? extends HRegion> c =</span> |
| <span class="source-line-no">7529</span><span id="line-7529"> regionClass.getConstructor(Path.class, WAL.class, FileSystem.class, Configuration.class,</span> |
| <span class="source-line-no">7530</span><span id="line-7530"> RegionInfo.class, TableDescriptor.class, RegionServerServices.class);</span> |
| <span class="source-line-no">7531</span><span id="line-7531"></span> |
| <span class="source-line-no">7532</span><span id="line-7532"> return c.newInstance(tableDir, wal, fs, conf, regionInfo, htd, rsServices);</span> |
| <span class="source-line-no">7533</span><span id="line-7533"> } catch (Throwable e) {</span> |
| <span class="source-line-no">7534</span><span id="line-7534"> // todo: what should I throw here?</span> |
| <span class="source-line-no">7535</span><span id="line-7535"> throw new IllegalStateException("Could not instantiate a region instance.", e);</span> |
| <span class="source-line-no">7536</span><span id="line-7536"> }</span> |
| <span class="source-line-no">7537</span><span id="line-7537"> }</span> |
| <span class="source-line-no">7538</span><span id="line-7538"></span> |
| <span class="source-line-no">7539</span><span id="line-7539"> /**</span> |
| <span class="source-line-no">7540</span><span id="line-7540"> * Convenience method creating new HRegions. Used by createTable.</span> |
| <span class="source-line-no">7541</span><span id="line-7541"> * @param info Info for region to create.</span> |
| <span class="source-line-no">7542</span><span id="line-7542"> * @param rootDir Root directory for HBase instance</span> |
| <span class="source-line-no">7543</span><span id="line-7543"> * @param wal shared WAL</span> |
| <span class="source-line-no">7544</span><span id="line-7544"> * @param initialize - true to initialize the region</span> |
| <span class="source-line-no">7545</span><span id="line-7545"> * @return new HRegion</span> |
| <span class="source-line-no">7546</span><span id="line-7546"> */</span> |
| <span class="source-line-no">7547</span><span id="line-7547"> public static HRegion createHRegion(final RegionInfo info, final Path rootDir,</span> |
| <span class="source-line-no">7548</span><span id="line-7548"> final Configuration conf, final TableDescriptor hTableDescriptor, final WAL wal,</span> |
| <span class="source-line-no">7549</span><span id="line-7549"> final boolean initialize) throws IOException {</span> |
| <span class="source-line-no">7550</span><span id="line-7550"> return createHRegion(info, rootDir, conf, hTableDescriptor, wal, initialize, null);</span> |
| <span class="source-line-no">7551</span><span id="line-7551"> }</span> |
| <span class="source-line-no">7552</span><span id="line-7552"></span> |
| <span class="source-line-no">7553</span><span id="line-7553"> /**</span> |
| <span class="source-line-no">7554</span><span id="line-7554"> * Convenience method creating new HRegions. Used by createTable.</span> |
| <span class="source-line-no">7555</span><span id="line-7555"> * @param info Info for region to create.</span> |
| <span class="source-line-no">7556</span><span id="line-7556"> * @param rootDir Root directory for HBase instance</span> |
| <span class="source-line-no">7557</span><span id="line-7557"> * @param wal shared WAL</span> |
| <span class="source-line-no">7558</span><span id="line-7558"> * @param initialize - true to initialize the region</span> |
| <span class="source-line-no">7559</span><span id="line-7559"> * @param rsRpcServices An interface we can request flushes against.</span> |
| <span class="source-line-no">7560</span><span id="line-7560"> * @return new HRegion</span> |
| <span class="source-line-no">7561</span><span id="line-7561"> */</span> |
| <span class="source-line-no">7562</span><span id="line-7562"> public static HRegion createHRegion(final RegionInfo info, final Path rootDir,</span> |
| <span class="source-line-no">7563</span><span id="line-7563"> final Configuration conf, final TableDescriptor hTableDescriptor, final WAL wal,</span> |
| <span class="source-line-no">7564</span><span id="line-7564"> final boolean initialize, RegionServerServices rsRpcServices) throws IOException {</span> |
| <span class="source-line-no">7565</span><span id="line-7565"> LOG.info("creating " + info + ", tableDescriptor="</span> |
| <span class="source-line-no">7566</span><span id="line-7566"> + (hTableDescriptor == null ? "null" : hTableDescriptor) + ", regionDir=" + rootDir);</span> |
| <span class="source-line-no">7567</span><span id="line-7567"> createRegionDir(conf, info, rootDir);</span> |
| <span class="source-line-no">7568</span><span id="line-7568"> FileSystem fs = rootDir.getFileSystem(conf);</span> |
| <span class="source-line-no">7569</span><span id="line-7569"> Path tableDir = CommonFSUtils.getTableDir(rootDir, info.getTable());</span> |
| <span class="source-line-no">7570</span><span id="line-7570"> HRegion region =</span> |
| <span class="source-line-no">7571</span><span id="line-7571"> HRegion.newHRegion(tableDir, wal, fs, conf, info, hTableDescriptor, rsRpcServices);</span> |
| <span class="source-line-no">7572</span><span id="line-7572"> if (initialize) {</span> |
| <span class="source-line-no">7573</span><span id="line-7573"> region.initialize(null);</span> |
| <span class="source-line-no">7574</span><span id="line-7574"> }</span> |
| <span class="source-line-no">7575</span><span id="line-7575"> return region;</span> |
| <span class="source-line-no">7576</span><span id="line-7576"> }</span> |
| <span class="source-line-no">7577</span><span id="line-7577"></span> |
| <span class="source-line-no">7578</span><span id="line-7578"> /**</span> |
| <span class="source-line-no">7579</span><span id="line-7579"> * Create a region under the given table directory.</span> |
| <span class="source-line-no">7580</span><span id="line-7580"> */</span> |
| <span class="source-line-no">7581</span><span id="line-7581"> public static HRegion createHRegion(Configuration conf, RegionInfo regionInfo, FileSystem fs,</span> |
| <span class="source-line-no">7582</span><span id="line-7582"> Path tableDir, TableDescriptor tableDesc) throws IOException {</span> |
| <span class="source-line-no">7583</span><span id="line-7583"> LOG.info("Creating {}, tableDescriptor={}, under table dir {}", regionInfo, tableDesc,</span> |
| <span class="source-line-no">7584</span><span id="line-7584"> tableDir);</span> |
| <span class="source-line-no">7585</span><span id="line-7585"> HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, regionInfo);</span> |
| <span class="source-line-no">7586</span><span id="line-7586"> HRegion region = HRegion.newHRegion(tableDir, null, fs, conf, regionInfo, tableDesc, null);</span> |
| <span class="source-line-no">7587</span><span id="line-7587"> return region;</span> |
| <span class="source-line-no">7588</span><span id="line-7588"> }</span> |
| <span class="source-line-no">7589</span><span id="line-7589"></span> |
| <span class="source-line-no">7590</span><span id="line-7590"> /**</span> |
| <span class="source-line-no">7591</span><span id="line-7591"> * Create the region directory in the filesystem.</span> |
| <span class="source-line-no">7592</span><span id="line-7592"> */</span> |
| <span class="source-line-no">7593</span><span id="line-7593"> public static HRegionFileSystem createRegionDir(Configuration configuration, RegionInfo ri,</span> |
| <span class="source-line-no">7594</span><span id="line-7594"> Path rootDir) throws IOException {</span> |
| <span class="source-line-no">7595</span><span id="line-7595"> FileSystem fs = rootDir.getFileSystem(configuration);</span> |
| <span class="source-line-no">7596</span><span id="line-7596"> Path tableDir = CommonFSUtils.getTableDir(rootDir, ri.getTable());</span> |
| <span class="source-line-no">7597</span><span id="line-7597"> // If directory already exists, will log warning and keep going. Will try to create</span> |
| <span class="source-line-no">7598</span><span id="line-7598"> // .regioninfo. If one exists, will overwrite.</span> |
| <span class="source-line-no">7599</span><span id="line-7599"> return HRegionFileSystem.createRegionOnFileSystem(configuration, fs, tableDir, ri);</span> |
| <span class="source-line-no">7600</span><span id="line-7600"> }</span> |
| <span class="source-line-no">7601</span><span id="line-7601"></span> |
| <span class="source-line-no">7602</span><span id="line-7602"> public static HRegion createHRegion(final RegionInfo info, final Path rootDir,</span> |
| <span class="source-line-no">7603</span><span id="line-7603"> final Configuration conf, final TableDescriptor hTableDescriptor, final WAL wal)</span> |
| <span class="source-line-no">7604</span><span id="line-7604"> throws IOException {</span> |
| <span class="source-line-no">7605</span><span id="line-7605"> return createHRegion(info, rootDir, conf, hTableDescriptor, wal, true);</span> |
| <span class="source-line-no">7606</span><span id="line-7606"> }</span> |
| <span class="source-line-no">7607</span><span id="line-7607"></span> |
| <span class="source-line-no">7608</span><span id="line-7608"> /**</span> |
| <span class="source-line-no">7609</span><span id="line-7609"> * Open a Region.</span> |
| <span class="source-line-no">7610</span><span id="line-7610"> * @param info Info for region to be opened.</span> |
| <span class="source-line-no">7611</span><span id="line-7611"> * @param wal WAL for region to use. This method will call WAL#setSequenceNumber(long) passing</span> |
| <span class="source-line-no">7612</span><span id="line-7612"> * the result of the call to HRegion#getMinSequenceId() to ensure the wal id is</span> |
| <span class="source-line-no">7613</span><span id="line-7613"> * properly kept up. HRegionStore does this every time it opens a new region.</span> |
| <span class="source-line-no">7614</span><span id="line-7614"> * @return new HRegion</span> |
| <span class="source-line-no">7615</span><span id="line-7615"> */</span> |
| <span class="source-line-no">7616</span><span id="line-7616"> public static HRegion openHRegion(final RegionInfo info, final TableDescriptor htd, final WAL wal,</span> |
| <span class="source-line-no">7617</span><span id="line-7617"> final Configuration conf) throws IOException {</span> |
| <span class="source-line-no">7618</span><span id="line-7618"> return openHRegion(info, htd, wal, conf, null, null);</span> |
| <span class="source-line-no">7619</span><span id="line-7619"> }</span> |
| <span class="source-line-no">7620</span><span id="line-7620"></span> |
| <span class="source-line-no">7621</span><span id="line-7621"> /**</span> |
| <span class="source-line-no">7622</span><span id="line-7622"> * Open a Region.</span> |
| <span class="source-line-no">7623</span><span id="line-7623"> * @param info Info for region to be opened</span> |
| <span class="source-line-no">7624</span><span id="line-7624"> * @param htd the table descriptor</span> |
| <span class="source-line-no">7625</span><span id="line-7625"> * @param wal WAL for region to use. This method will call WAL#setSequenceNumber(long)</span> |
| <span class="source-line-no">7626</span><span id="line-7626"> * passing the result of the call to HRegion#getMinSequenceId() to ensure the</span> |
| <span class="source-line-no">7627</span><span id="line-7627"> * wal id is properly kept up. HRegionStore does this every time it opens a new</span> |
| <span class="source-line-no">7628</span><span id="line-7628"> * region.</span> |
| <span class="source-line-no">7629</span><span id="line-7629"> * @param conf The Configuration object to use.</span> |
| <span class="source-line-no">7630</span><span id="line-7630"> * @param rsServices An interface we can request flushes against.</span> |
| <span class="source-line-no">7631</span><span id="line-7631"> * @param reporter An interface we can report progress against.</span> |
| <span class="source-line-no">7632</span><span id="line-7632"> * @return new HRegion</span> |
| <span class="source-line-no">7633</span><span id="line-7633"> */</span> |
| <span class="source-line-no">7634</span><span id="line-7634"> public static HRegion openHRegion(final RegionInfo info, final TableDescriptor htd, final WAL wal,</span> |
| <span class="source-line-no">7635</span><span id="line-7635"> final Configuration conf, final RegionServerServices rsServices,</span> |
| <span class="source-line-no">7636</span><span id="line-7636"> final CancelableProgressable reporter) throws IOException {</span> |
| <span class="source-line-no">7637</span><span id="line-7637"> return openHRegion(CommonFSUtils.getRootDir(conf), info, htd, wal, conf, rsServices, reporter);</span> |
| <span class="source-line-no">7638</span><span id="line-7638"> }</span> |
| <span class="source-line-no">7639</span><span id="line-7639"></span> |
| <span class="source-line-no">7640</span><span id="line-7640"> /**</span> |
| <span class="source-line-no">7641</span><span id="line-7641"> * Open a Region.</span> |
| <span class="source-line-no">7642</span><span id="line-7642"> * @param rootDir Root directory for HBase instance</span> |
| <span class="source-line-no">7643</span><span id="line-7643"> * @param info Info for region to be opened.</span> |
| <span class="source-line-no">7644</span><span id="line-7644"> * @param htd the table descriptor</span> |
| <span class="source-line-no">7645</span><span id="line-7645"> * @param wal WAL for region to use. This method will call WAL#setSequenceNumber(long) passing</span> |
| <span class="source-line-no">7646</span><span id="line-7646"> * the result of the call to HRegion#getMinSequenceId() to ensure the wal id is</span> |
| <span class="source-line-no">7647</span><span id="line-7647"> * properly kept up. HRegionStore does this every time it opens a new region.</span> |
| <span class="source-line-no">7648</span><span id="line-7648"> * @param conf The Configuration object to use.</span> |
| <span class="source-line-no">7649</span><span id="line-7649"> * @return new HRegion</span> |
| <span class="source-line-no">7650</span><span id="line-7650"> */</span> |
| <span class="source-line-no">7651</span><span id="line-7651"> public static HRegion openHRegion(Path rootDir, final RegionInfo info, final TableDescriptor htd,</span> |
| <span class="source-line-no">7652</span><span id="line-7652"> final WAL wal, final Configuration conf) throws IOException {</span> |
| <span class="source-line-no">7653</span><span id="line-7653"> return openHRegion(rootDir, info, htd, wal, conf, null, null);</span> |
| <span class="source-line-no">7654</span><span id="line-7654"> }</span> |
| <span class="source-line-no">7655</span><span id="line-7655"></span> |
| <span class="source-line-no">7656</span><span id="line-7656"> /**</span> |
| <span class="source-line-no">7657</span><span id="line-7657"> * Open a Region.</span> |
| <span class="source-line-no">7658</span><span id="line-7658"> * @param rootDir Root directory for HBase instance</span> |
| <span class="source-line-no">7659</span><span id="line-7659"> * @param info Info for region to be opened.</span> |
| <span class="source-line-no">7660</span><span id="line-7660"> * @param htd the table descriptor</span> |
| <span class="source-line-no">7661</span><span id="line-7661"> * @param wal WAL for region to use. This method will call WAL#setSequenceNumber(long)</span> |
| <span class="source-line-no">7662</span><span id="line-7662"> * passing the result of the call to HRegion#getMinSequenceId() to ensure the</span> |
| <span class="source-line-no">7663</span><span id="line-7663"> * wal id is properly kept up. HRegionStore does this every time it opens a new</span> |
| <span class="source-line-no">7664</span><span id="line-7664"> * region.</span> |
| <span class="source-line-no">7665</span><span id="line-7665"> * @param conf The Configuration object to use.</span> |
| <span class="source-line-no">7666</span><span id="line-7666"> * @param rsServices An interface we can request flushes against.</span> |
| <span class="source-line-no">7667</span><span id="line-7667"> * @param reporter An interface we can report progress against.</span> |
| <span class="source-line-no">7668</span><span id="line-7668"> * @return new HRegion</span> |
| <span class="source-line-no">7669</span><span id="line-7669"> */</span> |
| <span class="source-line-no">7670</span><span id="line-7670"> public static HRegion openHRegion(final Path rootDir, final RegionInfo info,</span> |
| <span class="source-line-no">7671</span><span id="line-7671"> final TableDescriptor htd, final WAL wal, final Configuration conf,</span> |
| <span class="source-line-no">7672</span><span id="line-7672"> final RegionServerServices rsServices, final CancelableProgressable reporter)</span> |
| <span class="source-line-no">7673</span><span id="line-7673"> throws IOException {</span> |
| <span class="source-line-no">7674</span><span id="line-7674"> FileSystem fs = null;</span> |
| <span class="source-line-no">7675</span><span id="line-7675"> if (rsServices != null) {</span> |
| <span class="source-line-no">7676</span><span id="line-7676"> fs = rsServices.getFileSystem();</span> |
| <span class="source-line-no">7677</span><span id="line-7677"> }</span> |
| <span class="source-line-no">7678</span><span id="line-7678"> if (fs == null) {</span> |
| <span class="source-line-no">7679</span><span id="line-7679"> fs = rootDir.getFileSystem(conf);</span> |
| <span class="source-line-no">7680</span><span id="line-7680"> }</span> |
| <span class="source-line-no">7681</span><span id="line-7681"> return openHRegion(conf, fs, rootDir, info, htd, wal, rsServices, reporter);</span> |
| <span class="source-line-no">7682</span><span id="line-7682"> }</span> |
| <span class="source-line-no">7683</span><span id="line-7683"></span> |
| <span class="source-line-no">7684</span><span id="line-7684"> /**</span> |
| <span class="source-line-no">7685</span><span id="line-7685"> * Open a Region.</span> |
| <span class="source-line-no">7686</span><span id="line-7686"> * @param conf The Configuration object to use.</span> |
| <span class="source-line-no">7687</span><span id="line-7687"> * @param fs Filesystem to use</span> |
| <span class="source-line-no">7688</span><span id="line-7688"> * @param rootDir Root directory for HBase instance</span> |
| <span class="source-line-no">7689</span><span id="line-7689"> * @param info Info for region to be opened.</span> |
| <span class="source-line-no">7690</span><span id="line-7690"> * @param htd the table descriptor</span> |
| <span class="source-line-no">7691</span><span id="line-7691"> * @param wal WAL for region to use. This method will call WAL#setSequenceNumber(long) passing</span> |
| <span class="source-line-no">7692</span><span id="line-7692"> * the result of the call to HRegion#getMinSequenceId() to ensure the wal id is</span> |
| <span class="source-line-no">7693</span><span id="line-7693"> * properly kept up. HRegionStore does this every time it opens a new region.</span> |
| <span class="source-line-no">7694</span><span id="line-7694"> * @return new HRegion</span> |
| <span class="source-line-no">7695</span><span id="line-7695"> */</span> |
| <span class="source-line-no">7696</span><span id="line-7696"> public static HRegion openHRegion(final Configuration conf, final FileSystem fs,</span> |
| <span class="source-line-no">7697</span><span id="line-7697"> final Path rootDir, final RegionInfo info, final TableDescriptor htd, final WAL wal)</span> |
| <span class="source-line-no">7698</span><span id="line-7698"> throws IOException {</span> |
| <span class="source-line-no">7699</span><span id="line-7699"> return openHRegion(conf, fs, rootDir, info, htd, wal, null, null);</span> |
| <span class="source-line-no">7700</span><span id="line-7700"> }</span> |
| <span class="source-line-no">7701</span><span id="line-7701"></span> |
| <span class="source-line-no">7702</span><span id="line-7702"> /**</span> |
| <span class="source-line-no">7703</span><span id="line-7703"> * Open a Region.</span> |
| <span class="source-line-no">7704</span><span id="line-7704"> * @param conf The Configuration object to use.</span> |
| <span class="source-line-no">7705</span><span id="line-7705"> * @param fs Filesystem to use</span> |
| <span class="source-line-no">7706</span><span id="line-7706"> * @param rootDir Root directory for HBase instance</span> |
| <span class="source-line-no">7707</span><span id="line-7707"> * @param info Info for region to be opened.</span> |
| <span class="source-line-no">7708</span><span id="line-7708"> * @param htd the table descriptor</span> |
| <span class="source-line-no">7709</span><span id="line-7709"> * @param wal WAL for region to use. This method will call WAL#setSequenceNumber(long)</span> |
| <span class="source-line-no">7710</span><span id="line-7710"> * passing the result of the call to HRegion#getMinSequenceId() to ensure the</span> |
| <span class="source-line-no">7711</span><span id="line-7711"> * wal id is properly kept up. HRegionStore does this every time it opens a new</span> |
| <span class="source-line-no">7712</span><span id="line-7712"> * region.</span> |
| <span class="source-line-no">7713</span><span id="line-7713"> * @param rsServices An interface we can request flushes against.</span> |
| <span class="source-line-no">7714</span><span id="line-7714"> * @param reporter An interface we can report progress against.</span> |
| <span class="source-line-no">7715</span><span id="line-7715"> * @return new HRegion</span> |
| <span class="source-line-no">7716</span><span id="line-7716"> */</span> |
| <span class="source-line-no">7717</span><span id="line-7717"> public static HRegion openHRegion(final Configuration conf, final FileSystem fs,</span> |
| <span class="source-line-no">7718</span><span id="line-7718"> final Path rootDir, final RegionInfo info, final TableDescriptor htd, final WAL wal,</span> |
| <span class="source-line-no">7719</span><span id="line-7719"> final RegionServerServices rsServices, final CancelableProgressable reporter)</span> |
| <span class="source-line-no">7720</span><span id="line-7720"> throws IOException {</span> |
| <span class="source-line-no">7721</span><span id="line-7721"> Path tableDir = CommonFSUtils.getTableDir(rootDir, info.getTable());</span> |
| <span class="source-line-no">7722</span><span id="line-7722"> return openHRegionFromTableDir(conf, fs, tableDir, info, htd, wal, rsServices, reporter);</span> |
| <span class="source-line-no">7723</span><span id="line-7723"> }</span> |
| <span class="source-line-no">7724</span><span id="line-7724"></span> |
| <span class="source-line-no">7725</span><span id="line-7725"> /**</span> |
| <span class="source-line-no">7726</span><span id="line-7726"> * Open a Region.</span> |
| <span class="source-line-no">7727</span><span id="line-7727"> * @param conf The Configuration object to use.</span> |
| <span class="source-line-no">7728</span><span id="line-7728"> * @param fs Filesystem to use</span> |
| <span class="source-line-no">7729</span><span id="line-7729"> * @param info Info for region to be opened.</span> |
| <span class="source-line-no">7730</span><span id="line-7730"> * @param htd the table descriptor</span> |
| <span class="source-line-no">7731</span><span id="line-7731"> * @param wal WAL for region to use. This method will call WAL#setSequenceNumber(long)</span> |
| <span class="source-line-no">7732</span><span id="line-7732"> * passing the result of the call to HRegion#getMinSequenceId() to ensure the</span> |
| <span class="source-line-no">7733</span><span id="line-7733"> * wal id is properly kept up. HRegionStore does this every time it opens a new</span> |
| <span class="source-line-no">7734</span><span id="line-7734"> * region.</span> |
| <span class="source-line-no">7735</span><span id="line-7735"> * @param rsServices An interface we can request flushes against.</span> |
| <span class="source-line-no">7736</span><span id="line-7736"> * @param reporter An interface we can report progress against.</span> |
| <span class="source-line-no">7737</span><span id="line-7737"> * @return new HRegion</span> |
| <span class="source-line-no">7738</span><span id="line-7738"> * @throws NullPointerException if {@code info} is {@code null}</span> |
| <span class="source-line-no">7739</span><span id="line-7739"> */</span> |
| <span class="source-line-no">7740</span><span id="line-7740"> public static HRegion openHRegionFromTableDir(final Configuration conf, final FileSystem fs,</span> |
| <span class="source-line-no">7741</span><span id="line-7741"> final Path tableDir, final RegionInfo info, final TableDescriptor htd, final WAL wal,</span> |
| <span class="source-line-no">7742</span><span id="line-7742"> final RegionServerServices rsServices, final CancelableProgressable reporter)</span> |
| <span class="source-line-no">7743</span><span id="line-7743"> throws IOException {</span> |
| <span class="source-line-no">7744</span><span id="line-7744"> Objects.requireNonNull(info, "RegionInfo cannot be null");</span> |
| <span class="source-line-no">7745</span><span id="line-7745"> LOG.debug("Opening region: {}", info);</span> |
| <span class="source-line-no">7746</span><span id="line-7746"> HRegion r = HRegion.newHRegion(tableDir, wal, fs, conf, info, htd, rsServices);</span> |
| <span class="source-line-no">7747</span><span id="line-7747"> return r.openHRegion(reporter);</span> |
| <span class="source-line-no">7748</span><span id="line-7748"> }</span> |
| <span class="source-line-no">7749</span><span id="line-7749"></span> |
| <span class="source-line-no">7750</span><span id="line-7750"> public NavigableMap<byte[], Integer> getReplicationScope() {</span> |
| <span class="source-line-no">7751</span><span id="line-7751"> return this.replicationScope;</span> |
| <span class="source-line-no">7752</span><span id="line-7752"> }</span> |
| <span class="source-line-no">7753</span><span id="line-7753"></span> |
| <span class="source-line-no">7754</span><span id="line-7754"> /**</span> |
| <span class="source-line-no">7755</span><span id="line-7755"> * Useful when reopening a closed region (normally for unit tests)</span> |
| <span class="source-line-no">7756</span><span id="line-7756"> * @param other original object</span> |
| <span class="source-line-no">7757</span><span id="line-7757"> * @param reporter An interface we can report progress against.</span> |
| <span class="source-line-no">7758</span><span id="line-7758"> * @return new HRegion</span> |
| <span class="source-line-no">7759</span><span id="line-7759"> */</span> |
| <span class="source-line-no">7760</span><span id="line-7760"> public static HRegion openHRegion(final HRegion other, final CancelableProgressable reporter)</span> |
| <span class="source-line-no">7761</span><span id="line-7761"> throws IOException {</span> |
| <span class="source-line-no">7762</span><span id="line-7762"> HRegionFileSystem regionFs = other.getRegionFileSystem();</span> |
| <span class="source-line-no">7763</span><span id="line-7763"> HRegion r = newHRegion(regionFs.getTableDir(), other.getWAL(), regionFs.getFileSystem(),</span> |
| <span class="source-line-no">7764</span><span id="line-7764"> other.baseConf, other.getRegionInfo(), other.getTableDescriptor(), null);</span> |
| <span class="source-line-no">7765</span><span id="line-7765"> return r.openHRegion(reporter);</span> |
| <span class="source-line-no">7766</span><span id="line-7766"> }</span> |
| <span class="source-line-no">7767</span><span id="line-7767"></span> |
| <span class="source-line-no">7768</span><span id="line-7768"> public static Region openHRegion(final Region other, final CancelableProgressable reporter)</span> |
| <span class="source-line-no">7769</span><span id="line-7769"> throws IOException {</span> |
| <span class="source-line-no">7770</span><span id="line-7770"> return openHRegion((HRegion) other, reporter);</span> |
| <span class="source-line-no">7771</span><span id="line-7771"> }</span> |
| <span class="source-line-no">7772</span><span id="line-7772"></span> |
| <span class="source-line-no">7773</span><span id="line-7773"> /**</span> |
| <span class="source-line-no">7774</span><span id="line-7774"> * Open HRegion.</span> |
| <span class="source-line-no">7775</span><span id="line-7775"> * <p/></span> |
| <span class="source-line-no">7776</span><span id="line-7776"> * Calls initialize and sets sequenceId.</span> |
| <span class="source-line-no">7777</span><span id="line-7777"> * @return Returns <code>this</code></span> |
| <span class="source-line-no">7778</span><span id="line-7778"> */</span> |
| <span class="source-line-no">7779</span><span id="line-7779"> private HRegion openHRegion(final CancelableProgressable reporter) throws IOException {</span> |
| <span class="source-line-no">7780</span><span id="line-7780"> try {</span> |
| <span class="source-line-no">7781</span><span id="line-7781"> CompoundConfiguration cConfig =</span> |
| <span class="source-line-no">7782</span><span id="line-7782"> new CompoundConfiguration().add(conf).addBytesMap(htableDescriptor.getValues());</span> |
| <span class="source-line-no">7783</span><span id="line-7783"> // Refuse to open the region if we are missing local compression support</span> |
| <span class="source-line-no">7784</span><span id="line-7784"> TableDescriptorChecker.checkCompression(cConfig, htableDescriptor);</span> |
| <span class="source-line-no">7785</span><span id="line-7785"> // Refuse to open the region if encryption configuration is incorrect or</span> |
| <span class="source-line-no">7786</span><span id="line-7786"> // codec support is missing</span> |
| <span class="source-line-no">7787</span><span id="line-7787"> LOG.debug("checking encryption for " + this.getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">7788</span><span id="line-7788"> TableDescriptorChecker.checkEncryption(cConfig, htableDescriptor);</span> |
| <span class="source-line-no">7789</span><span id="line-7789"> // Refuse to open the region if a required class cannot be loaded</span> |
| <span class="source-line-no">7790</span><span id="line-7790"> LOG.debug("checking classloading for " + this.getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">7791</span><span id="line-7791"> TableDescriptorChecker.checkClassLoading(cConfig, htableDescriptor);</span> |
| <span class="source-line-no">7792</span><span id="line-7792"> this.openSeqNum = initialize(reporter);</span> |
| <span class="source-line-no">7793</span><span id="line-7793"> this.mvcc.advanceTo(openSeqNum);</span> |
| <span class="source-line-no">7794</span><span id="line-7794"> // The openSeqNum must be increased every time when a region is assigned, as we rely on it to</span> |
| <span class="source-line-no">7795</span><span id="line-7795"> // determine whether a region has been successfully reopened. So here we always write open</span> |
| <span class="source-line-no">7796</span><span id="line-7796"> // marker, even if the table is read only.</span> |
| <span class="source-line-no">7797</span><span id="line-7797"> if (</span> |
| <span class="source-line-no">7798</span><span id="line-7798"> wal != null && getRegionServerServices() != null</span> |
| <span class="source-line-no">7799</span><span id="line-7799"> && RegionReplicaUtil.isDefaultReplica(getRegionInfo())</span> |
| <span class="source-line-no">7800</span><span id="line-7800"> ) {</span> |
| <span class="source-line-no">7801</span><span id="line-7801"> writeRegionOpenMarker(wal, openSeqNum);</span> |
| <span class="source-line-no">7802</span><span id="line-7802"> }</span> |
| <span class="source-line-no">7803</span><span id="line-7803"> } catch (Throwable t) {</span> |
| <span class="source-line-no">7804</span><span id="line-7804"> // By coprocessor path wrong region will open failed,</span> |
| <span class="source-line-no">7805</span><span id="line-7805"> // MetricsRegionWrapperImpl is already init and not close,</span> |
| <span class="source-line-no">7806</span><span id="line-7806"> // add region close when open failed</span> |
| <span class="source-line-no">7807</span><span id="line-7807"> try {</span> |
| <span class="source-line-no">7808</span><span id="line-7808"> // It is not required to write sequence id file when region open is failed.</span> |
| <span class="source-line-no">7809</span><span id="line-7809"> // Passing true to skip the sequence id file write.</span> |
| <span class="source-line-no">7810</span><span id="line-7810"> this.close(true);</span> |
| <span class="source-line-no">7811</span><span id="line-7811"> } catch (Throwable e) {</span> |
| <span class="source-line-no">7812</span><span id="line-7812"> LOG.warn("Open region: {} failed. Try close region but got exception ",</span> |
| <span class="source-line-no">7813</span><span id="line-7813"> this.getRegionInfo(), e);</span> |
| <span class="source-line-no">7814</span><span id="line-7814"> }</span> |
| <span class="source-line-no">7815</span><span id="line-7815"> throw t;</span> |
| <span class="source-line-no">7816</span><span id="line-7816"> }</span> |
| <span class="source-line-no">7817</span><span id="line-7817"> return this;</span> |
| <span class="source-line-no">7818</span><span id="line-7818"> }</span> |
| <span class="source-line-no">7819</span><span id="line-7819"></span> |
| <span class="source-line-no">7820</span><span id="line-7820"> /**</span> |
| <span class="source-line-no">7821</span><span id="line-7821"> * Open a Region on a read-only file-system (like hdfs snapshots)</span> |
| <span class="source-line-no">7822</span><span id="line-7822"> * @param conf The Configuration object to use.</span> |
| <span class="source-line-no">7823</span><span id="line-7823"> * @param fs Filesystem to use</span> |
| <span class="source-line-no">7824</span><span id="line-7824"> * @param info Info for region to be opened.</span> |
| <span class="source-line-no">7825</span><span id="line-7825"> * @param htd the table descriptor</span> |
| <span class="source-line-no">7826</span><span id="line-7826"> * @return new HRegion</span> |
| <span class="source-line-no">7827</span><span id="line-7827"> * @throws NullPointerException if {@code info} is {@code null}</span> |
| <span class="source-line-no">7828</span><span id="line-7828"> */</span> |
| <span class="source-line-no">7829</span><span id="line-7829"> public static HRegion openReadOnlyFileSystemHRegion(final Configuration conf, final FileSystem fs,</span> |
| <span class="source-line-no">7830</span><span id="line-7830"> final Path tableDir, RegionInfo info, final TableDescriptor htd) throws IOException {</span> |
| <span class="source-line-no">7831</span><span id="line-7831"> Objects.requireNonNull(info, "RegionInfo cannot be null");</span> |
| <span class="source-line-no">7832</span><span id="line-7832"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">7833</span><span id="line-7833"> LOG.debug("Opening region (readOnly filesystem): " + info);</span> |
| <span class="source-line-no">7834</span><span id="line-7834"> }</span> |
| <span class="source-line-no">7835</span><span id="line-7835"> if (info.getReplicaId() <= 0) {</span> |
| <span class="source-line-no">7836</span><span id="line-7836"> info = RegionReplicaUtil.getRegionInfoForReplica(info, 1);</span> |
| <span class="source-line-no">7837</span><span id="line-7837"> }</span> |
| <span class="source-line-no">7838</span><span id="line-7838"> HRegion r = HRegion.newHRegion(tableDir, null, fs, conf, info, htd, null);</span> |
| <span class="source-line-no">7839</span><span id="line-7839"> r.writestate.setReadOnly(true);</span> |
| <span class="source-line-no">7840</span><span id="line-7840"> return r.openHRegion(null);</span> |
| <span class="source-line-no">7841</span><span id="line-7841"> }</span> |
| <span class="source-line-no">7842</span><span id="line-7842"></span> |
| <span class="source-line-no">7843</span><span id="line-7843"> public static HRegion warmupHRegion(final RegionInfo info, final TableDescriptor htd,</span> |
| <span class="source-line-no">7844</span><span id="line-7844"> final WAL wal, final Configuration conf, final RegionServerServices rsServices,</span> |
| <span class="source-line-no">7845</span><span id="line-7845"> final CancelableProgressable reporter) throws IOException {</span> |
| <span class="source-line-no">7846</span><span id="line-7846"></span> |
| <span class="source-line-no">7847</span><span id="line-7847"> Objects.requireNonNull(info, "RegionInfo cannot be null");</span> |
| <span class="source-line-no">7848</span><span id="line-7848"> LOG.debug("Warmup {}", info);</span> |
| <span class="source-line-no">7849</span><span id="line-7849"> Path rootDir = CommonFSUtils.getRootDir(conf);</span> |
| <span class="source-line-no">7850</span><span id="line-7850"> Path tableDir = CommonFSUtils.getTableDir(rootDir, info.getTable());</span> |
| <span class="source-line-no">7851</span><span id="line-7851"> FileSystem fs = null;</span> |
| <span class="source-line-no">7852</span><span id="line-7852"> if (rsServices != null) {</span> |
| <span class="source-line-no">7853</span><span id="line-7853"> fs = rsServices.getFileSystem();</span> |
| <span class="source-line-no">7854</span><span id="line-7854"> }</span> |
| <span class="source-line-no">7855</span><span id="line-7855"> if (fs == null) {</span> |
| <span class="source-line-no">7856</span><span id="line-7856"> fs = rootDir.getFileSystem(conf);</span> |
| <span class="source-line-no">7857</span><span id="line-7857"> }</span> |
| <span class="source-line-no">7858</span><span id="line-7858"> HRegion r = HRegion.newHRegion(tableDir, wal, fs, conf, info, htd, null);</span> |
| <span class="source-line-no">7859</span><span id="line-7859"> r.initializeWarmup(reporter);</span> |
| <span class="source-line-no">7860</span><span id="line-7860"> r.close();</span> |
| <span class="source-line-no">7861</span><span id="line-7861"> return r;</span> |
| <span class="source-line-no">7862</span><span id="line-7862"> }</span> |
| <span class="source-line-no">7863</span><span id="line-7863"></span> |
| <span class="source-line-no">7864</span><span id="line-7864"> /**</span> |
| <span class="source-line-no">7865</span><span id="line-7865"> * Computes the Path of the HRegion</span> |
| <span class="source-line-no">7866</span><span id="line-7866"> * @param tabledir qualified path for table</span> |
| <span class="source-line-no">7867</span><span id="line-7867"> * @param name ENCODED region name</span> |
| <span class="source-line-no">7868</span><span id="line-7868"> * @return Path of HRegion directory</span> |
| <span class="source-line-no">7869</span><span id="line-7869"> * @deprecated For tests only; to be removed.</span> |
| <span class="source-line-no">7870</span><span id="line-7870"> */</span> |
| <span class="source-line-no">7871</span><span id="line-7871"> @Deprecated</span> |
| <span class="source-line-no">7872</span><span id="line-7872"> public static Path getRegionDir(final Path tabledir, final String name) {</span> |
| <span class="source-line-no">7873</span><span id="line-7873"> return new Path(tabledir, name);</span> |
| <span class="source-line-no">7874</span><span id="line-7874"> }</span> |
| <span class="source-line-no">7875</span><span id="line-7875"></span> |
| <span class="source-line-no">7876</span><span id="line-7876"> /**</span> |
| <span class="source-line-no">7877</span><span id="line-7877"> * Determines if the specified row is within the row range specified by the specified RegionInfo</span> |
| <span class="source-line-no">7878</span><span id="line-7878"> * @param info RegionInfo that specifies the row range</span> |
| <span class="source-line-no">7879</span><span id="line-7879"> * @param row row to be checked</span> |
| <span class="source-line-no">7880</span><span id="line-7880"> * @return true if the row is within the range specified by the RegionInfo</span> |
| <span class="source-line-no">7881</span><span id="line-7881"> */</span> |
| <span class="source-line-no">7882</span><span id="line-7882"> public static boolean rowIsInRange(RegionInfo info, final byte[] row) {</span> |
| <span class="source-line-no">7883</span><span id="line-7883"> return ((info.getStartKey().length == 0) || (Bytes.compareTo(info.getStartKey(), row) <= 0))</span> |
| <span class="source-line-no">7884</span><span id="line-7884"> && ((info.getEndKey().length == 0) || (Bytes.compareTo(info.getEndKey(), row) > 0));</span> |
| <span class="source-line-no">7885</span><span id="line-7885"> }</span> |
| <span class="source-line-no">7886</span><span id="line-7886"></span> |
| <span class="source-line-no">7887</span><span id="line-7887"> public static boolean rowIsInRange(RegionInfo info, final byte[] row, final int offset,</span> |
| <span class="source-line-no">7888</span><span id="line-7888"> final short length) {</span> |
| <span class="source-line-no">7889</span><span id="line-7889"> return ((info.getStartKey().length == 0)</span> |
| <span class="source-line-no">7890</span><span id="line-7890"> || (Bytes.compareTo(info.getStartKey(), 0, info.getStartKey().length, row, offset, length)</span> |
| <span class="source-line-no">7891</span><span id="line-7891"> <= 0))</span> |
| <span class="source-line-no">7892</span><span id="line-7892"> && ((info.getEndKey().length == 0)</span> |
| <span class="source-line-no">7893</span><span id="line-7893"> || (Bytes.compareTo(info.getEndKey(), 0, info.getEndKey().length, row, offset, length)</span> |
| <span class="source-line-no">7894</span><span id="line-7894"> > 0));</span> |
| <span class="source-line-no">7895</span><span id="line-7895"> }</span> |
| <span class="source-line-no">7896</span><span id="line-7896"></span> |
| <span class="source-line-no">7897</span><span id="line-7897"> @Override</span> |
| <span class="source-line-no">7898</span><span id="line-7898"> public Result get(final Get get) throws IOException {</span> |
| <span class="source-line-no">7899</span><span id="line-7899"> prepareGet(get);</span> |
| <span class="source-line-no">7900</span><span id="line-7900"> List<Cell> results = get(get, true);</span> |
| <span class="source-line-no">7901</span><span id="line-7901"> boolean stale = this.getRegionInfo().getReplicaId() != 0;</span> |
| <span class="source-line-no">7902</span><span id="line-7902"> return Result.create(results, get.isCheckExistenceOnly() ? !results.isEmpty() : null, stale);</span> |
| <span class="source-line-no">7903</span><span id="line-7903"> }</span> |
| <span class="source-line-no">7904</span><span id="line-7904"></span> |
| <span class="source-line-no">7905</span><span id="line-7905"> void prepareGet(final Get get) throws IOException {</span> |
| <span class="source-line-no">7906</span><span id="line-7906"> checkRow(get.getRow(), "Get");</span> |
| <span class="source-line-no">7907</span><span id="line-7907"> // Verify families are all valid</span> |
| <span class="source-line-no">7908</span><span id="line-7908"> if (get.hasFamilies()) {</span> |
| <span class="source-line-no">7909</span><span id="line-7909"> for (byte[] family : get.familySet()) {</span> |
| <span class="source-line-no">7910</span><span id="line-7910"> checkFamily(family);</span> |
| <span class="source-line-no">7911</span><span id="line-7911"> }</span> |
| <span class="source-line-no">7912</span><span id="line-7912"> } else { // Adding all families to scanner</span> |
| <span class="source-line-no">7913</span><span id="line-7913"> for (byte[] family : this.htableDescriptor.getColumnFamilyNames()) {</span> |
| <span class="source-line-no">7914</span><span id="line-7914"> get.addFamily(family);</span> |
| <span class="source-line-no">7915</span><span id="line-7915"> }</span> |
| <span class="source-line-no">7916</span><span id="line-7916"> }</span> |
| <span class="source-line-no">7917</span><span id="line-7917"> }</span> |
| <span class="source-line-no">7918</span><span id="line-7918"></span> |
| <span class="source-line-no">7919</span><span id="line-7919"> @Override</span> |
| <span class="source-line-no">7920</span><span id="line-7920"> public List<Cell> get(Get get, boolean withCoprocessor) throws IOException {</span> |
| <span class="source-line-no">7921</span><span id="line-7921"> return get(get, withCoprocessor, HConstants.NO_NONCE, HConstants.NO_NONCE);</span> |
| <span class="source-line-no">7922</span><span id="line-7922"> }</span> |
| <span class="source-line-no">7923</span><span id="line-7923"></span> |
| <span class="source-line-no">7924</span><span id="line-7924"> private List<Cell> get(Get get, boolean withCoprocessor, long nonceGroup, long nonce)</span> |
| <span class="source-line-no">7925</span><span id="line-7925"> throws IOException {</span> |
| <span class="source-line-no">7926</span><span id="line-7926"> return TraceUtil.trace(() -> getInternal(get, withCoprocessor, nonceGroup, nonce),</span> |
| <span class="source-line-no">7927</span><span id="line-7927"> () -> createRegionSpan("Region.get"));</span> |
| <span class="source-line-no">7928</span><span id="line-7928"> }</span> |
| <span class="source-line-no">7929</span><span id="line-7929"></span> |
| <span class="source-line-no">7930</span><span id="line-7930"> private List<Cell> getInternal(Get get, boolean withCoprocessor, long nonceGroup, long nonce)</span> |
| <span class="source-line-no">7931</span><span id="line-7931"> throws IOException {</span> |
| <span class="source-line-no">7932</span><span id="line-7932"> List<Cell> results = new ArrayList<>();</span> |
| <span class="source-line-no">7933</span><span id="line-7933"></span> |
| <span class="source-line-no">7934</span><span id="line-7934"> // pre-get CP hook</span> |
| <span class="source-line-no">7935</span><span id="line-7935"> if (withCoprocessor && (coprocessorHost != null)) {</span> |
| <span class="source-line-no">7936</span><span id="line-7936"> if (coprocessorHost.preGet(get, results)) {</span> |
| <span class="source-line-no">7937</span><span id="line-7937"> metricsUpdateForGet();</span> |
| <span class="source-line-no">7938</span><span id="line-7938"> return results;</span> |
| <span class="source-line-no">7939</span><span id="line-7939"> }</span> |
| <span class="source-line-no">7940</span><span id="line-7940"> }</span> |
| <span class="source-line-no">7941</span><span id="line-7941"> Scan scan = new Scan(get);</span> |
| <span class="source-line-no">7942</span><span id="line-7942"> if (scan.getLoadColumnFamiliesOnDemandValue() == null) {</span> |
| <span class="source-line-no">7943</span><span id="line-7943"> scan.setLoadColumnFamiliesOnDemand(isLoadingCfsOnDemandDefault());</span> |
| <span class="source-line-no">7944</span><span id="line-7944"> }</span> |
| <span class="source-line-no">7945</span><span id="line-7945"> try (RegionScanner scanner = getScanner(scan, null, nonceGroup, nonce)) {</span> |
| <span class="source-line-no">7946</span><span id="line-7946"> List<Cell> tmp = new ArrayList<>();</span> |
| <span class="source-line-no">7947</span><span id="line-7947"> scanner.next(tmp);</span> |
| <span class="source-line-no">7948</span><span id="line-7948"> // Copy EC to heap, then close the scanner.</span> |
| <span class="source-line-no">7949</span><span id="line-7949"> // This can be an EXPENSIVE call. It may make an extra copy from offheap to onheap buffers.</span> |
| <span class="source-line-no">7950</span><span id="line-7950"> // See more details in HBASE-26036.</span> |
| <span class="source-line-no">7951</span><span id="line-7951"> for (Cell cell : tmp) {</span> |
| <span class="source-line-no">7952</span><span id="line-7952"> results.add(CellUtil.cloneIfNecessary(cell));</span> |
| <span class="source-line-no">7953</span><span id="line-7953"> }</span> |
| <span class="source-line-no">7954</span><span id="line-7954"> }</span> |
| <span class="source-line-no">7955</span><span id="line-7955"></span> |
| <span class="source-line-no">7956</span><span id="line-7956"> // post-get CP hook</span> |
| <span class="source-line-no">7957</span><span id="line-7957"> if (withCoprocessor && (coprocessorHost != null)) {</span> |
| <span class="source-line-no">7958</span><span id="line-7958"> coprocessorHost.postGet(get, results);</span> |
| <span class="source-line-no">7959</span><span id="line-7959"> }</span> |
| <span class="source-line-no">7960</span><span id="line-7960"></span> |
| <span class="source-line-no">7961</span><span id="line-7961"> metricsUpdateForGet();</span> |
| <span class="source-line-no">7962</span><span id="line-7962"></span> |
| <span class="source-line-no">7963</span><span id="line-7963"> return results;</span> |
| <span class="source-line-no">7964</span><span id="line-7964"> }</span> |
| <span class="source-line-no">7965</span><span id="line-7965"></span> |
| <span class="source-line-no">7966</span><span id="line-7966"> void metricsUpdateForGet() {</span> |
| <span class="source-line-no">7967</span><span id="line-7967"> if (this.metricsRegion != null) {</span> |
| <span class="source-line-no">7968</span><span id="line-7968"> this.metricsRegion.updateGet();</span> |
| <span class="source-line-no">7969</span><span id="line-7969"> }</span> |
| <span class="source-line-no">7970</span><span id="line-7970"> if (this.rsServices != null && this.rsServices.getMetrics() != null) {</span> |
| <span class="source-line-no">7971</span><span id="line-7971"> rsServices.getMetrics().updateReadQueryMeter(this, 1);</span> |
| <span class="source-line-no">7972</span><span id="line-7972"> }</span> |
| <span class="source-line-no">7973</span><span id="line-7973"></span> |
| <span class="source-line-no">7974</span><span id="line-7974"> }</span> |
| <span class="source-line-no">7975</span><span id="line-7975"></span> |
| <span class="source-line-no">7976</span><span id="line-7976"> @Override</span> |
| <span class="source-line-no">7977</span><span id="line-7977"> public Result mutateRow(RowMutations rm) throws IOException {</span> |
| <span class="source-line-no">7978</span><span id="line-7978"> return mutateRow(rm, HConstants.NO_NONCE, HConstants.NO_NONCE);</span> |
| <span class="source-line-no">7979</span><span id="line-7979"> }</span> |
| <span class="source-line-no">7980</span><span id="line-7980"></span> |
| <span class="source-line-no">7981</span><span id="line-7981"> public Result mutateRow(RowMutations rm, long nonceGroup, long nonce) throws IOException {</span> |
| <span class="source-line-no">7982</span><span id="line-7982"> final List<Mutation> m = rm.getMutations();</span> |
| <span class="source-line-no">7983</span><span id="line-7983"> OperationStatus[] statuses = batchMutate(m.toArray(new Mutation[0]), true, nonceGroup, nonce);</span> |
| <span class="source-line-no">7984</span><span id="line-7984"></span> |
| <span class="source-line-no">7985</span><span id="line-7985"> List<Result> results = new ArrayList<>();</span> |
| <span class="source-line-no">7986</span><span id="line-7986"> for (OperationStatus status : statuses) {</span> |
| <span class="source-line-no">7987</span><span id="line-7987"> if (status.getResult() != null) {</span> |
| <span class="source-line-no">7988</span><span id="line-7988"> results.add(status.getResult());</span> |
| <span class="source-line-no">7989</span><span id="line-7989"> }</span> |
| <span class="source-line-no">7990</span><span id="line-7990"> }</span> |
| <span class="source-line-no">7991</span><span id="line-7991"></span> |
| <span class="source-line-no">7992</span><span id="line-7992"> if (results.isEmpty()) {</span> |
| <span class="source-line-no">7993</span><span id="line-7993"> return null;</span> |
| <span class="source-line-no">7994</span><span id="line-7994"> }</span> |
| <span class="source-line-no">7995</span><span id="line-7995"></span> |
| <span class="source-line-no">7996</span><span id="line-7996"> // Merge the results of the Increment/Append operations</span> |
| <span class="source-line-no">7997</span><span id="line-7997"> List<Cell> cells = new ArrayList<>();</span> |
| <span class="source-line-no">7998</span><span id="line-7998"> for (Result result : results) {</span> |
| <span class="source-line-no">7999</span><span id="line-7999"> if (result.rawCells() != null) {</span> |
| <span class="source-line-no">8000</span><span id="line-8000"> cells.addAll(Arrays.asList(result.rawCells()));</span> |
| <span class="source-line-no">8001</span><span id="line-8001"> }</span> |
| <span class="source-line-no">8002</span><span id="line-8002"> }</span> |
| <span class="source-line-no">8003</span><span id="line-8003"> return Result.create(cells);</span> |
| <span class="source-line-no">8004</span><span id="line-8004"> }</span> |
| <span class="source-line-no">8005</span><span id="line-8005"></span> |
| <span class="source-line-no">8006</span><span id="line-8006"> /**</span> |
| <span class="source-line-no">8007</span><span id="line-8007"> * Perform atomic (all or none) mutations within the region.</span> |
| <span class="source-line-no">8008</span><span id="line-8008"> * @param mutations The list of mutations to perform. <code>mutations</code> can contain</span> |
| <span class="source-line-no">8009</span><span id="line-8009"> * operations for multiple rows. Caller has to ensure that all rows are</span> |
| <span class="source-line-no">8010</span><span id="line-8010"> * contained in this region.</span> |
| <span class="source-line-no">8011</span><span id="line-8011"> * @param rowsToLock Rows to lock</span> |
| <span class="source-line-no">8012</span><span id="line-8012"> * @param nonceGroup Optional nonce group of the operation (client Id)</span> |
| <span class="source-line-no">8013</span><span id="line-8013"> * @param nonce Optional nonce of the operation (unique random id to ensure "more</span> |
| <span class="source-line-no">8014</span><span id="line-8014"> * idempotence") If multiple rows are locked care should be taken that</span> |
| <span class="source-line-no">8015</span><span id="line-8015"> * <code>rowsToLock</code> is sorted in order to avoid deadlocks.</span> |
| <span class="source-line-no">8016</span><span id="line-8016"> */</span> |
| <span class="source-line-no">8017</span><span id="line-8017"> @Override</span> |
| <span class="source-line-no">8018</span><span id="line-8018"> public void mutateRowsWithLocks(Collection<Mutation> mutations, Collection<byte[]> rowsToLock,</span> |
| <span class="source-line-no">8019</span><span id="line-8019"> long nonceGroup, long nonce) throws IOException {</span> |
| <span class="source-line-no">8020</span><span id="line-8020"> batchMutate(new MutationBatchOperation(this, mutations.toArray(new Mutation[mutations.size()]),</span> |
| <span class="source-line-no">8021</span><span id="line-8021"> true, nonceGroup, nonce) {</span> |
| <span class="source-line-no">8022</span><span id="line-8022"> @Override</span> |
| <span class="source-line-no">8023</span><span id="line-8023"> public MiniBatchOperationInProgress<Mutation></span> |
| <span class="source-line-no">8024</span><span id="line-8024"> lockRowsAndBuildMiniBatch(List<RowLock> acquiredRowLocks) throws IOException {</span> |
| <span class="source-line-no">8025</span><span id="line-8025"> RowLock prevRowLock = null;</span> |
| <span class="source-line-no">8026</span><span id="line-8026"> for (byte[] row : rowsToLock) {</span> |
| <span class="source-line-no">8027</span><span id="line-8027"> try {</span> |
| <span class="source-line-no">8028</span><span id="line-8028"> RowLock rowLock = region.getRowLock(row, false, prevRowLock); // write lock</span> |
| <span class="source-line-no">8029</span><span id="line-8029"> if (rowLock != prevRowLock) {</span> |
| <span class="source-line-no">8030</span><span id="line-8030"> acquiredRowLocks.add(rowLock);</span> |
| <span class="source-line-no">8031</span><span id="line-8031"> prevRowLock = rowLock;</span> |
| <span class="source-line-no">8032</span><span id="line-8032"> }</span> |
| <span class="source-line-no">8033</span><span id="line-8033"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">8034</span><span id="line-8034"> LOG.warn("Failed getting lock, row={}, in region {}", Bytes.toStringBinary(row), this,</span> |
| <span class="source-line-no">8035</span><span id="line-8035"> ioe);</span> |
| <span class="source-line-no">8036</span><span id="line-8036"> throw ioe;</span> |
| <span class="source-line-no">8037</span><span id="line-8037"> }</span> |
| <span class="source-line-no">8038</span><span id="line-8038"> }</span> |
| <span class="source-line-no">8039</span><span id="line-8039"> return createMiniBatch(size(), size());</span> |
| <span class="source-line-no">8040</span><span id="line-8040"> }</span> |
| <span class="source-line-no">8041</span><span id="line-8041"> });</span> |
| <span class="source-line-no">8042</span><span id="line-8042"> }</span> |
| <span class="source-line-no">8043</span><span id="line-8043"></span> |
| <span class="source-line-no">8044</span><span id="line-8044"> /** Returns statistics about the current load of the region */</span> |
| <span class="source-line-no">8045</span><span id="line-8045"> public ClientProtos.RegionLoadStats getLoadStatistics() {</span> |
| <span class="source-line-no">8046</span><span id="line-8046"> if (!regionStatsEnabled) {</span> |
| <span class="source-line-no">8047</span><span id="line-8047"> return null;</span> |
| <span class="source-line-no">8048</span><span id="line-8048"> }</span> |
| <span class="source-line-no">8049</span><span id="line-8049"> ClientProtos.RegionLoadStats.Builder stats = ClientProtos.RegionLoadStats.newBuilder();</span> |
| <span class="source-line-no">8050</span><span id="line-8050"> stats.setMemStoreLoad((int) (Math.min(100,</span> |
| <span class="source-line-no">8051</span><span id="line-8051"> (this.memStoreSizing.getMemStoreSize().getHeapSize() * 100) / this.memstoreFlushSize)));</span> |
| <span class="source-line-no">8052</span><span id="line-8052"> if (rsServices.getHeapMemoryManager() != null) {</span> |
| <span class="source-line-no">8053</span><span id="line-8053"> // the HeapMemoryManager uses -0.0 to signal a problem asking the JVM,</span> |
| <span class="source-line-no">8054</span><span id="line-8054"> // so we could just do the calculation below and we'll get a 0.</span> |
| <span class="source-line-no">8055</span><span id="line-8055"> // treating it as a special case analogous to no HMM instead so that it can be</span> |
| <span class="source-line-no">8056</span><span id="line-8056"> // programatically treated different from using <1% of heap.</span> |
| <span class="source-line-no">8057</span><span id="line-8057"> final float occupancy = rsServices.getHeapMemoryManager().getHeapOccupancyPercent();</span> |
| <span class="source-line-no">8058</span><span id="line-8058"> if (occupancy != HeapMemoryManager.HEAP_OCCUPANCY_ERROR_VALUE) {</span> |
| <span class="source-line-no">8059</span><span id="line-8059"> stats.setHeapOccupancy((int) (occupancy * 100));</span> |
| <span class="source-line-no">8060</span><span id="line-8060"> }</span> |
| <span class="source-line-no">8061</span><span id="line-8061"> }</span> |
| <span class="source-line-no">8062</span><span id="line-8062"> stats.setCompactionPressure((int) (rsServices.getCompactionPressure() * 100 > 100</span> |
| <span class="source-line-no">8063</span><span id="line-8063"> ? 100</span> |
| <span class="source-line-no">8064</span><span id="line-8064"> : rsServices.getCompactionPressure() * 100));</span> |
| <span class="source-line-no">8065</span><span id="line-8065"> return stats.build();</span> |
| <span class="source-line-no">8066</span><span id="line-8066"> }</span> |
| <span class="source-line-no">8067</span><span id="line-8067"></span> |
| <span class="source-line-no">8068</span><span id="line-8068"> @Override</span> |
| <span class="source-line-no">8069</span><span id="line-8069"> public Result append(Append append) throws IOException {</span> |
| <span class="source-line-no">8070</span><span id="line-8070"> return append(append, HConstants.NO_NONCE, HConstants.NO_NONCE);</span> |
| <span class="source-line-no">8071</span><span id="line-8071"> }</span> |
| <span class="source-line-no">8072</span><span id="line-8072"></span> |
| <span class="source-line-no">8073</span><span id="line-8073"> public Result append(Append append, long nonceGroup, long nonce) throws IOException {</span> |
| <span class="source-line-no">8074</span><span id="line-8074"> return TraceUtil.trace(() -> {</span> |
| <span class="source-line-no">8075</span><span id="line-8075"> checkReadOnly();</span> |
| <span class="source-line-no">8076</span><span id="line-8076"> checkResources();</span> |
| <span class="source-line-no">8077</span><span id="line-8077"> startRegionOperation(Operation.APPEND);</span> |
| <span class="source-line-no">8078</span><span id="line-8078"> try {</span> |
| <span class="source-line-no">8079</span><span id="line-8079"> // All edits for the given row (across all column families) must happen atomically.</span> |
| <span class="source-line-no">8080</span><span id="line-8080"> return mutate(append, true, nonceGroup, nonce).getResult();</span> |
| <span class="source-line-no">8081</span><span id="line-8081"> } finally {</span> |
| <span class="source-line-no">8082</span><span id="line-8082"> closeRegionOperation(Operation.APPEND);</span> |
| <span class="source-line-no">8083</span><span id="line-8083"> }</span> |
| <span class="source-line-no">8084</span><span id="line-8084"> }, () -> createRegionSpan("Region.append"));</span> |
| <span class="source-line-no">8085</span><span id="line-8085"> }</span> |
| <span class="source-line-no">8086</span><span id="line-8086"></span> |
| <span class="source-line-no">8087</span><span id="line-8087"> @Override</span> |
| <span class="source-line-no">8088</span><span id="line-8088"> public Result increment(Increment increment) throws IOException {</span> |
| <span class="source-line-no">8089</span><span id="line-8089"> return increment(increment, HConstants.NO_NONCE, HConstants.NO_NONCE);</span> |
| <span class="source-line-no">8090</span><span id="line-8090"> }</span> |
| <span class="source-line-no">8091</span><span id="line-8091"></span> |
| <span class="source-line-no">8092</span><span id="line-8092"> public Result increment(Increment increment, long nonceGroup, long nonce) throws IOException {</span> |
| <span class="source-line-no">8093</span><span id="line-8093"> return TraceUtil.trace(() -> {</span> |
| <span class="source-line-no">8094</span><span id="line-8094"> checkReadOnly();</span> |
| <span class="source-line-no">8095</span><span id="line-8095"> checkResources();</span> |
| <span class="source-line-no">8096</span><span id="line-8096"> startRegionOperation(Operation.INCREMENT);</span> |
| <span class="source-line-no">8097</span><span id="line-8097"> try {</span> |
| <span class="source-line-no">8098</span><span id="line-8098"> // All edits for the given row (across all column families) must happen atomically.</span> |
| <span class="source-line-no">8099</span><span id="line-8099"> return mutate(increment, true, nonceGroup, nonce).getResult();</span> |
| <span class="source-line-no">8100</span><span id="line-8100"> } finally {</span> |
| <span class="source-line-no">8101</span><span id="line-8101"> closeRegionOperation(Operation.INCREMENT);</span> |
| <span class="source-line-no">8102</span><span id="line-8102"> }</span> |
| <span class="source-line-no">8103</span><span id="line-8103"> }, () -> createRegionSpan("Region.increment"));</span> |
| <span class="source-line-no">8104</span><span id="line-8104"> }</span> |
| <span class="source-line-no">8105</span><span id="line-8105"></span> |
| <span class="source-line-no">8106</span><span id="line-8106"> private WALKeyImpl createWALKeyForWALAppend(boolean isReplay, BatchOperation<?> batchOp, long now,</span> |
| <span class="source-line-no">8107</span><span id="line-8107"> long nonceGroup, long nonce) {</span> |
| <span class="source-line-no">8108</span><span id="line-8108"> WALKeyImpl walKey = isReplay</span> |
| <span class="source-line-no">8109</span><span id="line-8109"> ? new WALKeyImpl(this.getRegionInfo().getEncodedNameAsBytes(),</span> |
| <span class="source-line-no">8110</span><span id="line-8110"> this.htableDescriptor.getTableName(), SequenceId.NO_SEQUENCE_ID, now,</span> |
| <span class="source-line-no">8111</span><span id="line-8111"> batchOp.getClusterIds(), nonceGroup, nonce, mvcc)</span> |
| <span class="source-line-no">8112</span><span id="line-8112"> : new WALKeyImpl(this.getRegionInfo().getEncodedNameAsBytes(),</span> |
| <span class="source-line-no">8113</span><span id="line-8113"> this.htableDescriptor.getTableName(), SequenceId.NO_SEQUENCE_ID, now,</span> |
| <span class="source-line-no">8114</span><span id="line-8114"> batchOp.getClusterIds(), nonceGroup, nonce, mvcc, this.getReplicationScope());</span> |
| <span class="source-line-no">8115</span><span id="line-8115"> if (isReplay) {</span> |
| <span class="source-line-no">8116</span><span id="line-8116"> walKey.setOrigLogSeqNum(batchOp.getOrigLogSeqNum());</span> |
| <span class="source-line-no">8117</span><span id="line-8117"> }</span> |
| <span class="source-line-no">8118</span><span id="line-8118"> return walKey;</span> |
| <span class="source-line-no">8119</span><span id="line-8119"> }</span> |
| <span class="source-line-no">8120</span><span id="line-8120"></span> |
| <span class="source-line-no">8121</span><span id="line-8121"> /** Returns writeEntry associated with this append */</span> |
| <span class="source-line-no">8122</span><span id="line-8122"> private WriteEntry doWALAppend(WALEdit walEdit, BatchOperation<?> batchOp,</span> |
| <span class="source-line-no">8123</span><span id="line-8123"> MiniBatchOperationInProgress<Mutation> miniBatchOp, long now, NonceKey nonceKey)</span> |
| <span class="source-line-no">8124</span><span id="line-8124"> throws IOException {</span> |
| <span class="source-line-no">8125</span><span id="line-8125"> Preconditions.checkArgument(walEdit != null && !walEdit.isEmpty(), "WALEdit is null or empty!");</span> |
| <span class="source-line-no">8126</span><span id="line-8126"> Preconditions.checkArgument(</span> |
| <span class="source-line-no">8127</span><span id="line-8127"> !walEdit.isReplay() || batchOp.getOrigLogSeqNum() != SequenceId.NO_SEQUENCE_ID,</span> |
| <span class="source-line-no">8128</span><span id="line-8128"> "Invalid replay sequence Id for replay WALEdit!");</span> |
| <span class="source-line-no">8129</span><span id="line-8129"></span> |
| <span class="source-line-no">8130</span><span id="line-8130"> WALKeyImpl walKey = createWALKeyForWALAppend(walEdit.isReplay(), batchOp, now,</span> |
| <span class="source-line-no">8131</span><span id="line-8131"> nonceKey.getNonceGroup(), nonceKey.getNonce());</span> |
| <span class="source-line-no">8132</span><span id="line-8132"> // don't call the coproc hook for writes to the WAL caused by</span> |
| <span class="source-line-no">8133</span><span id="line-8133"> // system lifecycle events like flushes or compactions</span> |
| <span class="source-line-no">8134</span><span id="line-8134"> if (this.coprocessorHost != null && !walEdit.isMetaEdit()) {</span> |
| <span class="source-line-no">8135</span><span id="line-8135"> this.coprocessorHost.preWALAppend(walKey, walEdit);</span> |
| <span class="source-line-no">8136</span><span id="line-8136"> }</span> |
| <span class="source-line-no">8137</span><span id="line-8137"> try {</span> |
| <span class="source-line-no">8138</span><span id="line-8138"> long txid = this.wal.appendData(this.getRegionInfo(), walKey, walEdit);</span> |
| <span class="source-line-no">8139</span><span id="line-8139"> WriteEntry writeEntry = walKey.getWriteEntry();</span> |
| <span class="source-line-no">8140</span><span id="line-8140"> // Call sync on our edit.</span> |
| <span class="source-line-no">8141</span><span id="line-8141"> if (txid != 0) {</span> |
| <span class="source-line-no">8142</span><span id="line-8142"> sync(txid, batchOp.durability);</span> |
| <span class="source-line-no">8143</span><span id="line-8143"> }</span> |
| <span class="source-line-no">8144</span><span id="line-8144"> /**</span> |
| <span class="source-line-no">8145</span><span id="line-8145"> * If above {@link HRegion#sync} throws Exception, the RegionServer should be aborted and</span> |
| <span class="source-line-no">8146</span><span id="line-8146"> * following {@link BatchOperation#writeMiniBatchOperationsToMemStore} will not be executed,</span> |
| <span class="source-line-no">8147</span><span id="line-8147"> * so there is no need to replicate to secondary replica, for this reason here we attach the</span> |
| <span class="source-line-no">8148</span><span id="line-8148"> * region replication action after the {@link HRegion#sync} is successful.</span> |
| <span class="source-line-no">8149</span><span id="line-8149"> */</span> |
| <span class="source-line-no">8150</span><span id="line-8150"> this.attachRegionReplicationInWALAppend(batchOp, miniBatchOp, walKey, walEdit, writeEntry);</span> |
| <span class="source-line-no">8151</span><span id="line-8151"> return writeEntry;</span> |
| <span class="source-line-no">8152</span><span id="line-8152"> } catch (IOException ioe) {</span> |
| <span class="source-line-no">8153</span><span id="line-8153"> if (walKey.getWriteEntry() != null) {</span> |
| <span class="source-line-no">8154</span><span id="line-8154"> mvcc.complete(walKey.getWriteEntry());</span> |
| <span class="source-line-no">8155</span><span id="line-8155"> }</span> |
| <span class="source-line-no">8156</span><span id="line-8156"></span> |
| <span class="source-line-no">8157</span><span id="line-8157"> /**</span> |
| <span class="source-line-no">8158</span><span id="line-8158"> * If {@link WAL#sync} get a timeout exception, the only correct way is to abort the region</span> |
| <span class="source-line-no">8159</span><span id="line-8159"> * server, as the design of {@link WAL#sync}, is to succeed or die, there is no 'failure'. It</span> |
| <span class="source-line-no">8160</span><span id="line-8160"> * is usually not a big deal is because we set a very large default value(5 minutes) for</span> |
| <span class="source-line-no">8161</span><span id="line-8161"> * {@link AbstractFSWAL#WAL_SYNC_TIMEOUT_MS}, usually the WAL system will abort the region</span> |
| <span class="source-line-no">8162</span><span id="line-8162"> * server if it can not finish the sync within 5 minutes.</span> |
| <span class="source-line-no">8163</span><span id="line-8163"> */</span> |
| <span class="source-line-no">8164</span><span id="line-8164"> if (ioe instanceof WALSyncTimeoutIOException) {</span> |
| <span class="source-line-no">8165</span><span id="line-8165"> if (rsServices != null) {</span> |
| <span class="source-line-no">8166</span><span id="line-8166"> rsServices.abort("WAL sync timeout,forcing server shutdown", ioe);</span> |
| <span class="source-line-no">8167</span><span id="line-8167"> }</span> |
| <span class="source-line-no">8168</span><span id="line-8168"> }</span> |
| <span class="source-line-no">8169</span><span id="line-8169"> throw ioe;</span> |
| <span class="source-line-no">8170</span><span id="line-8170"> }</span> |
| <span class="source-line-no">8171</span><span id="line-8171"> }</span> |
| <span class="source-line-no">8172</span><span id="line-8172"></span> |
| <span class="source-line-no">8173</span><span id="line-8173"> /**</span> |
| <span class="source-line-no">8174</span><span id="line-8174"> * Attach {@link RegionReplicationSink#add} to the mvcc writeEntry for replicating to region</span> |
| <span class="source-line-no">8175</span><span id="line-8175"> * replica.</span> |
| <span class="source-line-no">8176</span><span id="line-8176"> */</span> |
| <span class="source-line-no">8177</span><span id="line-8177"> private void attachRegionReplicationInWALAppend(BatchOperation<?> batchOp,</span> |
| <span class="source-line-no">8178</span><span id="line-8178"> MiniBatchOperationInProgress<Mutation> miniBatchOp, WALKeyImpl walKey, WALEdit walEdit,</span> |
| <span class="source-line-no">8179</span><span id="line-8179"> WriteEntry writeEntry) {</span> |
| <span class="source-line-no">8180</span><span id="line-8180"> if (!regionReplicationSink.isPresent()) {</span> |
| <span class="source-line-no">8181</span><span id="line-8181"> return;</span> |
| <span class="source-line-no">8182</span><span id="line-8182"> }</span> |
| <span class="source-line-no">8183</span><span id="line-8183"> /**</span> |
| <span class="source-line-no">8184</span><span id="line-8184"> * If {@link HRegion#regionReplicationSink} is present,only {@link MutationBatchOperation} is</span> |
| <span class="source-line-no">8185</span><span id="line-8185"> * used and {@link NonceKey} is all the same for {@link Mutation}s in</span> |
| <span class="source-line-no">8186</span><span id="line-8186"> * {@link MutationBatchOperation},so for HBASE-26993 case 1,if</span> |
| <span class="source-line-no">8187</span><span id="line-8187"> * {@link MiniBatchOperationInProgress#getWalEditForReplicateSkipWAL} is not null and we could</span> |
| <span class="source-line-no">8188</span><span id="line-8188"> * enter {@link HRegion#doWALAppend},that means partial {@link Mutation}s are</span> |
| <span class="source-line-no">8189</span><span id="line-8189"> * {@link Durability#SKIP_WAL}, we use</span> |
| <span class="source-line-no">8190</span><span id="line-8190"> * {@link MiniBatchOperationInProgress#getWalEditForReplicateSkipWAL} to replicate to region</span> |
| <span class="source-line-no">8191</span><span id="line-8191"> * replica,but if {@link MiniBatchOperationInProgress#getWalEditForReplicateSkipWAL} is</span> |
| <span class="source-line-no">8192</span><span id="line-8192"> * null,that means there is no {@link Mutation} is {@link Durability#SKIP_WAL},so we just use</span> |
| <span class="source-line-no">8193</span><span id="line-8193"> * walEdit to replicate.</span> |
| <span class="source-line-no">8194</span><span id="line-8194"> */</span> |
| <span class="source-line-no">8195</span><span id="line-8195"> assert batchOp instanceof MutationBatchOperation;</span> |
| <span class="source-line-no">8196</span><span id="line-8196"> WALEdit walEditToUse = miniBatchOp.getWalEditForReplicateIfExistsSkipWAL();</span> |
| <span class="source-line-no">8197</span><span id="line-8197"> if (walEditToUse == null) {</span> |
| <span class="source-line-no">8198</span><span id="line-8198"> walEditToUse = walEdit;</span> |
| <span class="source-line-no">8199</span><span id="line-8199"> }</span> |
| <span class="source-line-no">8200</span><span id="line-8200"> doAttachReplicateRegionReplicaAction(walKey, walEditToUse, writeEntry);</span> |
| <span class="source-line-no">8201</span><span id="line-8201"> }</span> |
| <span class="source-line-no">8202</span><span id="line-8202"></span> |
| <span class="source-line-no">8203</span><span id="line-8203"> /**</span> |
| <span class="source-line-no">8204</span><span id="line-8204"> * Attach {@link RegionReplicationSink#add} to the mvcc writeEntry for replicating to region</span> |
| <span class="source-line-no">8205</span><span id="line-8205"> * replica.</span> |
| <span class="source-line-no">8206</span><span id="line-8206"> */</span> |
| <span class="source-line-no">8207</span><span id="line-8207"> private void doAttachReplicateRegionReplicaAction(WALKeyImpl walKey, WALEdit walEdit,</span> |
| <span class="source-line-no">8208</span><span id="line-8208"> WriteEntry writeEntry) {</span> |
| <span class="source-line-no">8209</span><span id="line-8209"> if (walEdit == null || walEdit.isEmpty()) {</span> |
| <span class="source-line-no">8210</span><span id="line-8210"> return;</span> |
| <span class="source-line-no">8211</span><span id="line-8211"> }</span> |
| <span class="source-line-no">8212</span><span id="line-8212"> final ServerCall<?> rpcCall = RpcServer.getCurrentServerCallWithCellScanner().orElse(null);</span> |
| <span class="source-line-no">8213</span><span id="line-8213"> regionReplicationSink.ifPresent(sink -> writeEntry.attachCompletionAction(() -> {</span> |
| <span class="source-line-no">8214</span><span id="line-8214"> sink.add(walKey, walEdit, rpcCall);</span> |
| <span class="source-line-no">8215</span><span id="line-8215"> }));</span> |
| <span class="source-line-no">8216</span><span id="line-8216"> }</span> |
| <span class="source-line-no">8217</span><span id="line-8217"></span> |
| <span class="source-line-no">8218</span><span id="line-8218"> public static final long FIXED_OVERHEAD = ClassSize.estimateBase(HRegion.class, false);</span> |
| <span class="source-line-no">8219</span><span id="line-8219"></span> |
| <span class="source-line-no">8220</span><span id="line-8220"> // woefully out of date - currently missing:</span> |
| <span class="source-line-no">8221</span><span id="line-8221"> // 1 x HashMap - coprocessorServiceHandlers</span> |
| <span class="source-line-no">8222</span><span id="line-8222"> // 6 x LongAdder - numMutationsWithoutWAL, dataInMemoryWithoutWAL,</span> |
| <span class="source-line-no">8223</span><span id="line-8223"> // checkAndMutateChecksPassed, checkAndMutateChecksFailed, readRequestsCount,</span> |
| <span class="source-line-no">8224</span><span id="line-8224"> // writeRequestsCount, cpRequestsCount</span> |
| <span class="source-line-no">8225</span><span id="line-8225"> // 1 x HRegion$WriteState - writestate</span> |
| <span class="source-line-no">8226</span><span id="line-8226"> // 1 x RegionCoprocessorHost - coprocessorHost</span> |
| <span class="source-line-no">8227</span><span id="line-8227"> // 1 x RegionSplitPolicy - splitPolicy</span> |
| <span class="source-line-no">8228</span><span id="line-8228"> // 1 x MetricsRegion - metricsRegion</span> |
| <span class="source-line-no">8229</span><span id="line-8229"> // 1 x MetricsRegionWrapperImpl - metricsRegionWrapper</span> |
| <span class="source-line-no">8230</span><span id="line-8230"> // 1 x ReadPointCalculationLock - smallestReadPointCalcLock</span> |
| <span class="source-line-no">8231</span><span id="line-8231"> public static final long DEEP_OVERHEAD = FIXED_OVERHEAD + ClassSize.OBJECT + // closeLock</span> |
| <span class="source-line-no">8232</span><span id="line-8232"> (2 * ClassSize.ATOMIC_BOOLEAN) + // closed, closing</span> |
| <span class="source-line-no">8233</span><span id="line-8233"> (3 * ClassSize.ATOMIC_LONG) + // numPutsWithoutWAL, dataInMemoryWithoutWAL,</span> |
| <span class="source-line-no">8234</span><span id="line-8234"> // compactionsFailed</span> |
| <span class="source-line-no">8235</span><span id="line-8235"> (3 * ClassSize.CONCURRENT_HASHMAP) + // lockedRows, scannerReadPoints, regionLockHolders</span> |
| <span class="source-line-no">8236</span><span id="line-8236"> WriteState.HEAP_SIZE + // writestate</span> |
| <span class="source-line-no">8237</span><span id="line-8237"> ClassSize.CONCURRENT_SKIPLISTMAP + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + // stores</span> |
| <span class="source-line-no">8238</span><span id="line-8238"> (2 * ClassSize.REENTRANT_LOCK) + // lock, updatesLock</span> |
| <span class="source-line-no">8239</span><span id="line-8239"> MultiVersionConcurrencyControl.FIXED_SIZE // mvcc</span> |
| <span class="source-line-no">8240</span><span id="line-8240"> + 2 * ClassSize.TREEMAP // maxSeqIdInStores, replicationScopes</span> |
| <span class="source-line-no">8241</span><span id="line-8241"> + 2 * ClassSize.ATOMIC_INTEGER // majorInProgress, minorInProgress</span> |
| <span class="source-line-no">8242</span><span id="line-8242"> + ClassSize.STORE_SERVICES // store services</span> |
| <span class="source-line-no">8243</span><span id="line-8243"> + StoreHotnessProtector.FIXED_SIZE;</span> |
| <span class="source-line-no">8244</span><span id="line-8244"></span> |
| <span class="source-line-no">8245</span><span id="line-8245"> @Override</span> |
| <span class="source-line-no">8246</span><span id="line-8246"> public long heapSize() {</span> |
| <span class="source-line-no">8247</span><span id="line-8247"> // this does not take into account row locks, recent flushes, mvcc entries, and more</span> |
| <span class="source-line-no">8248</span><span id="line-8248"> return DEEP_OVERHEAD + stores.values().stream().mapToLong(HStore::heapSize).sum();</span> |
| <span class="source-line-no">8249</span><span id="line-8249"> }</span> |
| <span class="source-line-no">8250</span><span id="line-8250"></span> |
| <span class="source-line-no">8251</span><span id="line-8251"> /**</span> |
| <span class="source-line-no">8252</span><span id="line-8252"> * Registers a new protocol buffer {@link Service} subclass as a coprocessor endpoint to be</span> |
| <span class="source-line-no">8253</span><span id="line-8253"> * available for handling {@link #execService(RpcController, CoprocessorServiceCall)} calls.</span> |
| <span class="source-line-no">8254</span><span id="line-8254"> * <p/></span> |
| <span class="source-line-no">8255</span><span id="line-8255"> * Only a single instance may be registered per region for a given {@link Service} subclass (the</span> |
| <span class="source-line-no">8256</span><span id="line-8256"> * instances are keyed on {@link ServiceDescriptor#getFullName()}.. After the first registration,</span> |
| <span class="source-line-no">8257</span><span id="line-8257"> * subsequent calls with the same service name will fail with a return value of {@code false}.</span> |
| <span class="source-line-no">8258</span><span id="line-8258"> * @param instance the {@code Service} subclass instance to expose as a coprocessor endpoint</span> |
| <span class="source-line-no">8259</span><span id="line-8259"> * @return {@code true} if the registration was successful, {@code false} otherwise</span> |
| <span class="source-line-no">8260</span><span id="line-8260"> */</span> |
| <span class="source-line-no">8261</span><span id="line-8261"> public boolean registerService(Service instance) {</span> |
| <span class="source-line-no">8262</span><span id="line-8262"> // No stacking of instances is allowed for a single service name</span> |
| <span class="source-line-no">8263</span><span id="line-8263"> ServiceDescriptor serviceDesc = instance.getDescriptorForType();</span> |
| <span class="source-line-no">8264</span><span id="line-8264"> String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc);</span> |
| <span class="source-line-no">8265</span><span id="line-8265"> if (coprocessorServiceHandlers.containsKey(serviceName)) {</span> |
| <span class="source-line-no">8266</span><span id="line-8266"> LOG.error("Coprocessor service {} already registered, rejecting request from {} in region {}",</span> |
| <span class="source-line-no">8267</span><span id="line-8267"> serviceName, instance, this);</span> |
| <span class="source-line-no">8268</span><span id="line-8268"> return false;</span> |
| <span class="source-line-no">8269</span><span id="line-8269"> }</span> |
| <span class="source-line-no">8270</span><span id="line-8270"></span> |
| <span class="source-line-no">8271</span><span id="line-8271"> coprocessorServiceHandlers.put(serviceName, instance);</span> |
| <span class="source-line-no">8272</span><span id="line-8272"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">8273</span><span id="line-8273"> LOG.debug("Registered coprocessor service: region="</span> |
| <span class="source-line-no">8274</span><span id="line-8274"> + Bytes.toStringBinary(getRegionInfo().getRegionName()) + " service=" + serviceName);</span> |
| <span class="source-line-no">8275</span><span id="line-8275"> }</span> |
| <span class="source-line-no">8276</span><span id="line-8276"> return true;</span> |
| <span class="source-line-no">8277</span><span id="line-8277"> }</span> |
| <span class="source-line-no">8278</span><span id="line-8278"></span> |
| <span class="source-line-no">8279</span><span id="line-8279"> /**</span> |
| <span class="source-line-no">8280</span><span id="line-8280"> * Executes a single protocol buffer coprocessor endpoint {@link Service} method using the</span> |
| <span class="source-line-no">8281</span><span id="line-8281"> * registered protocol handlers. {@link Service} implementations must be registered via the</span> |
| <span class="source-line-no">8282</span><span id="line-8282"> * {@link #registerService(Service)} method before they are available.</span> |
| <span class="source-line-no">8283</span><span id="line-8283"> * @param controller an {@code RpcContoller} implementation to pass to the invoked service</span> |
| <span class="source-line-no">8284</span><span id="line-8284"> * @param call a {@code CoprocessorServiceCall} instance identifying the service, method,</span> |
| <span class="source-line-no">8285</span><span id="line-8285"> * and parameters for the method invocation</span> |
| <span class="source-line-no">8286</span><span id="line-8286"> * @return a protocol buffer {@code Message} instance containing the method's result</span> |
| <span class="source-line-no">8287</span><span id="line-8287"> * @throws IOException if no registered service handler is found or an error occurs during the</span> |
| <span class="source-line-no">8288</span><span id="line-8288"> * invocation</span> |
| <span class="source-line-no">8289</span><span id="line-8289"> * @see #registerService(Service)</span> |
| <span class="source-line-no">8290</span><span id="line-8290"> */</span> |
| <span class="source-line-no">8291</span><span id="line-8291"> public Message execService(RpcController controller, CoprocessorServiceCall call)</span> |
| <span class="source-line-no">8292</span><span id="line-8292"> throws IOException {</span> |
| <span class="source-line-no">8293</span><span id="line-8293"> String serviceName = call.getServiceName();</span> |
| <span class="source-line-no">8294</span><span id="line-8294"> Service service = coprocessorServiceHandlers.get(serviceName);</span> |
| <span class="source-line-no">8295</span><span id="line-8295"> if (service == null) {</span> |
| <span class="source-line-no">8296</span><span id="line-8296"> throw new UnknownProtocolException(null, "No registered coprocessor service found for "</span> |
| <span class="source-line-no">8297</span><span id="line-8297"> + serviceName + " in region " + Bytes.toStringBinary(getRegionInfo().getRegionName()));</span> |
| <span class="source-line-no">8298</span><span id="line-8298"> }</span> |
| <span class="source-line-no">8299</span><span id="line-8299"> ServiceDescriptor serviceDesc = service.getDescriptorForType();</span> |
| <span class="source-line-no">8300</span><span id="line-8300"></span> |
| <span class="source-line-no">8301</span><span id="line-8301"> cpRequestsCount.increment();</span> |
| <span class="source-line-no">8302</span><span id="line-8302"> String methodName = call.getMethodName();</span> |
| <span class="source-line-no">8303</span><span id="line-8303"> MethodDescriptor methodDesc = CoprocessorRpcUtils.getMethodDescriptor(methodName, serviceDesc);</span> |
| <span class="source-line-no">8304</span><span id="line-8304"></span> |
| <span class="source-line-no">8305</span><span id="line-8305"> Message.Builder builder = service.getRequestPrototype(methodDesc).newBuilderForType();</span> |
| <span class="source-line-no">8306</span><span id="line-8306"></span> |
| <span class="source-line-no">8307</span><span id="line-8307"> ProtobufUtil.mergeFrom(builder, call.getRequest().toByteArray());</span> |
| <span class="source-line-no">8308</span><span id="line-8308"> Message request = CoprocessorRpcUtils.getRequest(service, methodDesc, call.getRequest());</span> |
| <span class="source-line-no">8309</span><span id="line-8309"></span> |
| <span class="source-line-no">8310</span><span id="line-8310"> if (coprocessorHost != null) {</span> |
| <span class="source-line-no">8311</span><span id="line-8311"> request = coprocessorHost.preEndpointInvocation(service, methodName, request);</span> |
| <span class="source-line-no">8312</span><span id="line-8312"> }</span> |
| <span class="source-line-no">8313</span><span id="line-8313"></span> |
| <span class="source-line-no">8314</span><span id="line-8314"> final Message.Builder responseBuilder =</span> |
| <span class="source-line-no">8315</span><span id="line-8315"> service.getResponsePrototype(methodDesc).newBuilderForType();</span> |
| <span class="source-line-no">8316</span><span id="line-8316"> service.callMethod(methodDesc, controller, request, new RpcCallback<Message>() {</span> |
| <span class="source-line-no">8317</span><span id="line-8317"> @Override</span> |
| <span class="source-line-no">8318</span><span id="line-8318"> public void run(Message message) {</span> |
| <span class="source-line-no">8319</span><span id="line-8319"> if (message != null) {</span> |
| <span class="source-line-no">8320</span><span id="line-8320"> responseBuilder.mergeFrom(message);</span> |
| <span class="source-line-no">8321</span><span id="line-8321"> }</span> |
| <span class="source-line-no">8322</span><span id="line-8322"> }</span> |
| <span class="source-line-no">8323</span><span id="line-8323"> });</span> |
| <span class="source-line-no">8324</span><span id="line-8324"></span> |
| <span class="source-line-no">8325</span><span id="line-8325"> if (coprocessorHost != null) {</span> |
| <span class="source-line-no">8326</span><span id="line-8326"> coprocessorHost.postEndpointInvocation(service, methodName, request, responseBuilder);</span> |
| <span class="source-line-no">8327</span><span id="line-8327"> }</span> |
| <span class="source-line-no">8328</span><span id="line-8328"> IOException exception =</span> |
| <span class="source-line-no">8329</span><span id="line-8329"> org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils.getControllerException(controller);</span> |
| <span class="source-line-no">8330</span><span id="line-8330"> if (exception != null) {</span> |
| <span class="source-line-no">8331</span><span id="line-8331"> throw exception;</span> |
| <span class="source-line-no">8332</span><span id="line-8332"> }</span> |
| <span class="source-line-no">8333</span><span id="line-8333"></span> |
| <span class="source-line-no">8334</span><span id="line-8334"> return responseBuilder.build();</span> |
| <span class="source-line-no">8335</span><span id="line-8335"> }</span> |
| <span class="source-line-no">8336</span><span id="line-8336"></span> |
| <span class="source-line-no">8337</span><span id="line-8337"> public Optional<byte[]> checkSplit() {</span> |
| <span class="source-line-no">8338</span><span id="line-8338"> return checkSplit(false);</span> |
| <span class="source-line-no">8339</span><span id="line-8339"> }</span> |
| <span class="source-line-no">8340</span><span id="line-8340"></span> |
| <span class="source-line-no">8341</span><span id="line-8341"> /**</span> |
| <span class="source-line-no">8342</span><span id="line-8342"> * Return the split point. An empty result indicates the region isn't splittable.</span> |
| <span class="source-line-no">8343</span><span id="line-8343"> */</span> |
| <span class="source-line-no">8344</span><span id="line-8344"> public Optional<byte[]> checkSplit(boolean force) {</span> |
| <span class="source-line-no">8345</span><span id="line-8345"> // Can't split META</span> |
| <span class="source-line-no">8346</span><span id="line-8346"> if (this.getRegionInfo().isMetaRegion()) {</span> |
| <span class="source-line-no">8347</span><span id="line-8347"> return Optional.empty();</span> |
| <span class="source-line-no">8348</span><span id="line-8348"> }</span> |
| <span class="source-line-no">8349</span><span id="line-8349"></span> |
| <span class="source-line-no">8350</span><span id="line-8350"> // Can't split a region that is closing.</span> |
| <span class="source-line-no">8351</span><span id="line-8351"> if (this.isClosing()) {</span> |
| <span class="source-line-no">8352</span><span id="line-8352"> return Optional.empty();</span> |
| <span class="source-line-no">8353</span><span id="line-8353"> }</span> |
| <span class="source-line-no">8354</span><span id="line-8354"></span> |
| <span class="source-line-no">8355</span><span id="line-8355"> if (!force && !splitPolicy.shouldSplit()) {</span> |
| <span class="source-line-no">8356</span><span id="line-8356"> return Optional.empty();</span> |
| <span class="source-line-no">8357</span><span id="line-8357"> }</span> |
| <span class="source-line-no">8358</span><span id="line-8358"></span> |
| <span class="source-line-no">8359</span><span id="line-8359"> byte[] ret = splitPolicy.getSplitPoint();</span> |
| <span class="source-line-no">8360</span><span id="line-8360"> if (ret != null && ret.length > 0) {</span> |
| <span class="source-line-no">8361</span><span id="line-8361"> ret = splitRestriction.getRestrictedSplitPoint(ret);</span> |
| <span class="source-line-no">8362</span><span id="line-8362"> }</span> |
| <span class="source-line-no">8363</span><span id="line-8363"></span> |
| <span class="source-line-no">8364</span><span id="line-8364"> if (ret != null) {</span> |
| <span class="source-line-no">8365</span><span id="line-8365"> try {</span> |
| <span class="source-line-no">8366</span><span id="line-8366"> checkRow(ret, "calculated split");</span> |
| <span class="source-line-no">8367</span><span id="line-8367"> } catch (IOException e) {</span> |
| <span class="source-line-no">8368</span><span id="line-8368"> LOG.error("Ignoring invalid split for region {}", this, e);</span> |
| <span class="source-line-no">8369</span><span id="line-8369"> return Optional.empty();</span> |
| <span class="source-line-no">8370</span><span id="line-8370"> }</span> |
| <span class="source-line-no">8371</span><span id="line-8371"> return Optional.of(ret);</span> |
| <span class="source-line-no">8372</span><span id="line-8372"> } else {</span> |
| <span class="source-line-no">8373</span><span id="line-8373"> return Optional.empty();</span> |
| <span class="source-line-no">8374</span><span id="line-8374"> }</span> |
| <span class="source-line-no">8375</span><span id="line-8375"> }</span> |
| <span class="source-line-no">8376</span><span id="line-8376"></span> |
| <span class="source-line-no">8377</span><span id="line-8377"> /** Returns The priority that this region should have in the compaction queue */</span> |
| <span class="source-line-no">8378</span><span id="line-8378"> public int getCompactPriority() {</span> |
| <span class="source-line-no">8379</span><span id="line-8379"> if (checkSplit().isPresent() && conf.getBoolean(SPLIT_IGNORE_BLOCKING_ENABLED_KEY, false)) {</span> |
| <span class="source-line-no">8380</span><span id="line-8380"> // if a region should split, split it before compact</span> |
| <span class="source-line-no">8381</span><span id="line-8381"> return Store.PRIORITY_USER;</span> |
| <span class="source-line-no">8382</span><span id="line-8382"> }</span> |
| <span class="source-line-no">8383</span><span id="line-8383"> return stores.values().stream().mapToInt(HStore::getCompactPriority).min()</span> |
| <span class="source-line-no">8384</span><span id="line-8384"> .orElse(Store.NO_PRIORITY);</span> |
| <span class="source-line-no">8385</span><span id="line-8385"> }</span> |
| <span class="source-line-no">8386</span><span id="line-8386"></span> |
| <span class="source-line-no">8387</span><span id="line-8387"> /** Returns the coprocessor host */</span> |
| <span class="source-line-no">8388</span><span id="line-8388"> public RegionCoprocessorHost getCoprocessorHost() {</span> |
| <span class="source-line-no">8389</span><span id="line-8389"> return coprocessorHost;</span> |
| <span class="source-line-no">8390</span><span id="line-8390"> }</span> |
| <span class="source-line-no">8391</span><span id="line-8391"></span> |
| <span class="source-line-no">8392</span><span id="line-8392"> /** @param coprocessorHost the new coprocessor host */</span> |
| <span class="source-line-no">8393</span><span id="line-8393"> public void setCoprocessorHost(final RegionCoprocessorHost coprocessorHost) {</span> |
| <span class="source-line-no">8394</span><span id="line-8394"> this.coprocessorHost = coprocessorHost;</span> |
| <span class="source-line-no">8395</span><span id="line-8395"> }</span> |
| <span class="source-line-no">8396</span><span id="line-8396"></span> |
| <span class="source-line-no">8397</span><span id="line-8397"> @Override</span> |
| <span class="source-line-no">8398</span><span id="line-8398"> public void startRegionOperation() throws IOException {</span> |
| <span class="source-line-no">8399</span><span id="line-8399"> startRegionOperation(Operation.ANY);</span> |
| <span class="source-line-no">8400</span><span id="line-8400"> }</span> |
| <span class="source-line-no">8401</span><span id="line-8401"></span> |
| <span class="source-line-no">8402</span><span id="line-8402"> @Override</span> |
| <span class="source-line-no">8403</span><span id="line-8403"> public void startRegionOperation(Operation op) throws IOException {</span> |
| <span class="source-line-no">8404</span><span id="line-8404"> boolean isInterruptableOp = false;</span> |
| <span class="source-line-no">8405</span><span id="line-8405"> switch (op) {</span> |
| <span class="source-line-no">8406</span><span id="line-8406"> case GET: // interruptible read operations</span> |
| <span class="source-line-no">8407</span><span id="line-8407"> case SCAN:</span> |
| <span class="source-line-no">8408</span><span id="line-8408"> isInterruptableOp = true;</span> |
| <span class="source-line-no">8409</span><span id="line-8409"> checkReadsEnabled();</span> |
| <span class="source-line-no">8410</span><span id="line-8410"> break;</span> |
| <span class="source-line-no">8411</span><span id="line-8411"> case INCREMENT: // interruptible write operations</span> |
| <span class="source-line-no">8412</span><span id="line-8412"> case APPEND:</span> |
| <span class="source-line-no">8413</span><span id="line-8413"> case PUT:</span> |
| <span class="source-line-no">8414</span><span id="line-8414"> case DELETE:</span> |
| <span class="source-line-no">8415</span><span id="line-8415"> case BATCH_MUTATE:</span> |
| <span class="source-line-no">8416</span><span id="line-8416"> case CHECK_AND_MUTATE:</span> |
| <span class="source-line-no">8417</span><span id="line-8417"> isInterruptableOp = true;</span> |
| <span class="source-line-no">8418</span><span id="line-8418"> break;</span> |
| <span class="source-line-no">8419</span><span id="line-8419"> default: // all others</span> |
| <span class="source-line-no">8420</span><span id="line-8420"> break;</span> |
| <span class="source-line-no">8421</span><span id="line-8421"> }</span> |
| <span class="source-line-no">8422</span><span id="line-8422"> if (</span> |
| <span class="source-line-no">8423</span><span id="line-8423"> op == Operation.MERGE_REGION || op == Operation.SPLIT_REGION || op == Operation.COMPACT_REGION</span> |
| <span class="source-line-no">8424</span><span id="line-8424"> || op == Operation.COMPACT_SWITCH</span> |
| <span class="source-line-no">8425</span><span id="line-8425"> ) {</span> |
| <span class="source-line-no">8426</span><span id="line-8426"> // split, merge or compact region doesn't need to check the closing/closed state or lock the</span> |
| <span class="source-line-no">8427</span><span id="line-8427"> // region</span> |
| <span class="source-line-no">8428</span><span id="line-8428"> return;</span> |
| <span class="source-line-no">8429</span><span id="line-8429"> }</span> |
| <span class="source-line-no">8430</span><span id="line-8430"> if (this.closing.get()) {</span> |
| <span class="source-line-no">8431</span><span id="line-8431"> throw new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " is closing");</span> |
| <span class="source-line-no">8432</span><span id="line-8432"> }</span> |
| <span class="source-line-no">8433</span><span id="line-8433"> lock(lock.readLock());</span> |
| <span class="source-line-no">8434</span><span id="line-8434"> // Update regionLockHolders ONLY for any startRegionOperation call that is invoked from</span> |
| <span class="source-line-no">8435</span><span id="line-8435"> // an RPC handler</span> |
| <span class="source-line-no">8436</span><span id="line-8436"> Thread thisThread = Thread.currentThread();</span> |
| <span class="source-line-no">8437</span><span id="line-8437"> if (isInterruptableOp) {</span> |
| <span class="source-line-no">8438</span><span id="line-8438"> regionLockHolders.put(thisThread, true);</span> |
| <span class="source-line-no">8439</span><span id="line-8439"> }</span> |
| <span class="source-line-no">8440</span><span id="line-8440"> if (this.closed.get()) {</span> |
| <span class="source-line-no">8441</span><span id="line-8441"> lock.readLock().unlock();</span> |
| <span class="source-line-no">8442</span><span id="line-8442"> throw new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " is closed");</span> |
| <span class="source-line-no">8443</span><span id="line-8443"> }</span> |
| <span class="source-line-no">8444</span><span id="line-8444"> // The unit for snapshot is a region. So, all stores for this region must be</span> |
| <span class="source-line-no">8445</span><span id="line-8445"> // prepared for snapshot operation before proceeding.</span> |
| <span class="source-line-no">8446</span><span id="line-8446"> if (op == Operation.SNAPSHOT) {</span> |
| <span class="source-line-no">8447</span><span id="line-8447"> stores.values().forEach(HStore::preSnapshotOperation);</span> |
| <span class="source-line-no">8448</span><span id="line-8448"> }</span> |
| <span class="source-line-no">8449</span><span id="line-8449"> try {</span> |
| <span class="source-line-no">8450</span><span id="line-8450"> if (coprocessorHost != null) {</span> |
| <span class="source-line-no">8451</span><span id="line-8451"> coprocessorHost.postStartRegionOperation(op);</span> |
| <span class="source-line-no">8452</span><span id="line-8452"> }</span> |
| <span class="source-line-no">8453</span><span id="line-8453"> } catch (Exception e) {</span> |
| <span class="source-line-no">8454</span><span id="line-8454"> if (isInterruptableOp) {</span> |
| <span class="source-line-no">8455</span><span id="line-8455"> // would be harmless to remove what we didn't add but we know by 'isInterruptableOp'</span> |
| <span class="source-line-no">8456</span><span id="line-8456"> // if we added this thread to regionLockHolders</span> |
| <span class="source-line-no">8457</span><span id="line-8457"> regionLockHolders.remove(thisThread);</span> |
| <span class="source-line-no">8458</span><span id="line-8458"> }</span> |
| <span class="source-line-no">8459</span><span id="line-8459"> lock.readLock().unlock();</span> |
| <span class="source-line-no">8460</span><span id="line-8460"> throw new IOException(e);</span> |
| <span class="source-line-no">8461</span><span id="line-8461"> }</span> |
| <span class="source-line-no">8462</span><span id="line-8462"> }</span> |
| <span class="source-line-no">8463</span><span id="line-8463"></span> |
| <span class="source-line-no">8464</span><span id="line-8464"> @Override</span> |
| <span class="source-line-no">8465</span><span id="line-8465"> public void closeRegionOperation() throws IOException {</span> |
| <span class="source-line-no">8466</span><span id="line-8466"> closeRegionOperation(Operation.ANY);</span> |
| <span class="source-line-no">8467</span><span id="line-8467"> }</span> |
| <span class="source-line-no">8468</span><span id="line-8468"></span> |
| <span class="source-line-no">8469</span><span id="line-8469"> @Override</span> |
| <span class="source-line-no">8470</span><span id="line-8470"> public void closeRegionOperation(Operation operation) throws IOException {</span> |
| <span class="source-line-no">8471</span><span id="line-8471"> if (operation == Operation.SNAPSHOT) {</span> |
| <span class="source-line-no">8472</span><span id="line-8472"> stores.values().forEach(HStore::postSnapshotOperation);</span> |
| <span class="source-line-no">8473</span><span id="line-8473"> }</span> |
| <span class="source-line-no">8474</span><span id="line-8474"> Thread thisThread = Thread.currentThread();</span> |
| <span class="source-line-no">8475</span><span id="line-8475"> regionLockHolders.remove(thisThread);</span> |
| <span class="source-line-no">8476</span><span id="line-8476"> lock.readLock().unlock();</span> |
| <span class="source-line-no">8477</span><span id="line-8477"> if (coprocessorHost != null) {</span> |
| <span class="source-line-no">8478</span><span id="line-8478"> coprocessorHost.postCloseRegionOperation(operation);</span> |
| <span class="source-line-no">8479</span><span id="line-8479"> }</span> |
| <span class="source-line-no">8480</span><span id="line-8480"> }</span> |
| <span class="source-line-no">8481</span><span id="line-8481"></span> |
| <span class="source-line-no">8482</span><span id="line-8482"> /**</span> |
| <span class="source-line-no">8483</span><span id="line-8483"> * This method needs to be called before any public call that reads or modifies stores in bulk. It</span> |
| <span class="source-line-no">8484</span><span id="line-8484"> * has to be called just before a try. #closeBulkRegionOperation needs to be called in the try's</span> |
| <span class="source-line-no">8485</span><span id="line-8485"> * finally block Acquires a writelock and checks if the region is closing or closed.</span> |
| <span class="source-line-no">8486</span><span id="line-8486"> * @throws NotServingRegionException when the region is closing or closed</span> |
| <span class="source-line-no">8487</span><span id="line-8487"> * @throws RegionTooBusyException if failed to get the lock in time</span> |
| <span class="source-line-no">8488</span><span id="line-8488"> * @throws InterruptedIOException if interrupted while waiting for a lock</span> |
| <span class="source-line-no">8489</span><span id="line-8489"> */</span> |
| <span class="source-line-no">8490</span><span id="line-8490"> private void startBulkRegionOperation(boolean writeLockNeeded) throws IOException {</span> |
| <span class="source-line-no">8491</span><span id="line-8491"> if (this.closing.get()) {</span> |
| <span class="source-line-no">8492</span><span id="line-8492"> throw new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " is closing");</span> |
| <span class="source-line-no">8493</span><span id="line-8493"> }</span> |
| <span class="source-line-no">8494</span><span id="line-8494"> if (writeLockNeeded) lock(lock.writeLock());</span> |
| <span class="source-line-no">8495</span><span id="line-8495"> else lock(lock.readLock());</span> |
| <span class="source-line-no">8496</span><span id="line-8496"> if (this.closed.get()) {</span> |
| <span class="source-line-no">8497</span><span id="line-8497"> if (writeLockNeeded) lock.writeLock().unlock();</span> |
| <span class="source-line-no">8498</span><span id="line-8498"> else lock.readLock().unlock();</span> |
| <span class="source-line-no">8499</span><span id="line-8499"> throw new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " is closed");</span> |
| <span class="source-line-no">8500</span><span id="line-8500"> }</span> |
| <span class="source-line-no">8501</span><span id="line-8501"> regionLockHolders.put(Thread.currentThread(), true);</span> |
| <span class="source-line-no">8502</span><span id="line-8502"> }</span> |
| <span class="source-line-no">8503</span><span id="line-8503"></span> |
| <span class="source-line-no">8504</span><span id="line-8504"> /**</span> |
| <span class="source-line-no">8505</span><span id="line-8505"> * Closes the lock. This needs to be called in the finally block corresponding to the try block of</span> |
| <span class="source-line-no">8506</span><span id="line-8506"> * #startRegionOperation</span> |
| <span class="source-line-no">8507</span><span id="line-8507"> */</span> |
| <span class="source-line-no">8508</span><span id="line-8508"> private void closeBulkRegionOperation() {</span> |
| <span class="source-line-no">8509</span><span id="line-8509"> regionLockHolders.remove(Thread.currentThread());</span> |
| <span class="source-line-no">8510</span><span id="line-8510"> if (lock.writeLock().isHeldByCurrentThread()) lock.writeLock().unlock();</span> |
| <span class="source-line-no">8511</span><span id="line-8511"> else lock.readLock().unlock();</span> |
| <span class="source-line-no">8512</span><span id="line-8512"> }</span> |
| <span class="source-line-no">8513</span><span id="line-8513"></span> |
| <span class="source-line-no">8514</span><span id="line-8514"> /**</span> |
| <span class="source-line-no">8515</span><span id="line-8515"> * Update LongAdders for number of puts without wal and the size of possible data loss. These</span> |
| <span class="source-line-no">8516</span><span id="line-8516"> * information are exposed by the region server metrics.</span> |
| <span class="source-line-no">8517</span><span id="line-8517"> */</span> |
| <span class="source-line-no">8518</span><span id="line-8518"> private void recordMutationWithoutWal(final Map<byte[], List<Cell>> familyMap) {</span> |
| <span class="source-line-no">8519</span><span id="line-8519"> numMutationsWithoutWAL.increment();</span> |
| <span class="source-line-no">8520</span><span id="line-8520"> if (numMutationsWithoutWAL.sum() <= 1) {</span> |
| <span class="source-line-no">8521</span><span id="line-8521"> LOG.info("writing data to region " + this</span> |
| <span class="source-line-no">8522</span><span id="line-8522"> + " with WAL disabled. Data may be lost in the event of a crash.");</span> |
| <span class="source-line-no">8523</span><span id="line-8523"> }</span> |
| <span class="source-line-no">8524</span><span id="line-8524"></span> |
| <span class="source-line-no">8525</span><span id="line-8525"> long mutationSize = 0;</span> |
| <span class="source-line-no">8526</span><span id="line-8526"> for (List<Cell> cells : familyMap.values()) {</span> |
| <span class="source-line-no">8527</span><span id="line-8527"> // Optimization: 'foreach' loop is not used. See:</span> |
| <span class="source-line-no">8528</span><span id="line-8528"> // HBASE-12023 HRegion.applyFamilyMapToMemstore creates too many iterator objects</span> |
| <span class="source-line-no">8529</span><span id="line-8529"> assert cells instanceof RandomAccess;</span> |
| <span class="source-line-no">8530</span><span id="line-8530"> int listSize = cells.size();</span> |
| <span class="source-line-no">8531</span><span id="line-8531"> for (int i = 0; i < listSize; i++) {</span> |
| <span class="source-line-no">8532</span><span id="line-8532"> Cell cell = cells.get(i);</span> |
| <span class="source-line-no">8533</span><span id="line-8533"> mutationSize += cell.getSerializedSize();</span> |
| <span class="source-line-no">8534</span><span id="line-8534"> }</span> |
| <span class="source-line-no">8535</span><span id="line-8535"> }</span> |
| <span class="source-line-no">8536</span><span id="line-8536"></span> |
| <span class="source-line-no">8537</span><span id="line-8537"> dataInMemoryWithoutWAL.add(mutationSize);</span> |
| <span class="source-line-no">8538</span><span id="line-8538"> }</span> |
| <span class="source-line-no">8539</span><span id="line-8539"></span> |
| <span class="source-line-no">8540</span><span id="line-8540"> private void lock(final Lock lock) throws IOException {</span> |
| <span class="source-line-no">8541</span><span id="line-8541"> lock(lock, 1);</span> |
| <span class="source-line-no">8542</span><span id="line-8542"> }</span> |
| <span class="source-line-no">8543</span><span id="line-8543"></span> |
| <span class="source-line-no">8544</span><span id="line-8544"> /**</span> |
| <span class="source-line-no">8545</span><span id="line-8545"> * Try to acquire a lock. Throw RegionTooBusyException if failed to get the lock in time. Throw</span> |
| <span class="source-line-no">8546</span><span id="line-8546"> * InterruptedIOException if interrupted while waiting for the lock.</span> |
| <span class="source-line-no">8547</span><span id="line-8547"> */</span> |
| <span class="source-line-no">8548</span><span id="line-8548"> private void lock(final Lock lock, final int multiplier) throws IOException {</span> |
| <span class="source-line-no">8549</span><span id="line-8549"> try {</span> |
| <span class="source-line-no">8550</span><span id="line-8550"> final long waitTime = Math.min(maxBusyWaitDuration,</span> |
| <span class="source-line-no">8551</span><span id="line-8551"> busyWaitDuration * Math.min(multiplier, maxBusyWaitMultiplier));</span> |
| <span class="source-line-no">8552</span><span id="line-8552"> if (!lock.tryLock(waitTime, TimeUnit.MILLISECONDS)) {</span> |
| <span class="source-line-no">8553</span><span id="line-8553"> // Don't print millis. Message is used as a key over in</span> |
| <span class="source-line-no">8554</span><span id="line-8554"> // RetriesExhaustedWithDetailsException processing.</span> |
| <span class="source-line-no">8555</span><span id="line-8555"> final String regionName =</span> |
| <span class="source-line-no">8556</span><span id="line-8556"> this.getRegionInfo() == null ? "unknown" : this.getRegionInfo().getRegionNameAsString();</span> |
| <span class="source-line-no">8557</span><span id="line-8557"> final String serverName = this.getRegionServerServices() == null</span> |
| <span class="source-line-no">8558</span><span id="line-8558"> ? "unknown"</span> |
| <span class="source-line-no">8559</span><span id="line-8559"> : (this.getRegionServerServices().getServerName() == null</span> |
| <span class="source-line-no">8560</span><span id="line-8560"> ? "unknown"</span> |
| <span class="source-line-no">8561</span><span id="line-8561"> : this.getRegionServerServices().getServerName().toString());</span> |
| <span class="source-line-no">8562</span><span id="line-8562"> RegionTooBusyException rtbe = new RegionTooBusyException(</span> |
| <span class="source-line-no">8563</span><span id="line-8563"> "Failed to obtain lock; regionName=" + regionName + ", server=" + serverName);</span> |
| <span class="source-line-no">8564</span><span id="line-8564"> LOG.warn("Region is too busy to allow lock acquisition.", rtbe);</span> |
| <span class="source-line-no">8565</span><span id="line-8565"> throw rtbe;</span> |
| <span class="source-line-no">8566</span><span id="line-8566"> }</span> |
| <span class="source-line-no">8567</span><span id="line-8567"> } catch (InterruptedException ie) {</span> |
| <span class="source-line-no">8568</span><span id="line-8568"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">8569</span><span id="line-8569"> LOG.debug("Interrupted while waiting for a lock in region {}", this);</span> |
| <span class="source-line-no">8570</span><span id="line-8570"> }</span> |
| <span class="source-line-no">8571</span><span id="line-8571"> throw throwOnInterrupt(ie);</span> |
| <span class="source-line-no">8572</span><span id="line-8572"> }</span> |
| <span class="source-line-no">8573</span><span id="line-8573"> }</span> |
| <span class="source-line-no">8574</span><span id="line-8574"></span> |
| <span class="source-line-no">8575</span><span id="line-8575"> /**</span> |
| <span class="source-line-no">8576</span><span id="line-8576"> * Calls sync with the given transaction ID</span> |
| <span class="source-line-no">8577</span><span id="line-8577"> * @param txid should sync up to which transaction</span> |
| <span class="source-line-no">8578</span><span id="line-8578"> * @throws IOException If anything goes wrong with DFS</span> |
| <span class="source-line-no">8579</span><span id="line-8579"> */</span> |
| <span class="source-line-no">8580</span><span id="line-8580"> private void sync(long txid, Durability durability) throws IOException {</span> |
| <span class="source-line-no">8581</span><span id="line-8581"> if (this.getRegionInfo().isMetaRegion()) {</span> |
| <span class="source-line-no">8582</span><span id="line-8582"> this.wal.sync(txid);</span> |
| <span class="source-line-no">8583</span><span id="line-8583"> } else {</span> |
| <span class="source-line-no">8584</span><span id="line-8584"> switch (durability) {</span> |
| <span class="source-line-no">8585</span><span id="line-8585"> case USE_DEFAULT:</span> |
| <span class="source-line-no">8586</span><span id="line-8586"> // do what table defaults to</span> |
| <span class="source-line-no">8587</span><span id="line-8587"> if (shouldSyncWAL()) {</span> |
| <span class="source-line-no">8588</span><span id="line-8588"> this.wal.sync(txid);</span> |
| <span class="source-line-no">8589</span><span id="line-8589"> }</span> |
| <span class="source-line-no">8590</span><span id="line-8590"> break;</span> |
| <span class="source-line-no">8591</span><span id="line-8591"> case SKIP_WAL:</span> |
| <span class="source-line-no">8592</span><span id="line-8592"> // nothing do to</span> |
| <span class="source-line-no">8593</span><span id="line-8593"> break;</span> |
| <span class="source-line-no">8594</span><span id="line-8594"> case ASYNC_WAL:</span> |
| <span class="source-line-no">8595</span><span id="line-8595"> // nothing do to</span> |
| <span class="source-line-no">8596</span><span id="line-8596"> break;</span> |
| <span class="source-line-no">8597</span><span id="line-8597"> case SYNC_WAL:</span> |
| <span class="source-line-no">8598</span><span id="line-8598"> this.wal.sync(txid, false);</span> |
| <span class="source-line-no">8599</span><span id="line-8599"> break;</span> |
| <span class="source-line-no">8600</span><span id="line-8600"> case FSYNC_WAL:</span> |
| <span class="source-line-no">8601</span><span id="line-8601"> this.wal.sync(txid, true);</span> |
| <span class="source-line-no">8602</span><span id="line-8602"> break;</span> |
| <span class="source-line-no">8603</span><span id="line-8603"> default:</span> |
| <span class="source-line-no">8604</span><span id="line-8604"> throw new RuntimeException("Unknown durability " + durability);</span> |
| <span class="source-line-no">8605</span><span id="line-8605"> }</span> |
| <span class="source-line-no">8606</span><span id="line-8606"> }</span> |
| <span class="source-line-no">8607</span><span id="line-8607"> }</span> |
| <span class="source-line-no">8608</span><span id="line-8608"></span> |
| <span class="source-line-no">8609</span><span id="line-8609"> /**</span> |
| <span class="source-line-no">8610</span><span id="line-8610"> * Check whether we should sync the wal from the table's durability settings</span> |
| <span class="source-line-no">8611</span><span id="line-8611"> */</span> |
| <span class="source-line-no">8612</span><span id="line-8612"> private boolean shouldSyncWAL() {</span> |
| <span class="source-line-no">8613</span><span id="line-8613"> return regionDurability.ordinal() > Durability.ASYNC_WAL.ordinal();</span> |
| <span class="source-line-no">8614</span><span id="line-8614"> }</span> |
| <span class="source-line-no">8615</span><span id="line-8615"></span> |
| <span class="source-line-no">8616</span><span id="line-8616"> /** Returns the latest sequence number that was read from storage when this region was opened */</span> |
| <span class="source-line-no">8617</span><span id="line-8617"> public long getOpenSeqNum() {</span> |
| <span class="source-line-no">8618</span><span id="line-8618"> return this.openSeqNum;</span> |
| <span class="source-line-no">8619</span><span id="line-8619"> }</span> |
| <span class="source-line-no">8620</span><span id="line-8620"></span> |
| <span class="source-line-no">8621</span><span id="line-8621"> @Override</span> |
| <span class="source-line-no">8622</span><span id="line-8622"> public Map<byte[], Long> getMaxStoreSeqId() {</span> |
| <span class="source-line-no">8623</span><span id="line-8623"> return this.maxSeqIdInStores;</span> |
| <span class="source-line-no">8624</span><span id="line-8624"> }</span> |
| <span class="source-line-no">8625</span><span id="line-8625"></span> |
| <span class="source-line-no">8626</span><span id="line-8626"> public long getOldestSeqIdOfStore(byte[] familyName) {</span> |
| <span class="source-line-no">8627</span><span id="line-8627"> return wal.getEarliestMemStoreSeqNum(getRegionInfo().getEncodedNameAsBytes(), familyName);</span> |
| <span class="source-line-no">8628</span><span id="line-8628"> }</span> |
| <span class="source-line-no">8629</span><span id="line-8629"></span> |
| <span class="source-line-no">8630</span><span id="line-8630"> @Override</span> |
| <span class="source-line-no">8631</span><span id="line-8631"> public CompactionState getCompactionState() {</span> |
| <span class="source-line-no">8632</span><span id="line-8632"> boolean hasMajor = majorInProgress.get() > 0, hasMinor = minorInProgress.get() > 0;</span> |
| <span class="source-line-no">8633</span><span id="line-8633"> return (hasMajor</span> |
| <span class="source-line-no">8634</span><span id="line-8634"> ? (hasMinor ? CompactionState.MAJOR_AND_MINOR : CompactionState.MAJOR)</span> |
| <span class="source-line-no">8635</span><span id="line-8635"> : (hasMinor ? CompactionState.MINOR : CompactionState.NONE));</span> |
| <span class="source-line-no">8636</span><span id="line-8636"> }</span> |
| <span class="source-line-no">8637</span><span id="line-8637"></span> |
| <span class="source-line-no">8638</span><span id="line-8638"> public void reportCompactionRequestStart(boolean isMajor) {</span> |
| <span class="source-line-no">8639</span><span id="line-8639"> (isMajor ? majorInProgress : minorInProgress).incrementAndGet();</span> |
| <span class="source-line-no">8640</span><span id="line-8640"> }</span> |
| <span class="source-line-no">8641</span><span id="line-8641"></span> |
| <span class="source-line-no">8642</span><span id="line-8642"> public void reportCompactionRequestEnd(boolean isMajor, int numFiles, long filesSizeCompacted) {</span> |
| <span class="source-line-no">8643</span><span id="line-8643"> int newValue = (isMajor ? majorInProgress : minorInProgress).decrementAndGet();</span> |
| <span class="source-line-no">8644</span><span id="line-8644"></span> |
| <span class="source-line-no">8645</span><span id="line-8645"> // metrics</span> |
| <span class="source-line-no">8646</span><span id="line-8646"> compactionsFinished.increment();</span> |
| <span class="source-line-no">8647</span><span id="line-8647"> compactionNumFilesCompacted.add(numFiles);</span> |
| <span class="source-line-no">8648</span><span id="line-8648"> compactionNumBytesCompacted.add(filesSizeCompacted);</span> |
| <span class="source-line-no">8649</span><span id="line-8649"></span> |
| <span class="source-line-no">8650</span><span id="line-8650"> assert newValue >= 0;</span> |
| <span class="source-line-no">8651</span><span id="line-8651"> }</span> |
| <span class="source-line-no">8652</span><span id="line-8652"></span> |
| <span class="source-line-no">8653</span><span id="line-8653"> public void reportCompactionRequestFailure() {</span> |
| <span class="source-line-no">8654</span><span id="line-8654"> compactionsFailed.increment();</span> |
| <span class="source-line-no">8655</span><span id="line-8655"> }</span> |
| <span class="source-line-no">8656</span><span id="line-8656"></span> |
| <span class="source-line-no">8657</span><span id="line-8657"> public void incrementCompactionsQueuedCount() {</span> |
| <span class="source-line-no">8658</span><span id="line-8658"> compactionsQueued.increment();</span> |
| <span class="source-line-no">8659</span><span id="line-8659"> }</span> |
| <span class="source-line-no">8660</span><span id="line-8660"></span> |
| <span class="source-line-no">8661</span><span id="line-8661"> public void decrementCompactionsQueuedCount() {</span> |
| <span class="source-line-no">8662</span><span id="line-8662"> compactionsQueued.decrement();</span> |
| <span class="source-line-no">8663</span><span id="line-8663"> }</span> |
| <span class="source-line-no">8664</span><span id="line-8664"></span> |
| <span class="source-line-no">8665</span><span id="line-8665"> public void incrementFlushesQueuedCount() {</span> |
| <span class="source-line-no">8666</span><span id="line-8666"> flushesQueued.increment();</span> |
| <span class="source-line-no">8667</span><span id="line-8667"> }</span> |
| <span class="source-line-no">8668</span><span id="line-8668"></span> |
| <span class="source-line-no">8669</span><span id="line-8669"> protected void decrementFlushesQueuedCount() {</span> |
| <span class="source-line-no">8670</span><span id="line-8670"> flushesQueued.decrement();</span> |
| <span class="source-line-no">8671</span><span id="line-8671"> }</span> |
| <span class="source-line-no">8672</span><span id="line-8672"></span> |
| <span class="source-line-no">8673</span><span id="line-8673"> /**</span> |
| <span class="source-line-no">8674</span><span id="line-8674"> * If a handler thread is eligible for interrupt, make it ineligible. Should be paired with</span> |
| <span class="source-line-no">8675</span><span id="line-8675"> * {{@link #enableInterrupts()}.</span> |
| <span class="source-line-no">8676</span><span id="line-8676"> */</span> |
| <span class="source-line-no">8677</span><span id="line-8677"> void disableInterrupts() {</span> |
| <span class="source-line-no">8678</span><span id="line-8678"> regionLockHolders.computeIfPresent(Thread.currentThread(), (t, b) -> false);</span> |
| <span class="source-line-no">8679</span><span id="line-8679"> }</span> |
| <span class="source-line-no">8680</span><span id="line-8680"></span> |
| <span class="source-line-no">8681</span><span id="line-8681"> /**</span> |
| <span class="source-line-no">8682</span><span id="line-8682"> * If a handler thread was made ineligible for interrupt via {{@link #disableInterrupts()}, make</span> |
| <span class="source-line-no">8683</span><span id="line-8683"> * it eligible again. No-op if interrupts are already enabled.</span> |
| <span class="source-line-no">8684</span><span id="line-8684"> */</span> |
| <span class="source-line-no">8685</span><span id="line-8685"> void enableInterrupts() {</span> |
| <span class="source-line-no">8686</span><span id="line-8686"> regionLockHolders.computeIfPresent(Thread.currentThread(), (t, b) -> true);</span> |
| <span class="source-line-no">8687</span><span id="line-8687"> }</span> |
| <span class="source-line-no">8688</span><span id="line-8688"></span> |
| <span class="source-line-no">8689</span><span id="line-8689"> /**</span> |
| <span class="source-line-no">8690</span><span id="line-8690"> * Interrupt any region options that have acquired the region lock via</span> |
| <span class="source-line-no">8691</span><span id="line-8691"> * {@link #startRegionOperation(org.apache.hadoop.hbase.regionserver.Region.Operation)}, or</span> |
| <span class="source-line-no">8692</span><span id="line-8692"> * {@link #startBulkRegionOperation(boolean)}.</span> |
| <span class="source-line-no">8693</span><span id="line-8693"> */</span> |
| <span class="source-line-no">8694</span><span id="line-8694"> private void interruptRegionOperations() {</span> |
| <span class="source-line-no">8695</span><span id="line-8695"> for (Map.Entry<Thread, Boolean> entry : regionLockHolders.entrySet()) {</span> |
| <span class="source-line-no">8696</span><span id="line-8696"> // An entry in this map will have a boolean value indicating if it is currently</span> |
| <span class="source-line-no">8697</span><span id="line-8697"> // eligible for interrupt; if so, we should interrupt it.</span> |
| <span class="source-line-no">8698</span><span id="line-8698"> if (entry.getValue().booleanValue()) {</span> |
| <span class="source-line-no">8699</span><span id="line-8699"> entry.getKey().interrupt();</span> |
| <span class="source-line-no">8700</span><span id="line-8700"> }</span> |
| <span class="source-line-no">8701</span><span id="line-8701"> }</span> |
| <span class="source-line-no">8702</span><span id="line-8702"> }</span> |
| <span class="source-line-no">8703</span><span id="line-8703"></span> |
| <span class="source-line-no">8704</span><span id="line-8704"> /**</span> |
| <span class="source-line-no">8705</span><span id="line-8705"> * Check thread interrupt status and throw an exception if interrupted.</span> |
| <span class="source-line-no">8706</span><span id="line-8706"> * @throws NotServingRegionException if region is closing</span> |
| <span class="source-line-no">8707</span><span id="line-8707"> * @throws InterruptedIOException if interrupted but region is not closing</span> |
| <span class="source-line-no">8708</span><span id="line-8708"> */</span> |
| <span class="source-line-no">8709</span><span id="line-8709"> // Package scope for tests</span> |
| <span class="source-line-no">8710</span><span id="line-8710"> void checkInterrupt() throws NotServingRegionException, InterruptedIOException {</span> |
| <span class="source-line-no">8711</span><span id="line-8711"> if (Thread.interrupted()) {</span> |
| <span class="source-line-no">8712</span><span id="line-8712"> if (this.closing.get()) {</span> |
| <span class="source-line-no">8713</span><span id="line-8713"> throw new NotServingRegionException(</span> |
| <span class="source-line-no">8714</span><span id="line-8714"> getRegionInfo().getRegionNameAsString() + " is closing");</span> |
| <span class="source-line-no">8715</span><span id="line-8715"> }</span> |
| <span class="source-line-no">8716</span><span id="line-8716"> throw new InterruptedIOException();</span> |
| <span class="source-line-no">8717</span><span id="line-8717"> }</span> |
| <span class="source-line-no">8718</span><span id="line-8718"> }</span> |
| <span class="source-line-no">8719</span><span id="line-8719"></span> |
| <span class="source-line-no">8720</span><span id="line-8720"> /**</span> |
| <span class="source-line-no">8721</span><span id="line-8721"> * Throw the correct exception upon interrupt</span> |
| <span class="source-line-no">8722</span><span id="line-8722"> * @param t cause</span> |
| <span class="source-line-no">8723</span><span id="line-8723"> */</span> |
| <span class="source-line-no">8724</span><span id="line-8724"> // Package scope for tests</span> |
| <span class="source-line-no">8725</span><span id="line-8725"> IOException throwOnInterrupt(Throwable t) {</span> |
| <span class="source-line-no">8726</span><span id="line-8726"> if (this.closing.get()) {</span> |
| <span class="source-line-no">8727</span><span id="line-8727"> return (NotServingRegionException) new NotServingRegionException(</span> |
| <span class="source-line-no">8728</span><span id="line-8728"> getRegionInfo().getRegionNameAsString() + " is closing").initCause(t);</span> |
| <span class="source-line-no">8729</span><span id="line-8729"> }</span> |
| <span class="source-line-no">8730</span><span id="line-8730"> return (InterruptedIOException) new InterruptedIOException().initCause(t);</span> |
| <span class="source-line-no">8731</span><span id="line-8731"> }</span> |
| <span class="source-line-no">8732</span><span id="line-8732"></span> |
| <span class="source-line-no">8733</span><span id="line-8733"> /**</span> |
| <span class="source-line-no">8734</span><span id="line-8734"> * {@inheritDoc}</span> |
| <span class="source-line-no">8735</span><span id="line-8735"> */</span> |
| <span class="source-line-no">8736</span><span id="line-8736"> @Override</span> |
| <span class="source-line-no">8737</span><span id="line-8737"> public void onConfigurationChange(Configuration conf) {</span> |
| <span class="source-line-no">8738</span><span id="line-8738"> this.storeHotnessProtector.update(conf);</span> |
| <span class="source-line-no">8739</span><span id="line-8739"> // update coprocessorHost if the configuration has changed.</span> |
| <span class="source-line-no">8740</span><span id="line-8740"> if (</span> |
| <span class="source-line-no">8741</span><span id="line-8741"> CoprocessorConfigurationUtil.checkConfigurationChange(getReadOnlyConfiguration(), conf,</span> |
| <span class="source-line-no">8742</span><span id="line-8742"> CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,</span> |
| <span class="source-line-no">8743</span><span id="line-8743"> CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY)</span> |
| <span class="source-line-no">8744</span><span id="line-8744"> ) {</span> |
| <span class="source-line-no">8745</span><span id="line-8745"> LOG.info("Update the system coprocessors because the configuration has changed");</span> |
| <span class="source-line-no">8746</span><span id="line-8746"> decorateRegionConfiguration(conf);</span> |
| <span class="source-line-no">8747</span><span id="line-8747"> this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);</span> |
| <span class="source-line-no">8748</span><span id="line-8748"> }</span> |
| <span class="source-line-no">8749</span><span id="line-8749"> }</span> |
| <span class="source-line-no">8750</span><span id="line-8750"></span> |
| <span class="source-line-no">8751</span><span id="line-8751"> /**</span> |
| <span class="source-line-no">8752</span><span id="line-8752"> * {@inheritDoc}</span> |
| <span class="source-line-no">8753</span><span id="line-8753"> */</span> |
| <span class="source-line-no">8754</span><span id="line-8754"> @Override</span> |
| <span class="source-line-no">8755</span><span id="line-8755"> public void registerChildren(ConfigurationManager manager) {</span> |
| <span class="source-line-no">8756</span><span id="line-8756"> configurationManager = manager;</span> |
| <span class="source-line-no">8757</span><span id="line-8757"> stores.values().forEach(manager::registerObserver);</span> |
| <span class="source-line-no">8758</span><span id="line-8758"> }</span> |
| <span class="source-line-no">8759</span><span id="line-8759"></span> |
| <span class="source-line-no">8760</span><span id="line-8760"> /**</span> |
| <span class="source-line-no">8761</span><span id="line-8761"> * {@inheritDoc}</span> |
| <span class="source-line-no">8762</span><span id="line-8762"> */</span> |
| <span class="source-line-no">8763</span><span id="line-8763"> @Override</span> |
| <span class="source-line-no">8764</span><span id="line-8764"> public void deregisterChildren(ConfigurationManager manager) {</span> |
| <span class="source-line-no">8765</span><span id="line-8765"> stores.values().forEach(configurationManager::deregisterObserver);</span> |
| <span class="source-line-no">8766</span><span id="line-8766"> }</span> |
| <span class="source-line-no">8767</span><span id="line-8767"></span> |
| <span class="source-line-no">8768</span><span id="line-8768"> @Override</span> |
| <span class="source-line-no">8769</span><span id="line-8769"> public CellComparator getCellComparator() {</span> |
| <span class="source-line-no">8770</span><span id="line-8770"> return cellComparator;</span> |
| <span class="source-line-no">8771</span><span id="line-8771"> }</span> |
| <span class="source-line-no">8772</span><span id="line-8772"></span> |
| <span class="source-line-no">8773</span><span id="line-8773"> public long getMemStoreFlushSize() {</span> |
| <span class="source-line-no">8774</span><span id="line-8774"> return this.memstoreFlushSize;</span> |
| <span class="source-line-no">8775</span><span id="line-8775"> }</span> |
| <span class="source-line-no">8776</span><span id="line-8776"></span> |
| <span class="source-line-no">8777</span><span id="line-8777"> //// method for debugging tests</span> |
| <span class="source-line-no">8778</span><span id="line-8778"> void throwException(String title, String regionName) {</span> |
| <span class="source-line-no">8779</span><span id="line-8779"> StringBuilder buf = new StringBuilder();</span> |
| <span class="source-line-no">8780</span><span id="line-8780"> buf.append(title + ", ");</span> |
| <span class="source-line-no">8781</span><span id="line-8781"> buf.append(getRegionInfo().toString());</span> |
| <span class="source-line-no">8782</span><span id="line-8782"> buf.append(getRegionInfo().isMetaRegion() ? " meta region " : " ");</span> |
| <span class="source-line-no">8783</span><span id="line-8783"> buf.append("stores: ");</span> |
| <span class="source-line-no">8784</span><span id="line-8784"> for (HStore s : stores.values()) {</span> |
| <span class="source-line-no">8785</span><span id="line-8785"> buf.append(s.getColumnFamilyDescriptor().getNameAsString());</span> |
| <span class="source-line-no">8786</span><span id="line-8786"> buf.append(" size: ");</span> |
| <span class="source-line-no">8787</span><span id="line-8787"> buf.append(s.getMemStoreSize().getDataSize());</span> |
| <span class="source-line-no">8788</span><span id="line-8788"> buf.append(" ");</span> |
| <span class="source-line-no">8789</span><span id="line-8789"> }</span> |
| <span class="source-line-no">8790</span><span id="line-8790"> buf.append("end-of-stores");</span> |
| <span class="source-line-no">8791</span><span id="line-8791"> buf.append(", memstore size ");</span> |
| <span class="source-line-no">8792</span><span id="line-8792"> buf.append(getMemStoreDataSize());</span> |
| <span class="source-line-no">8793</span><span id="line-8793"> if (getRegionInfo().getRegionNameAsString().startsWith(regionName)) {</span> |
| <span class="source-line-no">8794</span><span id="line-8794"> throw new RuntimeException(buf.toString());</span> |
| <span class="source-line-no">8795</span><span id="line-8795"> }</span> |
| <span class="source-line-no">8796</span><span id="line-8796"> }</span> |
| <span class="source-line-no">8797</span><span id="line-8797"></span> |
| <span class="source-line-no">8798</span><span id="line-8798"> @Override</span> |
| <span class="source-line-no">8799</span><span id="line-8799"> public void requestCompaction(String why, int priority, boolean major,</span> |
| <span class="source-line-no">8800</span><span id="line-8800"> CompactionLifeCycleTracker tracker) throws IOException {</span> |
| <span class="source-line-no">8801</span><span id="line-8801"> if (major) {</span> |
| <span class="source-line-no">8802</span><span id="line-8802"> stores.values().forEach(HStore::triggerMajorCompaction);</span> |
| <span class="source-line-no">8803</span><span id="line-8803"> }</span> |
| <span class="source-line-no">8804</span><span id="line-8804"> rsServices.getCompactionRequestor().requestCompaction(this, why, priority, tracker,</span> |
| <span class="source-line-no">8805</span><span id="line-8805"> RpcServer.getRequestUser().orElse(null));</span> |
| <span class="source-line-no">8806</span><span id="line-8806"> }</span> |
| <span class="source-line-no">8807</span><span id="line-8807"></span> |
| <span class="source-line-no">8808</span><span id="line-8808"> @Override</span> |
| <span class="source-line-no">8809</span><span id="line-8809"> public void requestCompaction(byte[] family, String why, int priority, boolean major,</span> |
| <span class="source-line-no">8810</span><span id="line-8810"> CompactionLifeCycleTracker tracker) throws IOException {</span> |
| <span class="source-line-no">8811</span><span id="line-8811"> HStore store = stores.get(family);</span> |
| <span class="source-line-no">8812</span><span id="line-8812"> if (store == null) {</span> |
| <span class="source-line-no">8813</span><span id="line-8813"> throw new NoSuchColumnFamilyException("column family " + Bytes.toString(family)</span> |
| <span class="source-line-no">8814</span><span id="line-8814"> + " does not exist in region " + getRegionInfo().getRegionNameAsString());</span> |
| <span class="source-line-no">8815</span><span id="line-8815"> }</span> |
| <span class="source-line-no">8816</span><span id="line-8816"> if (major) {</span> |
| <span class="source-line-no">8817</span><span id="line-8817"> store.triggerMajorCompaction();</span> |
| <span class="source-line-no">8818</span><span id="line-8818"> }</span> |
| <span class="source-line-no">8819</span><span id="line-8819"> rsServices.getCompactionRequestor().requestCompaction(this, store, why, priority, tracker,</span> |
| <span class="source-line-no">8820</span><span id="line-8820"> RpcServer.getRequestUser().orElse(null));</span> |
| <span class="source-line-no">8821</span><span id="line-8821"> }</span> |
| <span class="source-line-no">8822</span><span id="line-8822"></span> |
| <span class="source-line-no">8823</span><span id="line-8823"> private void requestFlushIfNeeded() throws RegionTooBusyException {</span> |
| <span class="source-line-no">8824</span><span id="line-8824"> if (isFlushSize(this.memStoreSizing.getMemStoreSize())) {</span> |
| <span class="source-line-no">8825</span><span id="line-8825"> requestFlush();</span> |
| <span class="source-line-no">8826</span><span id="line-8826"> }</span> |
| <span class="source-line-no">8827</span><span id="line-8827"> }</span> |
| <span class="source-line-no">8828</span><span id="line-8828"></span> |
| <span class="source-line-no">8829</span><span id="line-8829"> private void requestFlush() {</span> |
| <span class="source-line-no">8830</span><span id="line-8830"> if (this.rsServices == null) {</span> |
| <span class="source-line-no">8831</span><span id="line-8831"> return;</span> |
| <span class="source-line-no">8832</span><span id="line-8832"> }</span> |
| <span class="source-line-no">8833</span><span id="line-8833"> requestFlush0(FlushLifeCycleTracker.DUMMY);</span> |
| <span class="source-line-no">8834</span><span id="line-8834"> }</span> |
| <span class="source-line-no">8835</span><span id="line-8835"></span> |
| <span class="source-line-no">8836</span><span id="line-8836"> private void requestFlush0(FlushLifeCycleTracker tracker) {</span> |
| <span class="source-line-no">8837</span><span id="line-8837"> boolean shouldFlush = false;</span> |
| <span class="source-line-no">8838</span><span id="line-8838"> synchronized (writestate) {</span> |
| <span class="source-line-no">8839</span><span id="line-8839"> if (!this.writestate.isFlushRequested()) {</span> |
| <span class="source-line-no">8840</span><span id="line-8840"> shouldFlush = true;</span> |
| <span class="source-line-no">8841</span><span id="line-8841"> writestate.flushRequested = true;</span> |
| <span class="source-line-no">8842</span><span id="line-8842"> }</span> |
| <span class="source-line-no">8843</span><span id="line-8843"> }</span> |
| <span class="source-line-no">8844</span><span id="line-8844"> if (shouldFlush) {</span> |
| <span class="source-line-no">8845</span><span id="line-8845"> // Make request outside of synchronize block; HBASE-818.</span> |
| <span class="source-line-no">8846</span><span id="line-8846"> this.rsServices.getFlushRequester().requestFlush(this, tracker);</span> |
| <span class="source-line-no">8847</span><span id="line-8847"> if (LOG.isDebugEnabled()) {</span> |
| <span class="source-line-no">8848</span><span id="line-8848"> LOG.debug("Flush requested on " + this.getRegionInfo().getEncodedName());</span> |
| <span class="source-line-no">8849</span><span id="line-8849"> }</span> |
| <span class="source-line-no">8850</span><span id="line-8850"> } else {</span> |
| <span class="source-line-no">8851</span><span id="line-8851"> tracker.notExecuted("Flush already requested on " + this);</span> |
| <span class="source-line-no">8852</span><span id="line-8852"> }</span> |
| <span class="source-line-no">8853</span><span id="line-8853"> }</span> |
| <span class="source-line-no">8854</span><span id="line-8854"></span> |
| <span class="source-line-no">8855</span><span id="line-8855"> @Override</span> |
| <span class="source-line-no">8856</span><span id="line-8856"> public void requestFlush(FlushLifeCycleTracker tracker) throws IOException {</span> |
| <span class="source-line-no">8857</span><span id="line-8857"> requestFlush0(tracker);</span> |
| <span class="source-line-no">8858</span><span id="line-8858"> }</span> |
| <span class="source-line-no">8859</span><span id="line-8859"></span> |
| <span class="source-line-no">8860</span><span id="line-8860"> /**</span> |
| <span class="source-line-no">8861</span><span id="line-8861"> * This method modifies the region's configuration in order to inject replication-related features</span> |
| <span class="source-line-no">8862</span><span id="line-8862"> * @param conf region configurations</span> |
| <span class="source-line-no">8863</span><span id="line-8863"> */</span> |
| <span class="source-line-no">8864</span><span id="line-8864"> private static void decorateRegionConfiguration(Configuration conf) {</span> |
| <span class="source-line-no">8865</span><span id="line-8865"> if (ReplicationUtils.isReplicationForBulkLoadDataEnabled(conf)) {</span> |
| <span class="source-line-no">8866</span><span id="line-8866"> String plugins = conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, "");</span> |
| <span class="source-line-no">8867</span><span id="line-8867"> String replicationCoprocessorClass = ReplicationObserver.class.getCanonicalName();</span> |
| <span class="source-line-no">8868</span><span id="line-8868"> if (!plugins.contains(replicationCoprocessorClass)) {</span> |
| <span class="source-line-no">8869</span><span id="line-8869"> conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,</span> |
| <span class="source-line-no">8870</span><span id="line-8870"> (plugins.equals("") ? "" : (plugins + ",")) + replicationCoprocessorClass);</span> |
| <span class="source-line-no">8871</span><span id="line-8871"> }</span> |
| <span class="source-line-no">8872</span><span id="line-8872"> }</span> |
| <span class="source-line-no">8873</span><span id="line-8873"> }</span> |
| <span class="source-line-no">8874</span><span id="line-8874"></span> |
| <span class="source-line-no">8875</span><span id="line-8875"> public Optional<RegionReplicationSink> getRegionReplicationSink() {</span> |
| <span class="source-line-no">8876</span><span id="line-8876"> return regionReplicationSink;</span> |
| <span class="source-line-no">8877</span><span id="line-8877"> }</span> |
| <span class="source-line-no">8878</span><span id="line-8878"></span> |
| <span class="source-line-no">8879</span><span id="line-8879"> public void addReadRequestsCount(long readRequestsCount) {</span> |
| <span class="source-line-no">8880</span><span id="line-8880"> this.readRequestsCount.add(readRequestsCount);</span> |
| <span class="source-line-no">8881</span><span id="line-8881"> }</span> |
| <span class="source-line-no">8882</span><span id="line-8882"></span> |
| <span class="source-line-no">8883</span><span id="line-8883"> public void addWriteRequestsCount(long writeRequestsCount) {</span> |
| <span class="source-line-no">8884</span><span id="line-8884"> this.writeRequestsCount.add(writeRequestsCount);</span> |
| <span class="source-line-no">8885</span><span id="line-8885"> }</span> |
| <span class="source-line-no">8886</span><span id="line-8886"></span> |
| <span class="source-line-no">8887</span><span id="line-8887"> @RestrictedApi(explanation = "Should only be called in tests", link = "",</span> |
| <span class="source-line-no">8888</span><span id="line-8888"> allowedOnPath = ".*/src/test/.*")</span> |
| <span class="source-line-no">8889</span><span id="line-8889"> boolean isReadsEnabled() {</span> |
| <span class="source-line-no">8890</span><span id="line-8890"> return this.writestate.readsEnabled;</span> |
| <span class="source-line-no">8891</span><span id="line-8891"> }</span> |
| <span class="source-line-no">8892</span><span id="line-8892">}</span> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </pre> |
| </div> |
| </main> |
| </body> |
| </html> |
