| <!DOCTYPE HTML> |
| <html lang="en"> |
| <head> |
| <!-- Generated by javadoc (17) --> |
| <title>Source code</title> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <meta name="description" content="source: package: org.apache.hadoop.hbase.regionserver, class: TestHStoreFile"> |
| <meta name="generator" content="javadoc/SourceToHTMLConverter"> |
| <link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style"> |
| </head> |
| <body class="source-page"> |
| <main role="main"> |
| <div class="source-container"> |
| <pre><span class="source-line-no">001</span><span id="line-1">/*</span> |
| <span class="source-line-no">002</span><span id="line-2"> * Licensed to the Apache Software Foundation (ASF) under one</span> |
| <span class="source-line-no">003</span><span id="line-3"> * or more contributor license agreements. See the NOTICE file</span> |
| <span class="source-line-no">004</span><span id="line-4"> * distributed with this work for additional information</span> |
| <span class="source-line-no">005</span><span id="line-5"> * regarding copyright ownership. The ASF licenses this file</span> |
| <span class="source-line-no">006</span><span id="line-6"> * to you under the Apache License, Version 2.0 (the</span> |
| <span class="source-line-no">007</span><span id="line-7"> * "License"); you may not use this file except in compliance</span> |
| <span class="source-line-no">008</span><span id="line-8"> * with the License. You may obtain a copy of the License at</span> |
| <span class="source-line-no">009</span><span id="line-9"> *</span> |
| <span class="source-line-no">010</span><span id="line-10"> * http://www.apache.org/licenses/LICENSE-2.0</span> |
| <span class="source-line-no">011</span><span id="line-11"> *</span> |
| <span class="source-line-no">012</span><span id="line-12"> * Unless required by applicable law or agreed to in writing, software</span> |
| <span class="source-line-no">013</span><span id="line-13"> * distributed under the License is distributed on an "AS IS" BASIS,</span> |
| <span class="source-line-no">014</span><span id="line-14"> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> |
| <span class="source-line-no">015</span><span id="line-15"> * See the License for the specific language governing permissions and</span> |
| <span class="source-line-no">016</span><span id="line-16"> * limitations under the License.</span> |
| <span class="source-line-no">017</span><span id="line-17"> */</span> |
| <span class="source-line-no">018</span><span id="line-18">package org.apache.hadoop.hbase.regionserver;</span> |
| <span class="source-line-no">019</span><span id="line-19"></span> |
| <span class="source-line-no">020</span><span id="line-20">import static org.apache.hadoop.hbase.io.hfile.BlockCompressedSizePredicator.BLOCK_COMPRESSED_SIZE_PREDICATOR;</span> |
| <span class="source-line-no">021</span><span id="line-21">import static org.junit.Assert.assertArrayEquals;</span> |
| <span class="source-line-no">022</span><span id="line-22">import static org.junit.Assert.assertEquals;</span> |
| <span class="source-line-no">023</span><span id="line-23">import static org.junit.Assert.assertFalse;</span> |
| <span class="source-line-no">024</span><span id="line-24">import static org.junit.Assert.assertNotNull;</span> |
| <span class="source-line-no">025</span><span id="line-25">import static org.junit.Assert.assertNull;</span> |
| <span class="source-line-no">026</span><span id="line-26">import static org.junit.Assert.assertTrue;</span> |
| <span class="source-line-no">027</span><span id="line-27">import static org.junit.Assert.fail;</span> |
| <span class="source-line-no">028</span><span id="line-28">import static org.mockito.ArgumentMatchers.any;</span> |
| <span class="source-line-no">029</span><span id="line-29">import static org.mockito.Mockito.mock;</span> |
| <span class="source-line-no">030</span><span id="line-30">import static org.mockito.Mockito.when;</span> |
| <span class="source-line-no">031</span><span id="line-31"></span> |
| <span class="source-line-no">032</span><span id="line-32">import java.io.IOException;</span> |
| <span class="source-line-no">033</span><span id="line-33">import java.nio.ByteBuffer;</span> |
| <span class="source-line-no">034</span><span id="line-34">import java.util.ArrayList;</span> |
| <span class="source-line-no">035</span><span id="line-35">import java.util.Arrays;</span> |
| <span class="source-line-no">036</span><span id="line-36">import java.util.Collections;</span> |
| <span class="source-line-no">037</span><span id="line-37">import java.util.Comparator;</span> |
| <span class="source-line-no">038</span><span id="line-38">import java.util.List;</span> |
| <span class="source-line-no">039</span><span id="line-39">import java.util.Map;</span> |
| <span class="source-line-no">040</span><span id="line-40">import java.util.OptionalLong;</span> |
| <span class="source-line-no">041</span><span id="line-41">import java.util.TreeSet;</span> |
| <span class="source-line-no">042</span><span id="line-42">import java.util.function.BiFunction;</span> |
| <span class="source-line-no">043</span><span id="line-43">import org.apache.hadoop.conf.Configuration;</span> |
| <span class="source-line-no">044</span><span id="line-44">import org.apache.hadoop.fs.FileSystem;</span> |
| <span class="source-line-no">045</span><span id="line-45">import org.apache.hadoop.fs.Path;</span> |
| <span class="source-line-no">046</span><span id="line-46">import org.apache.hadoop.hbase.Cell;</span> |
| <span class="source-line-no">047</span><span id="line-47">import org.apache.hadoop.hbase.CellUtil;</span> |
| <span class="source-line-no">048</span><span id="line-48">import org.apache.hadoop.hbase.ExtendedCell;</span> |
| <span class="source-line-no">049</span><span id="line-49">import org.apache.hadoop.hbase.HBaseClassTestRule;</span> |
| <span class="source-line-no">050</span><span id="line-50">import org.apache.hadoop.hbase.HBaseTestingUtil;</span> |
| <span class="source-line-no">051</span><span id="line-51">import org.apache.hadoop.hbase.HConstants;</span> |
| <span class="source-line-no">052</span><span id="line-52">import org.apache.hadoop.hbase.KeyValue;</span> |
| <span class="source-line-no">053</span><span id="line-53">import org.apache.hadoop.hbase.KeyValueUtil;</span> |
| <span class="source-line-no">054</span><span id="line-54">import org.apache.hadoop.hbase.PrivateCellUtil;</span> |
| <span class="source-line-no">055</span><span id="line-55">import org.apache.hadoop.hbase.TableDescriptors;</span> |
| <span class="source-line-no">056</span><span id="line-56">import org.apache.hadoop.hbase.TableName;</span> |
| <span class="source-line-no">057</span><span id="line-57">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;</span> |
| <span class="source-line-no">058</span><span id="line-58">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;</span> |
| <span class="source-line-no">059</span><span id="line-59">import org.apache.hadoop.hbase.client.RegionInfo;</span> |
| <span class="source-line-no">060</span><span id="line-60">import org.apache.hadoop.hbase.client.RegionInfoBuilder;</span> |
| <span class="source-line-no">061</span><span id="line-61">import org.apache.hadoop.hbase.client.Scan;</span> |
| <span class="source-line-no">062</span><span id="line-62">import org.apache.hadoop.hbase.client.TableDescriptor;</span> |
| <span class="source-line-no">063</span><span id="line-63">import org.apache.hadoop.hbase.client.TableDescriptorBuilder;</span> |
| <span class="source-line-no">064</span><span id="line-64">import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;</span> |
| <span class="source-line-no">065</span><span id="line-65">import org.apache.hadoop.hbase.io.HFileLink;</span> |
| <span class="source-line-no">066</span><span id="line-66">import org.apache.hadoop.hbase.io.compress.Compression;</span> |
| <span class="source-line-no">067</span><span id="line-67">import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;</span> |
| <span class="source-line-no">068</span><span id="line-68">import org.apache.hadoop.hbase.io.hfile.BlockCache;</span> |
| <span class="source-line-no">069</span><span id="line-69">import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory;</span> |
| <span class="source-line-no">070</span><span id="line-70">import org.apache.hadoop.hbase.io.hfile.CacheConfig;</span> |
| <span class="source-line-no">071</span><span id="line-71">import org.apache.hadoop.hbase.io.hfile.CacheStats;</span> |
| <span class="source-line-no">072</span><span id="line-72">import org.apache.hadoop.hbase.io.hfile.FixedFileTrailer;</span> |
| <span class="source-line-no">073</span><span id="line-73">import org.apache.hadoop.hbase.io.hfile.HFile;</span> |
| <span class="source-line-no">074</span><span id="line-74">import org.apache.hadoop.hbase.io.hfile.HFileBlock;</span> |
| <span class="source-line-no">075</span><span id="line-75">import org.apache.hadoop.hbase.io.hfile.HFileContext;</span> |
| <span class="source-line-no">076</span><span id="line-76">import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;</span> |
| <span class="source-line-no">077</span><span id="line-77">import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;</span> |
| <span class="source-line-no">078</span><span id="line-78">import org.apache.hadoop.hbase.io.hfile.HFileScanner;</span> |
| <span class="source-line-no">079</span><span id="line-79">import org.apache.hadoop.hbase.io.hfile.PreviousBlockCompressionRatePredicator;</span> |
| <span class="source-line-no">080</span><span id="line-80">import org.apache.hadoop.hbase.io.hfile.ReaderContext;</span> |
| <span class="source-line-no">081</span><span id="line-81">import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder;</span> |
| <span class="source-line-no">082</span><span id="line-82">import org.apache.hadoop.hbase.io.hfile.UncompressedBlockSizePredicator;</span> |
| <span class="source-line-no">083</span><span id="line-83">import org.apache.hadoop.hbase.master.MasterServices;</span> |
| <span class="source-line-no">084</span><span id="line-84">import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;</span> |
| <span class="source-line-no">085</span><span id="line-85">import org.apache.hadoop.hbase.testclassification.MediumTests;</span> |
| <span class="source-line-no">086</span><span id="line-86">import org.apache.hadoop.hbase.testclassification.RegionServerTests;</span> |
| <span class="source-line-no">087</span><span id="line-87">import org.apache.hadoop.hbase.util.BloomFilterFactory;</span> |
| <span class="source-line-no">088</span><span id="line-88">import org.apache.hadoop.hbase.util.Bytes;</span> |
| <span class="source-line-no">089</span><span id="line-89">import org.apache.hadoop.hbase.util.ChecksumType;</span> |
| <span class="source-line-no">090</span><span id="line-90">import org.apache.hadoop.hbase.util.CommonFSUtils;</span> |
| <span class="source-line-no">091</span><span id="line-91">import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;</span> |
| <span class="source-line-no">092</span><span id="line-92">import org.junit.AfterClass;</span> |
| <span class="source-line-no">093</span><span id="line-93">import org.junit.Before;</span> |
| <span class="source-line-no">094</span><span id="line-94">import org.junit.ClassRule;</span> |
| <span class="source-line-no">095</span><span id="line-95">import org.junit.Rule;</span> |
| <span class="source-line-no">096</span><span id="line-96">import org.junit.Test;</span> |
| <span class="source-line-no">097</span><span id="line-97">import org.junit.experimental.categories.Category;</span> |
| <span class="source-line-no">098</span><span id="line-98">import org.junit.rules.TestName;</span> |
| <span class="source-line-no">099</span><span id="line-99">import org.mockito.Mockito;</span> |
| <span class="source-line-no">100</span><span id="line-100">import org.slf4j.Logger;</span> |
| <span class="source-line-no">101</span><span id="line-101">import org.slf4j.LoggerFactory;</span> |
| <span class="source-line-no">102</span><span id="line-102"></span> |
| <span class="source-line-no">103</span><span id="line-103">import org.apache.hbase.thirdparty.com.google.common.base.Joiner;</span> |
| <span class="source-line-no">104</span><span id="line-104">import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;</span> |
| <span class="source-line-no">105</span><span id="line-105">import org.apache.hbase.thirdparty.com.google.common.collect.Lists;</span> |
| <span class="source-line-no">106</span><span id="line-106"></span> |
| <span class="source-line-no">107</span><span id="line-107">/**</span> |
| <span class="source-line-no">108</span><span id="line-108"> * Test HStoreFile</span> |
| <span class="source-line-no">109</span><span id="line-109"> */</span> |
| <span class="source-line-no">110</span><span id="line-110">@Category({ RegionServerTests.class, MediumTests.class })</span> |
| <span class="source-line-no">111</span><span id="line-111">public class TestHStoreFile {</span> |
| <span class="source-line-no">112</span><span id="line-112"></span> |
| <span class="source-line-no">113</span><span id="line-113"> @ClassRule</span> |
| <span class="source-line-no">114</span><span id="line-114"> public static final HBaseClassTestRule CLASS_RULE =</span> |
| <span class="source-line-no">115</span><span id="line-115"> HBaseClassTestRule.forClass(TestHStoreFile.class);</span> |
| <span class="source-line-no">116</span><span id="line-116"></span> |
| <span class="source-line-no">117</span><span id="line-117"> private static final Logger LOG = LoggerFactory.getLogger(TestHStoreFile.class);</span> |
| <span class="source-line-no">118</span><span id="line-118"> private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();</span> |
| <span class="source-line-no">119</span><span id="line-119"> private CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());</span> |
| <span class="source-line-no">120</span><span id="line-120"> private static Path ROOT_DIR = TEST_UTIL.getDataTestDir("TestStoreFile");</span> |
| <span class="source-line-no">121</span><span id="line-121"> private static final ChecksumType CKTYPE = ChecksumType.CRC32C;</span> |
| <span class="source-line-no">122</span><span id="line-122"> private static final int CKBYTES = 512;</span> |
| <span class="source-line-no">123</span><span id="line-123"> private static String TEST_FAMILY = "cf";</span> |
| <span class="source-line-no">124</span><span id="line-124"> private static final char FIRST_CHAR = 'a';</span> |
| <span class="source-line-no">125</span><span id="line-125"> private static final char LAST_CHAR = 'z';</span> |
| <span class="source-line-no">126</span><span id="line-126"></span> |
| <span class="source-line-no">127</span><span id="line-127"> @Rule</span> |
| <span class="source-line-no">128</span><span id="line-128"> public TestName name = new TestName();</span> |
| <span class="source-line-no">129</span><span id="line-129"></span> |
| <span class="source-line-no">130</span><span id="line-130"> private Configuration conf;</span> |
| <span class="source-line-no">131</span><span id="line-131"> private Path testDir;</span> |
| <span class="source-line-no">132</span><span id="line-132"> private FileSystem fs;</span> |
| <span class="source-line-no">133</span><span id="line-133"></span> |
| <span class="source-line-no">134</span><span id="line-134"> @Before</span> |
| <span class="source-line-no">135</span><span id="line-135"> public void setUp() throws IOException {</span> |
| <span class="source-line-no">136</span><span id="line-136"> conf = TEST_UTIL.getConfiguration();</span> |
| <span class="source-line-no">137</span><span id="line-137"> testDir = TEST_UTIL.getDataTestDir(name.getMethodName());</span> |
| <span class="source-line-no">138</span><span id="line-138"> fs = testDir.getFileSystem(conf);</span> |
| <span class="source-line-no">139</span><span id="line-139"> }</span> |
| <span class="source-line-no">140</span><span id="line-140"></span> |
| <span class="source-line-no">141</span><span id="line-141"> @AfterClass</span> |
| <span class="source-line-no">142</span><span id="line-142"> public static void tearDownAfterClass() {</span> |
| <span class="source-line-no">143</span><span id="line-143"> TEST_UTIL.cleanupTestDir();</span> |
| <span class="source-line-no">144</span><span id="line-144"> }</span> |
| <span class="source-line-no">145</span><span id="line-145"></span> |
| <span class="source-line-no">146</span><span id="line-146"> /**</span> |
| <span class="source-line-no">147</span><span id="line-147"> * Write a file and then assert that we can read from top and bottom halves using two</span> |
| <span class="source-line-no">148</span><span id="line-148"> * HalfMapFiles, as well as one HalfMapFile and one HFileLink file.</span> |
| <span class="source-line-no">149</span><span id="line-149"> */</span> |
| <span class="source-line-no">150</span><span id="line-150"> @Test</span> |
| <span class="source-line-no">151</span><span id="line-151"> public void testBasicHalfAndHFileLinkMapFile() throws Exception {</span> |
| <span class="source-line-no">152</span><span id="line-152"> final RegionInfo hri =</span> |
| <span class="source-line-no">153</span><span id="line-153"> RegionInfoBuilder.newBuilder(TableName.valueOf("testBasicHalfAndHFileLinkMapFile")).build();</span> |
| <span class="source-line-no">154</span><span id="line-154"> // The locations of HFileLink refers hfiles only should be consistent with the table dir</span> |
| <span class="source-line-no">155</span><span id="line-155"> // create by CommonFSUtils directory, so we should make the region directory under</span> |
| <span class="source-line-no">156</span><span id="line-156"> // the mode of CommonFSUtils.getTableDir here.</span> |
| <span class="source-line-no">157</span><span id="line-157"> HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,</span> |
| <span class="source-line-no">158</span><span id="line-158"> CommonFSUtils.getTableDir(CommonFSUtils.getRootDir(conf), hri.getTable()), hri);</span> |
| <span class="source-line-no">159</span><span id="line-159"></span> |
| <span class="source-line-no">160</span><span id="line-160"> HFileContext meta = new HFileContextBuilder().withBlockSize(2 * 1024).build();</span> |
| <span class="source-line-no">161</span><span id="line-161"> StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)</span> |
| <span class="source-line-no">162</span><span id="line-162"> .withFilePath(regionFs.createTempName()).withFileContext(meta).build();</span> |
| <span class="source-line-no">163</span><span id="line-163"> writeStoreFile(writer);</span> |
| <span class="source-line-no">164</span><span id="line-164"></span> |
| <span class="source-line-no">165</span><span id="line-165"> Path sfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());</span> |
| <span class="source-line-no">166</span><span id="line-166"> HStoreFile sf = new HStoreFile(this.fs, sfPath, conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">167</span><span id="line-167"> checkHalfHFile(regionFs, sf);</span> |
| <span class="source-line-no">168</span><span id="line-168"> }</span> |
| <span class="source-line-no">169</span><span id="line-169"></span> |
| <span class="source-line-no">170</span><span id="line-170"> private void writeStoreFile(final StoreFileWriter writer) throws IOException {</span> |
| <span class="source-line-no">171</span><span id="line-171"> writeStoreFile(writer, Bytes.toBytes(name.getMethodName()),</span> |
| <span class="source-line-no">172</span><span id="line-172"> Bytes.toBytes(name.getMethodName()));</span> |
| <span class="source-line-no">173</span><span id="line-173"> }</span> |
| <span class="source-line-no">174</span><span id="line-174"></span> |
| <span class="source-line-no">175</span><span id="line-175"> // pick an split point (roughly halfway)</span> |
| <span class="source-line-no">176</span><span id="line-176"> byte[] SPLITKEY = new byte[] { (LAST_CHAR + FIRST_CHAR) / 2, FIRST_CHAR };</span> |
| <span class="source-line-no">177</span><span id="line-177"></span> |
| <span class="source-line-no">178</span><span id="line-178"> /*</span> |
| <span class="source-line-no">179</span><span id="line-179"> * Writes HStoreKey and ImmutableBytes data to passed writer and then closes it.</span> |
| <span class="source-line-no">180</span><span id="line-180"> */</span> |
| <span class="source-line-no">181</span><span id="line-181"> public static void writeStoreFile(final StoreFileWriter writer, byte[] fam, byte[] qualifier)</span> |
| <span class="source-line-no">182</span><span id="line-182"> throws IOException {</span> |
| <span class="source-line-no">183</span><span id="line-183"> long now = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">184</span><span id="line-184"> try {</span> |
| <span class="source-line-no">185</span><span id="line-185"> for (char d = FIRST_CHAR; d <= LAST_CHAR; d++) {</span> |
| <span class="source-line-no">186</span><span id="line-186"> for (char e = FIRST_CHAR; e <= LAST_CHAR; e++) {</span> |
| <span class="source-line-no">187</span><span id="line-187"> byte[] b = new byte[] { (byte) d, (byte) e };</span> |
| <span class="source-line-no">188</span><span id="line-188"> writer.append(new KeyValue(b, fam, qualifier, now, b));</span> |
| <span class="source-line-no">189</span><span id="line-189"> }</span> |
| <span class="source-line-no">190</span><span id="line-190"> }</span> |
| <span class="source-line-no">191</span><span id="line-191"> } finally {</span> |
| <span class="source-line-no">192</span><span id="line-192"> writer.close();</span> |
| <span class="source-line-no">193</span><span id="line-193"> }</span> |
| <span class="source-line-no">194</span><span id="line-194"> }</span> |
| <span class="source-line-no">195</span><span id="line-195"></span> |
| <span class="source-line-no">196</span><span id="line-196"> public static void writeLargeStoreFile(final StoreFileWriter writer, byte[] fam, byte[] qualifier,</span> |
| <span class="source-line-no">197</span><span id="line-197"> int rounds) throws IOException {</span> |
| <span class="source-line-no">198</span><span id="line-198"> long now = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">199</span><span id="line-199"> try {</span> |
| <span class="source-line-no">200</span><span id="line-200"> for (int i = 0; i < rounds; i++) {</span> |
| <span class="source-line-no">201</span><span id="line-201"> for (char d = FIRST_CHAR; d <= LAST_CHAR; d++) {</span> |
| <span class="source-line-no">202</span><span id="line-202"> for (char e = FIRST_CHAR; e <= LAST_CHAR; e++) {</span> |
| <span class="source-line-no">203</span><span id="line-203"> byte[] b = new byte[] { (byte) d, (byte) e };</span> |
| <span class="source-line-no">204</span><span id="line-204"> byte[] key = new byte[] { (byte) i };</span> |
| <span class="source-line-no">205</span><span id="line-205"> writer.append(new KeyValue(key, fam, qualifier, now, b));</span> |
| <span class="source-line-no">206</span><span id="line-206"> }</span> |
| <span class="source-line-no">207</span><span id="line-207"> }</span> |
| <span class="source-line-no">208</span><span id="line-208"> }</span> |
| <span class="source-line-no">209</span><span id="line-209"> } finally {</span> |
| <span class="source-line-no">210</span><span id="line-210"> writer.close();</span> |
| <span class="source-line-no">211</span><span id="line-211"> }</span> |
| <span class="source-line-no">212</span><span id="line-212"> }</span> |
| <span class="source-line-no">213</span><span id="line-213"></span> |
| <span class="source-line-no">214</span><span id="line-214"> /**</span> |
| <span class="source-line-no">215</span><span id="line-215"> * Test that our mechanism of writing store files in one region to reference store files in other</span> |
| <span class="source-line-no">216</span><span id="line-216"> * regions works.</span> |
| <span class="source-line-no">217</span><span id="line-217"> */</span> |
| <span class="source-line-no">218</span><span id="line-218"> @Test</span> |
| <span class="source-line-no">219</span><span id="line-219"> public void testReference() throws IOException {</span> |
| <span class="source-line-no">220</span><span id="line-220"> final RegionInfo hri =</span> |
| <span class="source-line-no">221</span><span id="line-221"> RegionInfoBuilder.newBuilder(TableName.valueOf("testReferenceTb")).build();</span> |
| <span class="source-line-no">222</span><span id="line-222"> HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,</span> |
| <span class="source-line-no">223</span><span id="line-223"> new Path(testDir, hri.getTable().getNameAsString()), hri);</span> |
| <span class="source-line-no">224</span><span id="line-224"></span> |
| <span class="source-line-no">225</span><span id="line-225"> HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();</span> |
| <span class="source-line-no">226</span><span id="line-226"> // Make a store file and write data to it.</span> |
| <span class="source-line-no">227</span><span id="line-227"> StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)</span> |
| <span class="source-line-no">228</span><span id="line-228"> .withFilePath(regionFs.createTempName()).withFileContext(meta).build();</span> |
| <span class="source-line-no">229</span><span id="line-229"> writeStoreFile(writer);</span> |
| <span class="source-line-no">230</span><span id="line-230"></span> |
| <span class="source-line-no">231</span><span id="line-231"> Path hsfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());</span> |
| <span class="source-line-no">232</span><span id="line-232"> HStoreFile hsf = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">233</span><span id="line-233"> hsf.initReader();</span> |
| <span class="source-line-no">234</span><span id="line-234"> StoreFileReader reader = hsf.getReader();</span> |
| <span class="source-line-no">235</span><span id="line-235"> // Split on a row, not in middle of row. Midkey returned by reader</span> |
| <span class="source-line-no">236</span><span id="line-236"> // may be in middle of row. Create new one with empty column and</span> |
| <span class="source-line-no">237</span><span id="line-237"> // timestamp.</span> |
| <span class="source-line-no">238</span><span id="line-238"> byte[] midRow = CellUtil.cloneRow(reader.midKey().get());</span> |
| <span class="source-line-no">239</span><span id="line-239"> byte[] finalRow = CellUtil.cloneRow(reader.getLastKey().get());</span> |
| <span class="source-line-no">240</span><span id="line-240"> hsf.closeStoreFile(true);</span> |
| <span class="source-line-no">241</span><span id="line-241"></span> |
| <span class="source-line-no">242</span><span id="line-242"> // Make a reference</span> |
| <span class="source-line-no">243</span><span id="line-243"> RegionInfo splitHri = RegionInfoBuilder.newBuilder(hri.getTable()).setEndKey(midRow).build();</span> |
| <span class="source-line-no">244</span><span id="line-244"> Path refPath = splitStoreFile(regionFs, splitHri, TEST_FAMILY, hsf, midRow, true);</span> |
| <span class="source-line-no">245</span><span id="line-245"> HStoreFile refHsf = new HStoreFile(this.fs, refPath, conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">246</span><span id="line-246"> refHsf.initReader();</span> |
| <span class="source-line-no">247</span><span id="line-247"> // Now confirm that I can read from the reference and that it only gets</span> |
| <span class="source-line-no">248</span><span id="line-248"> // keys from top half of the file.</span> |
| <span class="source-line-no">249</span><span id="line-249"> try (HFileScanner s = refHsf.getReader().getScanner(false, false, false)) {</span> |
| <span class="source-line-no">250</span><span id="line-250"> Cell kv = null;</span> |
| <span class="source-line-no">251</span><span id="line-251"> for (boolean first = true; (!s.isSeeked() && s.seekTo()) || s.next();) {</span> |
| <span class="source-line-no">252</span><span id="line-252"> ByteBuffer bb = ByteBuffer.wrap(((KeyValue) s.getKey()).getKey());</span> |
| <span class="source-line-no">253</span><span id="line-253"> kv = KeyValueUtil.createKeyValueFromKey(bb);</span> |
| <span class="source-line-no">254</span><span id="line-254"> if (first) {</span> |
| <span class="source-line-no">255</span><span id="line-255"> assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), midRow, 0,</span> |
| <span class="source-line-no">256</span><span id="line-256"> midRow.length));</span> |
| <span class="source-line-no">257</span><span id="line-257"> first = false;</span> |
| <span class="source-line-no">258</span><span id="line-258"> }</span> |
| <span class="source-line-no">259</span><span id="line-259"> }</span> |
| <span class="source-line-no">260</span><span id="line-260"> assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), finalRow, 0,</span> |
| <span class="source-line-no">261</span><span id="line-261"> finalRow.length));</span> |
| <span class="source-line-no">262</span><span id="line-262"> }</span> |
| <span class="source-line-no">263</span><span id="line-263"> }</span> |
| <span class="source-line-no">264</span><span id="line-264"></span> |
| <span class="source-line-no">265</span><span id="line-265"> @Test</span> |
| <span class="source-line-no">266</span><span id="line-266"> public void testStoreFileReference() throws Exception {</span> |
| <span class="source-line-no">267</span><span id="line-267"> final RegionInfo hri =</span> |
| <span class="source-line-no">268</span><span id="line-268"> RegionInfoBuilder.newBuilder(TableName.valueOf("testStoreFileReference")).build();</span> |
| <span class="source-line-no">269</span><span id="line-269"> HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,</span> |
| <span class="source-line-no">270</span><span id="line-270"> new Path(testDir, hri.getTable().getNameAsString()), hri);</span> |
| <span class="source-line-no">271</span><span id="line-271"> HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();</span> |
| <span class="source-line-no">272</span><span id="line-272"></span> |
| <span class="source-line-no">273</span><span id="line-273"> // Make a store file and write data to it.</span> |
| <span class="source-line-no">274</span><span id="line-274"> StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)</span> |
| <span class="source-line-no">275</span><span id="line-275"> .withFilePath(regionFs.createTempName()).withFileContext(meta).build();</span> |
| <span class="source-line-no">276</span><span id="line-276"> writeStoreFile(writer);</span> |
| <span class="source-line-no">277</span><span id="line-277"> Path hsfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());</span> |
| <span class="source-line-no">278</span><span id="line-278"> writer.close();</span> |
| <span class="source-line-no">279</span><span id="line-279"></span> |
| <span class="source-line-no">280</span><span id="line-280"> HStoreFile file = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">281</span><span id="line-281"> file.initReader();</span> |
| <span class="source-line-no">282</span><span id="line-282"> StoreFileReader r = file.getReader();</span> |
| <span class="source-line-no">283</span><span id="line-283"> assertNotNull(r);</span> |
| <span class="source-line-no">284</span><span id="line-284"> StoreFileScanner scanner =</span> |
| <span class="source-line-no">285</span><span id="line-285"> new StoreFileScanner(r, mock(HFileScanner.class), false, false, 0, 0, false, false);</span> |
| <span class="source-line-no">286</span><span id="line-286"></span> |
| <span class="source-line-no">287</span><span id="line-287"> // Verify after instantiating scanner refCount is increased</span> |
| <span class="source-line-no">288</span><span id="line-288"> assertTrue("Verify file is being referenced", file.isReferencedInReads());</span> |
| <span class="source-line-no">289</span><span id="line-289"> scanner.close();</span> |
| <span class="source-line-no">290</span><span id="line-290"> // Verify after closing scanner refCount is decreased</span> |
| <span class="source-line-no">291</span><span id="line-291"> assertFalse("Verify file is not being referenced", file.isReferencedInReads());</span> |
| <span class="source-line-no">292</span><span id="line-292"> }</span> |
| <span class="source-line-no">293</span><span id="line-293"></span> |
| <span class="source-line-no">294</span><span id="line-294"> @Test</span> |
| <span class="source-line-no">295</span><span id="line-295"> public void testEmptyStoreFileRestrictKeyRanges() throws Exception {</span> |
| <span class="source-line-no">296</span><span id="line-296"> StoreFileReader reader = mock(StoreFileReader.class);</span> |
| <span class="source-line-no">297</span><span id="line-297"> HStore store = mock(HStore.class);</span> |
| <span class="source-line-no">298</span><span id="line-298"> byte[] cf = Bytes.toBytes("ty");</span> |
| <span class="source-line-no">299</span><span id="line-299"> ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.of(cf);</span> |
| <span class="source-line-no">300</span><span id="line-300"> when(store.getColumnFamilyDescriptor()).thenReturn(cfd);</span> |
| <span class="source-line-no">301</span><span id="line-301"> try (StoreFileScanner scanner =</span> |
| <span class="source-line-no">302</span><span id="line-302"> new StoreFileScanner(reader, mock(HFileScanner.class), false, false, 0, 0, true, false)) {</span> |
| <span class="source-line-no">303</span><span id="line-303"> Scan scan = new Scan();</span> |
| <span class="source-line-no">304</span><span id="line-304"> scan.setColumnFamilyTimeRange(cf, 0, 1);</span> |
| <span class="source-line-no">305</span><span id="line-305"> assertFalse(scanner.shouldUseScanner(scan, store, 0));</span> |
| <span class="source-line-no">306</span><span id="line-306"> }</span> |
| <span class="source-line-no">307</span><span id="line-307"> }</span> |
| <span class="source-line-no">308</span><span id="line-308"></span> |
| <span class="source-line-no">309</span><span id="line-309"> @Test</span> |
| <span class="source-line-no">310</span><span id="line-310"> public void testHFileLink() throws IOException {</span> |
| <span class="source-line-no">311</span><span id="line-311"> final RegionInfo hri =</span> |
| <span class="source-line-no">312</span><span id="line-312"> RegionInfoBuilder.newBuilder(TableName.valueOf("testHFileLinkTb")).build();</span> |
| <span class="source-line-no">313</span><span id="line-313"> // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/</span> |
| <span class="source-line-no">314</span><span id="line-314"> Configuration testConf = new Configuration(this.conf);</span> |
| <span class="source-line-no">315</span><span id="line-315"> CommonFSUtils.setRootDir(testConf, testDir);</span> |
| <span class="source-line-no">316</span><span id="line-316"> HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,</span> |
| <span class="source-line-no">317</span><span id="line-317"> CommonFSUtils.getTableDir(testDir, hri.getTable()), hri);</span> |
| <span class="source-line-no">318</span><span id="line-318"> HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();</span> |
| <span class="source-line-no">319</span><span id="line-319"></span> |
| <span class="source-line-no">320</span><span id="line-320"> // Make a store file and write data to it.</span> |
| <span class="source-line-no">321</span><span id="line-321"> StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)</span> |
| <span class="source-line-no">322</span><span id="line-322"> .withFilePath(regionFs.createTempName()).withFileContext(meta).build();</span> |
| <span class="source-line-no">323</span><span id="line-323"> writeStoreFile(writer);</span> |
| <span class="source-line-no">324</span><span id="line-324"></span> |
| <span class="source-line-no">325</span><span id="line-325"> Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());</span> |
| <span class="source-line-no">326</span><span id="line-326"> Path dstPath = new Path(regionFs.getTableDir(), new Path("test-region", TEST_FAMILY));</span> |
| <span class="source-line-no">327</span><span id="line-327"> HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());</span> |
| <span class="source-line-no">328</span><span id="line-328"> Path linkFilePath =</span> |
| <span class="source-line-no">329</span><span id="line-329"> new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));</span> |
| <span class="source-line-no">330</span><span id="line-330"></span> |
| <span class="source-line-no">331</span><span id="line-331"> // Try to open store file from link</span> |
| <span class="source-line-no">332</span><span id="line-332"> StoreFileInfo storeFileInfo = new StoreFileInfo(testConf, this.fs, linkFilePath, true);</span> |
| <span class="source-line-no">333</span><span id="line-333"> HStoreFile hsf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf);</span> |
| <span class="source-line-no">334</span><span id="line-334"> assertTrue(storeFileInfo.isLink());</span> |
| <span class="source-line-no">335</span><span id="line-335"> hsf.initReader();</span> |
| <span class="source-line-no">336</span><span id="line-336"></span> |
| <span class="source-line-no">337</span><span id="line-337"> // Now confirm that I can read from the link</span> |
| <span class="source-line-no">338</span><span id="line-338"> int count = 0;</span> |
| <span class="source-line-no">339</span><span id="line-339"> try (StoreFileScanner scanner = hsf.getPreadScanner(false, Long.MAX_VALUE, 0, false)) {</span> |
| <span class="source-line-no">340</span><span id="line-340"> scanner.seek(KeyValue.LOWESTKEY);</span> |
| <span class="source-line-no">341</span><span id="line-341"> while (scanner.next() != null) {</span> |
| <span class="source-line-no">342</span><span id="line-342"> count++;</span> |
| <span class="source-line-no">343</span><span id="line-343"> }</span> |
| <span class="source-line-no">344</span><span id="line-344"> }</span> |
| <span class="source-line-no">345</span><span id="line-345"> assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);</span> |
| <span class="source-line-no">346</span><span id="line-346"> }</span> |
| <span class="source-line-no">347</span><span id="line-347"></span> |
| <span class="source-line-no">348</span><span id="line-348"> /**</span> |
| <span class="source-line-no">349</span><span id="line-349"> * This test creates an hfile and then the dir structures and files to verify that references to</span> |
| <span class="source-line-no">350</span><span id="line-350"> * hfilelinks (created by snapshot clones) can be properly interpreted.</span> |
| <span class="source-line-no">351</span><span id="line-351"> */</span> |
| <span class="source-line-no">352</span><span id="line-352"> @Test</span> |
| <span class="source-line-no">353</span><span id="line-353"> public void testReferenceToHFileLink() throws IOException {</span> |
| <span class="source-line-no">354</span><span id="line-354"> // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/</span> |
| <span class="source-line-no">355</span><span id="line-355"> Configuration testConf = new Configuration(this.conf);</span> |
| <span class="source-line-no">356</span><span id="line-356"> CommonFSUtils.setRootDir(testConf, testDir);</span> |
| <span class="source-line-no">357</span><span id="line-357"></span> |
| <span class="source-line-no">358</span><span id="line-358"> // adding legal table name chars to verify regex handles it.</span> |
| <span class="source-line-no">359</span><span id="line-359"> RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf("_original-evil-name")).build();</span> |
| <span class="source-line-no">360</span><span id="line-360"> HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,</span> |
| <span class="source-line-no">361</span><span id="line-361"> CommonFSUtils.getTableDir(testDir, hri.getTable()), hri);</span> |
| <span class="source-line-no">362</span><span id="line-362"></span> |
| <span class="source-line-no">363</span><span id="line-363"> HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();</span> |
| <span class="source-line-no">364</span><span id="line-364"> // Make a store file and write data to it. <root>/<tablename>/<rgn>/<cf>/<file></span> |
| <span class="source-line-no">365</span><span id="line-365"> StoreFileWriter writer = new StoreFileWriter.Builder(testConf, cacheConf, this.fs)</span> |
| <span class="source-line-no">366</span><span id="line-366"> .withFilePath(regionFs.createTempName()).withFileContext(meta).build();</span> |
| <span class="source-line-no">367</span><span id="line-367"> writeStoreFile(writer);</span> |
| <span class="source-line-no">368</span><span id="line-368"> Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());</span> |
| <span class="source-line-no">369</span><span id="line-369"></span> |
| <span class="source-line-no">370</span><span id="line-370"> // create link to store file. <root>/clone/region/<cf>/<hfile>-<region>-<table></span> |
| <span class="source-line-no">371</span><span id="line-371"> RegionInfo hriClone = RegionInfoBuilder.newBuilder(TableName.valueOf("clone")).build();</span> |
| <span class="source-line-no">372</span><span id="line-372"> HRegionFileSystem cloneRegionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,</span> |
| <span class="source-line-no">373</span><span id="line-373"> CommonFSUtils.getTableDir(testDir, hri.getTable()), hriClone);</span> |
| <span class="source-line-no">374</span><span id="line-374"> Path dstPath = cloneRegionFs.getStoreDir(TEST_FAMILY);</span> |
| <span class="source-line-no">375</span><span id="line-375"> HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());</span> |
| <span class="source-line-no">376</span><span id="line-376"> Path linkFilePath =</span> |
| <span class="source-line-no">377</span><span id="line-377"> new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));</span> |
| <span class="source-line-no">378</span><span id="line-378"></span> |
| <span class="source-line-no">379</span><span id="line-379"> // create splits of the link.</span> |
| <span class="source-line-no">380</span><span id="line-380"> // <root>/clone/splitA/<cf>/<reftohfilelink>,</span> |
| <span class="source-line-no">381</span><span id="line-381"> // <root>/clone/splitB/<cf>/<reftohfilelink></span> |
| <span class="source-line-no">382</span><span id="line-382"> RegionInfo splitHriA = RegionInfoBuilder.newBuilder(hri.getTable()).setEndKey(SPLITKEY).build();</span> |
| <span class="source-line-no">383</span><span id="line-383"> RegionInfo splitHriB =</span> |
| <span class="source-line-no">384</span><span id="line-384"> RegionInfoBuilder.newBuilder(hri.getTable()).setStartKey(SPLITKEY).build();</span> |
| <span class="source-line-no">385</span><span id="line-385"> HStoreFile f = new HStoreFile(fs, linkFilePath, testConf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">386</span><span id="line-386"> f.initReader();</span> |
| <span class="source-line-no">387</span><span id="line-387"> Path pathA = splitStoreFile(cloneRegionFs, splitHriA, TEST_FAMILY, f, SPLITKEY, true); // top</span> |
| <span class="source-line-no">388</span><span id="line-388"> Path pathB = splitStoreFile(cloneRegionFs, splitHriB, TEST_FAMILY, f, SPLITKEY, false);// bottom</span> |
| <span class="source-line-no">389</span><span id="line-389"> f.closeStoreFile(true);</span> |
| <span class="source-line-no">390</span><span id="line-390"> // OK test the thing</span> |
| <span class="source-line-no">391</span><span id="line-391"> CommonFSUtils.logFileSystemState(fs, testDir, LOG);</span> |
| <span class="source-line-no">392</span><span id="line-392"></span> |
| <span class="source-line-no">393</span><span id="line-393"> // There is a case where a file with the hfilelink pattern is actually a daughter</span> |
| <span class="source-line-no">394</span><span id="line-394"> // reference to a hfile link. This code in StoreFile that handles this case.</span> |
| <span class="source-line-no">395</span><span id="line-395"></span> |
| <span class="source-line-no">396</span><span id="line-396"> // Try to open store file from link</span> |
| <span class="source-line-no">397</span><span id="line-397"> HStoreFile hsfA = new HStoreFile(this.fs, pathA, testConf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">398</span><span id="line-398"> hsfA.initReader();</span> |
| <span class="source-line-no">399</span><span id="line-399"></span> |
| <span class="source-line-no">400</span><span id="line-400"> // Now confirm that I can read from the ref to link</span> |
| <span class="source-line-no">401</span><span id="line-401"> int count = 0;</span> |
| <span class="source-line-no">402</span><span id="line-402"> try (StoreFileScanner scanner = hsfA.getPreadScanner(false, Long.MAX_VALUE, 0, false)) {</span> |
| <span class="source-line-no">403</span><span id="line-403"> scanner.seek(KeyValue.LOWESTKEY);</span> |
| <span class="source-line-no">404</span><span id="line-404"> while (scanner.next() != null) {</span> |
| <span class="source-line-no">405</span><span id="line-405"> count++;</span> |
| <span class="source-line-no">406</span><span id="line-406"> }</span> |
| <span class="source-line-no">407</span><span id="line-407"> assertTrue(count > 0); // read some rows here</span> |
| <span class="source-line-no">408</span><span id="line-408"> }</span> |
| <span class="source-line-no">409</span><span id="line-409"></span> |
| <span class="source-line-no">410</span><span id="line-410"> // Try to open store file from link</span> |
| <span class="source-line-no">411</span><span id="line-411"> HStoreFile hsfB = new HStoreFile(this.fs, pathB, testConf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">412</span><span id="line-412"> hsfB.initReader();</span> |
| <span class="source-line-no">413</span><span id="line-413"></span> |
| <span class="source-line-no">414</span><span id="line-414"> // Now confirm that I can read from the ref to link</span> |
| <span class="source-line-no">415</span><span id="line-415"> try (StoreFileScanner scanner = hsfB.getPreadScanner(false, Long.MAX_VALUE, 0, false)) {</span> |
| <span class="source-line-no">416</span><span id="line-416"> scanner.seek(KeyValue.LOWESTKEY);</span> |
| <span class="source-line-no">417</span><span id="line-417"> while (scanner.next() != null) {</span> |
| <span class="source-line-no">418</span><span id="line-418"> count++;</span> |
| <span class="source-line-no">419</span><span id="line-419"> }</span> |
| <span class="source-line-no">420</span><span id="line-420"> }</span> |
| <span class="source-line-no">421</span><span id="line-421"></span> |
| <span class="source-line-no">422</span><span id="line-422"> // read the rest of the rows</span> |
| <span class="source-line-no">423</span><span id="line-423"> assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);</span> |
| <span class="source-line-no">424</span><span id="line-424"> }</span> |
| <span class="source-line-no">425</span><span id="line-425"></span> |
| <span class="source-line-no">426</span><span id="line-426"> private void checkHalfHFile(final HRegionFileSystem regionFs, final HStoreFile f)</span> |
| <span class="source-line-no">427</span><span id="line-427"> throws IOException {</span> |
| <span class="source-line-no">428</span><span id="line-428"> f.initReader();</span> |
| <span class="source-line-no">429</span><span id="line-429"> Cell midkey = f.getReader().midKey().get();</span> |
| <span class="source-line-no">430</span><span id="line-430"> KeyValue midKV = (KeyValue) midkey;</span> |
| <span class="source-line-no">431</span><span id="line-431"> // 1. test using the midRow as the splitKey, this test will generate two Reference files</span> |
| <span class="source-line-no">432</span><span id="line-432"> // in the children</span> |
| <span class="source-line-no">433</span><span id="line-433"> byte[] midRow = CellUtil.cloneRow(midKV);</span> |
| <span class="source-line-no">434</span><span id="line-434"> // Create top split.</span> |
| <span class="source-line-no">435</span><span id="line-435"> RegionInfo topHri =</span> |
| <span class="source-line-no">436</span><span id="line-436"> RegionInfoBuilder.newBuilder(regionFs.getRegionInfo().getTable()).setEndKey(SPLITKEY).build();</span> |
| <span class="source-line-no">437</span><span id="line-437"> Path topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, midRow, true);</span> |
| <span class="source-line-no">438</span><span id="line-438"> // Create bottom split.</span> |
| <span class="source-line-no">439</span><span id="line-439"> RegionInfo bottomHri = RegionInfoBuilder.newBuilder(regionFs.getRegionInfo().getTable())</span> |
| <span class="source-line-no">440</span><span id="line-440"> .setStartKey(SPLITKEY).build();</span> |
| <span class="source-line-no">441</span><span id="line-441"> Path bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, midRow, false);</span> |
| <span class="source-line-no">442</span><span id="line-442"> // Make readers on top and bottom.</span> |
| <span class="source-line-no">443</span><span id="line-443"> HStoreFile topF = new HStoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">444</span><span id="line-444"> topF.initReader();</span> |
| <span class="source-line-no">445</span><span id="line-445"> StoreFileReader top = topF.getReader();</span> |
| <span class="source-line-no">446</span><span id="line-446"> HStoreFile bottomF = new HStoreFile(this.fs, bottomPath, conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">447</span><span id="line-447"> bottomF.initReader();</span> |
| <span class="source-line-no">448</span><span id="line-448"> StoreFileReader bottom = bottomF.getReader();</span> |
| <span class="source-line-no">449</span><span id="line-449"> ByteBuffer previous = null;</span> |
| <span class="source-line-no">450</span><span id="line-450"> LOG.info("Midkey: " + midKV.toString());</span> |
| <span class="source-line-no">451</span><span id="line-451"> ByteBuffer bbMidkeyBytes = ByteBuffer.wrap(midKV.getKey());</span> |
| <span class="source-line-no">452</span><span id="line-452"> try {</span> |
| <span class="source-line-no">453</span><span id="line-453"> // Now make two HalfMapFiles and assert they can read the full backing</span> |
| <span class="source-line-no">454</span><span id="line-454"> // file, one from the top and the other from the bottom.</span> |
| <span class="source-line-no">455</span><span id="line-455"> // Test bottom half first.</span> |
| <span class="source-line-no">456</span><span id="line-456"> // Now test reading from the top.</span> |
| <span class="source-line-no">457</span><span id="line-457"> boolean first = true;</span> |
| <span class="source-line-no">458</span><span id="line-458"> ByteBuffer key = null;</span> |
| <span class="source-line-no">459</span><span id="line-459"> try (HFileScanner topScanner = top.getScanner(false, false, false)) {</span> |
| <span class="source-line-no">460</span><span id="line-460"> while (</span> |
| <span class="source-line-no">461</span><span id="line-461"> (!topScanner.isSeeked() && topScanner.seekTo())</span> |
| <span class="source-line-no">462</span><span id="line-462"> || (topScanner.isSeeked() && topScanner.next())</span> |
| <span class="source-line-no">463</span><span id="line-463"> ) {</span> |
| <span class="source-line-no">464</span><span id="line-464"> key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey());</span> |
| <span class="source-line-no">465</span><span id="line-465"></span> |
| <span class="source-line-no">466</span><span id="line-466"> if (</span> |
| <span class="source-line-no">467</span><span id="line-467"> (PrivateCellUtil.compare(topScanner.getReader().getComparator(), midKV, key.array(),</span> |
| <span class="source-line-no">468</span><span id="line-468"> key.arrayOffset(), key.limit())) > 0</span> |
| <span class="source-line-no">469</span><span id="line-469"> ) {</span> |
| <span class="source-line-no">470</span><span id="line-470"> fail("key=" + Bytes.toStringBinary(key) + " < midkey=" + midkey);</span> |
| <span class="source-line-no">471</span><span id="line-471"> }</span> |
| <span class="source-line-no">472</span><span id="line-472"> if (first) {</span> |
| <span class="source-line-no">473</span><span id="line-473"> first = false;</span> |
| <span class="source-line-no">474</span><span id="line-474"> LOG.info("First in top: " + Bytes.toString(Bytes.toBytes(key)));</span> |
| <span class="source-line-no">475</span><span id="line-475"> }</span> |
| <span class="source-line-no">476</span><span id="line-476"> }</span> |
| <span class="source-line-no">477</span><span id="line-477"> }</span> |
| <span class="source-line-no">478</span><span id="line-478"> LOG.info("Last in top: " + Bytes.toString(Bytes.toBytes(key)));</span> |
| <span class="source-line-no">479</span><span id="line-479"></span> |
| <span class="source-line-no">480</span><span id="line-480"> first = true;</span> |
| <span class="source-line-no">481</span><span id="line-481"> try (HFileScanner bottomScanner = bottom.getScanner(false, false, false)) {</span> |
| <span class="source-line-no">482</span><span id="line-482"> while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || bottomScanner.next()) {</span> |
| <span class="source-line-no">483</span><span id="line-483"> previous = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());</span> |
| <span class="source-line-no">484</span><span id="line-484"> key = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());</span> |
| <span class="source-line-no">485</span><span id="line-485"> if (first) {</span> |
| <span class="source-line-no">486</span><span id="line-486"> first = false;</span> |
| <span class="source-line-no">487</span><span id="line-487"> LOG.info("First in bottom: " + Bytes.toString(Bytes.toBytes(previous)));</span> |
| <span class="source-line-no">488</span><span id="line-488"> }</span> |
| <span class="source-line-no">489</span><span id="line-489"> assertTrue(key.compareTo(bbMidkeyBytes) < 0);</span> |
| <span class="source-line-no">490</span><span id="line-490"> }</span> |
| <span class="source-line-no">491</span><span id="line-491"> if (previous != null) {</span> |
| <span class="source-line-no">492</span><span id="line-492"> LOG.info("Last in bottom: " + Bytes.toString(Bytes.toBytes(previous)));</span> |
| <span class="source-line-no">493</span><span id="line-493"> }</span> |
| <span class="source-line-no">494</span><span id="line-494"> }</span> |
| <span class="source-line-no">495</span><span id="line-495"> // Remove references.</span> |
| <span class="source-line-no">496</span><span id="line-496"> regionFs.cleanupDaughterRegion(topHri);</span> |
| <span class="source-line-no">497</span><span id="line-497"> regionFs.cleanupDaughterRegion(bottomHri);</span> |
| <span class="source-line-no">498</span><span id="line-498"></span> |
| <span class="source-line-no">499</span><span id="line-499"> // 2. test using a midkey which will generate one Reference file and one HFileLink file.</span> |
| <span class="source-line-no">500</span><span id="line-500"> // First, do a key that is < than first key. Ensure splits behave</span> |
| <span class="source-line-no">501</span><span id="line-501"> // properly.</span> |
| <span class="source-line-no">502</span><span id="line-502"> byte[] badmidkey = Bytes.toBytes(" .");</span> |
| <span class="source-line-no">503</span><span id="line-503"> assertTrue(fs.exists(f.getPath()));</span> |
| <span class="source-line-no">504</span><span id="line-504"> topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, badmidkey, true);</span> |
| <span class="source-line-no">505</span><span id="line-505"> bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, badmidkey, false);</span> |
| <span class="source-line-no">506</span><span id="line-506"></span> |
| <span class="source-line-no">507</span><span id="line-507"> assertNull(bottomPath);</span> |
| <span class="source-line-no">508</span><span id="line-508"></span> |
| <span class="source-line-no">509</span><span id="line-509"> topF = new HStoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">510</span><span id="line-510"> topF.initReader();</span> |
| <span class="source-line-no">511</span><span id="line-511"> top = topF.getReader();</span> |
| <span class="source-line-no">512</span><span id="line-512"> // Now read from the top.</span> |
| <span class="source-line-no">513</span><span id="line-513"> first = true;</span> |
| <span class="source-line-no">514</span><span id="line-514"> try (HFileScanner topScanner = top.getScanner(false, false, false)) {</span> |
| <span class="source-line-no">515</span><span id="line-515"> KeyValue.KeyOnlyKeyValue keyOnlyKV = new KeyValue.KeyOnlyKeyValue();</span> |
| <span class="source-line-no">516</span><span id="line-516"> while ((!topScanner.isSeeked() && topScanner.seekTo()) || topScanner.next()) {</span> |
| <span class="source-line-no">517</span><span id="line-517"> key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey());</span> |
| <span class="source-line-no">518</span><span id="line-518"> keyOnlyKV.setKey(key.array(), 0 + key.arrayOffset(), key.limit());</span> |
| <span class="source-line-no">519</span><span id="line-519"> assertTrue(PrivateCellUtil.compare(topScanner.getReader().getComparator(), keyOnlyKV,</span> |
| <span class="source-line-no">520</span><span id="line-520"> badmidkey, 0, badmidkey.length) >= 0);</span> |
| <span class="source-line-no">521</span><span id="line-521"> if (first) {</span> |
| <span class="source-line-no">522</span><span id="line-522"> first = false;</span> |
| <span class="source-line-no">523</span><span id="line-523"> KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);</span> |
| <span class="source-line-no">524</span><span id="line-524"> LOG.info("First top when key < bottom: " + keyKV);</span> |
| <span class="source-line-no">525</span><span id="line-525"> String tmp =</span> |
| <span class="source-line-no">526</span><span id="line-526"> Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());</span> |
| <span class="source-line-no">527</span><span id="line-527"> for (int i = 0; i < tmp.length(); i++) {</span> |
| <span class="source-line-no">528</span><span id="line-528"> assertTrue(tmp.charAt(i) == 'a');</span> |
| <span class="source-line-no">529</span><span id="line-529"> }</span> |
| <span class="source-line-no">530</span><span id="line-530"> }</span> |
| <span class="source-line-no">531</span><span id="line-531"> }</span> |
| <span class="source-line-no">532</span><span id="line-532"> KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);</span> |
| <span class="source-line-no">533</span><span id="line-533"> LOG.info("Last top when key < bottom: " + keyKV);</span> |
| <span class="source-line-no">534</span><span id="line-534"> String tmp =</span> |
| <span class="source-line-no">535</span><span id="line-535"> Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());</span> |
| <span class="source-line-no">536</span><span id="line-536"> for (int i = 0; i < tmp.length(); i++) {</span> |
| <span class="source-line-no">537</span><span id="line-537"> assertTrue(tmp.charAt(i) == 'z');</span> |
| <span class="source-line-no">538</span><span id="line-538"> }</span> |
| <span class="source-line-no">539</span><span id="line-539"> }</span> |
| <span class="source-line-no">540</span><span id="line-540"> // Remove references.</span> |
| <span class="source-line-no">541</span><span id="line-541"> regionFs.cleanupDaughterRegion(topHri);</span> |
| <span class="source-line-no">542</span><span id="line-542"> regionFs.cleanupDaughterRegion(bottomHri);</span> |
| <span class="source-line-no">543</span><span id="line-543"></span> |
| <span class="source-line-no">544</span><span id="line-544"> // Test when badkey is > than last key in file ('||' > 'zz').</span> |
| <span class="source-line-no">545</span><span id="line-545"> badmidkey = Bytes.toBytes("|||");</span> |
| <span class="source-line-no">546</span><span id="line-546"> topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, badmidkey, true);</span> |
| <span class="source-line-no">547</span><span id="line-547"> bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, badmidkey, false);</span> |
| <span class="source-line-no">548</span><span id="line-548"> assertNull(topPath);</span> |
| <span class="source-line-no">549</span><span id="line-549"></span> |
| <span class="source-line-no">550</span><span id="line-550"> bottomF = new HStoreFile(this.fs, bottomPath, conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">551</span><span id="line-551"> bottomF.initReader();</span> |
| <span class="source-line-no">552</span><span id="line-552"> bottom = bottomF.getReader();</span> |
| <span class="source-line-no">553</span><span id="line-553"> first = true;</span> |
| <span class="source-line-no">554</span><span id="line-554"> try (HFileScanner bottomScanner = bottom.getScanner(false, false, false)) {</span> |
| <span class="source-line-no">555</span><span id="line-555"> while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || bottomScanner.next()) {</span> |
| <span class="source-line-no">556</span><span id="line-556"> key = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());</span> |
| <span class="source-line-no">557</span><span id="line-557"> if (first) {</span> |
| <span class="source-line-no">558</span><span id="line-558"> first = false;</span> |
| <span class="source-line-no">559</span><span id="line-559"> KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);</span> |
| <span class="source-line-no">560</span><span id="line-560"> LOG.info("First bottom when key > top: " + keyKV);</span> |
| <span class="source-line-no">561</span><span id="line-561"> String tmp =</span> |
| <span class="source-line-no">562</span><span id="line-562"> Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());</span> |
| <span class="source-line-no">563</span><span id="line-563"> for (int i = 0; i < tmp.length(); i++) {</span> |
| <span class="source-line-no">564</span><span id="line-564"> assertTrue(tmp.charAt(i) == 'a');</span> |
| <span class="source-line-no">565</span><span id="line-565"> }</span> |
| <span class="source-line-no">566</span><span id="line-566"> }</span> |
| <span class="source-line-no">567</span><span id="line-567"> }</span> |
| <span class="source-line-no">568</span><span id="line-568"> KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);</span> |
| <span class="source-line-no">569</span><span id="line-569"> LOG.info("Last bottom when key > top: " + keyKV);</span> |
| <span class="source-line-no">570</span><span id="line-570"> String tmp =</span> |
| <span class="source-line-no">571</span><span id="line-571"> Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());</span> |
| <span class="source-line-no">572</span><span id="line-572"> for (int i = 0; i < tmp.length(); i++) {</span> |
| <span class="source-line-no">573</span><span id="line-573"> assertTrue(Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength())</span> |
| <span class="source-line-no">574</span><span id="line-574"> .charAt(i) == 'z');</span> |
| <span class="source-line-no">575</span><span id="line-575"> }</span> |
| <span class="source-line-no">576</span><span id="line-576"> }</span> |
| <span class="source-line-no">577</span><span id="line-577"> } finally {</span> |
| <span class="source-line-no">578</span><span id="line-578"> if (top != null) {</span> |
| <span class="source-line-no">579</span><span id="line-579"> top.close(true); // evict since we are about to delete the file</span> |
| <span class="source-line-no">580</span><span id="line-580"> }</span> |
| <span class="source-line-no">581</span><span id="line-581"> if (bottom != null) {</span> |
| <span class="source-line-no">582</span><span id="line-582"> bottom.close(true); // evict since we are about to delete the file</span> |
| <span class="source-line-no">583</span><span id="line-583"> }</span> |
| <span class="source-line-no">584</span><span id="line-584"> fs.delete(f.getPath(), true);</span> |
| <span class="source-line-no">585</span><span id="line-585"> }</span> |
| <span class="source-line-no">586</span><span id="line-586"> }</span> |
| <span class="source-line-no">587</span><span id="line-587"></span> |
| <span class="source-line-no">588</span><span id="line-588"> private static StoreFileScanner getStoreFileScanner(StoreFileReader reader, boolean cacheBlocks,</span> |
| <span class="source-line-no">589</span><span id="line-589"> boolean pread) {</span> |
| <span class="source-line-no">590</span><span id="line-590"> return reader.getStoreFileScanner(cacheBlocks, pread, false, 0, 0, false);</span> |
| <span class="source-line-no">591</span><span id="line-591"> }</span> |
| <span class="source-line-no">592</span><span id="line-592"></span> |
| <span class="source-line-no">593</span><span id="line-593"> private static final String localFormatter = "%010d";</span> |
| <span class="source-line-no">594</span><span id="line-594"></span> |
| <span class="source-line-no">595</span><span id="line-595"> private void bloomWriteRead(StoreFileWriter writer, FileSystem fs) throws Exception {</span> |
| <span class="source-line-no">596</span><span id="line-596"> float err = conf.getFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, 0);</span> |
| <span class="source-line-no">597</span><span id="line-597"> Path f = writer.getPath();</span> |
| <span class="source-line-no">598</span><span id="line-598"> long now = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">599</span><span id="line-599"> for (int i = 0; i < 2000; i += 2) {</span> |
| <span class="source-line-no">600</span><span id="line-600"> String row = String.format(localFormatter, i);</span> |
| <span class="source-line-no">601</span><span id="line-601"> KeyValue kv = new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("col"),</span> |
| <span class="source-line-no">602</span><span id="line-602"> now, Bytes.toBytes("value"));</span> |
| <span class="source-line-no">603</span><span id="line-603"> writer.append(kv);</span> |
| <span class="source-line-no">604</span><span id="line-604"> }</span> |
| <span class="source-line-no">605</span><span id="line-605"> writer.close();</span> |
| <span class="source-line-no">606</span><span id="line-606"></span> |
| <span class="source-line-no">607</span><span id="line-607"> ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build();</span> |
| <span class="source-line-no">608</span><span id="line-608"> StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, f, true);</span> |
| <span class="source-line-no">609</span><span id="line-609"> storeFileInfo.initHFileInfo(context);</span> |
| <span class="source-line-no">610</span><span id="line-610"> StoreFileReader reader = storeFileInfo.createReader(context, cacheConf);</span> |
| <span class="source-line-no">611</span><span id="line-611"> storeFileInfo.getHFileInfo().initMetaAndIndex(reader.getHFileReader());</span> |
| <span class="source-line-no">612</span><span id="line-612"> reader.loadFileInfo();</span> |
| <span class="source-line-no">613</span><span id="line-613"> reader.loadBloomfilter();</span> |
| <span class="source-line-no">614</span><span id="line-614"> StoreFileScanner scanner = getStoreFileScanner(reader, false, false);</span> |
| <span class="source-line-no">615</span><span id="line-615"></span> |
| <span class="source-line-no">616</span><span id="line-616"> // check false positives rate</span> |
| <span class="source-line-no">617</span><span id="line-617"> int falsePos = 0;</span> |
| <span class="source-line-no">618</span><span id="line-618"> int falseNeg = 0;</span> |
| <span class="source-line-no">619</span><span id="line-619"> for (int i = 0; i < 2000; i++) {</span> |
| <span class="source-line-no">620</span><span id="line-620"> String row = String.format(localFormatter, i);</span> |
| <span class="source-line-no">621</span><span id="line-621"> TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">622</span><span id="line-622"> columns.add(Bytes.toBytes("family:col"));</span> |
| <span class="source-line-no">623</span><span id="line-623"></span> |
| <span class="source-line-no">624</span><span id="line-624"> Scan scan = new Scan().withStartRow(Bytes.toBytes(row)).withStopRow(Bytes.toBytes(row), true);</span> |
| <span class="source-line-no">625</span><span id="line-625"> scan.addColumn(Bytes.toBytes("family"), Bytes.toBytes("family:col"));</span> |
| <span class="source-line-no">626</span><span id="line-626"> HStore store = mock(HStore.class);</span> |
| <span class="source-line-no">627</span><span id="line-627"> when(store.getColumnFamilyDescriptor())</span> |
| <span class="source-line-no">628</span><span id="line-628"> .thenReturn(ColumnFamilyDescriptorBuilder.of("family"));</span> |
| <span class="source-line-no">629</span><span id="line-629"> boolean exists = scanner.shouldUseScanner(scan, store, Long.MIN_VALUE);</span> |
| <span class="source-line-no">630</span><span id="line-630"> if (i % 2 == 0) {</span> |
| <span class="source-line-no">631</span><span id="line-631"> if (!exists) {</span> |
| <span class="source-line-no">632</span><span id="line-632"> falseNeg++;</span> |
| <span class="source-line-no">633</span><span id="line-633"> }</span> |
| <span class="source-line-no">634</span><span id="line-634"> } else {</span> |
| <span class="source-line-no">635</span><span id="line-635"> if (exists) {</span> |
| <span class="source-line-no">636</span><span id="line-636"> falsePos++;</span> |
| <span class="source-line-no">637</span><span id="line-637"> }</span> |
| <span class="source-line-no">638</span><span id="line-638"> }</span> |
| <span class="source-line-no">639</span><span id="line-639"> }</span> |
| <span class="source-line-no">640</span><span id="line-640"> reader.close(true); // evict because we are about to delete the file</span> |
| <span class="source-line-no">641</span><span id="line-641"> fs.delete(f, true);</span> |
| <span class="source-line-no">642</span><span id="line-642"> assertEquals("False negatives: " + falseNeg, 0, falseNeg);</span> |
| <span class="source-line-no">643</span><span id="line-643"> int maxFalsePos = (int) (2 * 2000 * err);</span> |
| <span class="source-line-no">644</span><span id="line-644"> assertTrue("Too many false positives: " + falsePos + " (err=" + err + ", expected no more than "</span> |
| <span class="source-line-no">645</span><span id="line-645"> + maxFalsePos + ")", falsePos <= maxFalsePos);</span> |
| <span class="source-line-no">646</span><span id="line-646"> }</span> |
| <span class="source-line-no">647</span><span id="line-647"></span> |
| <span class="source-line-no">648</span><span id="line-648"> private static final int BLOCKSIZE_SMALL = 8192;</span> |
| <span class="source-line-no">649</span><span id="line-649"></span> |
| <span class="source-line-no">650</span><span id="line-650"> @Test</span> |
| <span class="source-line-no">651</span><span id="line-651"> public void testBloomFilter() throws Exception {</span> |
| <span class="source-line-no">652</span><span id="line-652"> conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01);</span> |
| <span class="source-line-no">653</span><span id="line-653"> conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);</span> |
| <span class="source-line-no">654</span><span id="line-654"></span> |
| <span class="source-line-no">655</span><span id="line-655"> // write the file</span> |
| <span class="source-line-no">656</span><span id="line-656"> if (!fs.exists(ROOT_DIR)) {</span> |
| <span class="source-line-no">657</span><span id="line-657"> fs.mkdirs(ROOT_DIR);</span> |
| <span class="source-line-no">658</span><span id="line-658"> }</span> |
| <span class="source-line-no">659</span><span id="line-659"> Path f = StoreFileWriter.getUniqueFile(fs, ROOT_DIR);</span> |
| <span class="source-line-no">660</span><span id="line-660"> HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL)</span> |
| <span class="source-line-no">661</span><span id="line-661"> .withChecksumType(CKTYPE).withBytesPerCheckSum(CKBYTES).build();</span> |
| <span class="source-line-no">662</span><span id="line-662"> // Make a store file and write data to it.</span> |
| <span class="source-line-no">663</span><span id="line-663"> StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f)</span> |
| <span class="source-line-no">664</span><span id="line-664"> .withBloomType(BloomType.ROW).withMaxKeyCount(2000).withFileContext(meta).build();</span> |
| <span class="source-line-no">665</span><span id="line-665"> bloomWriteRead(writer, fs);</span> |
| <span class="source-line-no">666</span><span id="line-666"> }</span> |
| <span class="source-line-no">667</span><span id="line-667"></span> |
| <span class="source-line-no">668</span><span id="line-668"> @Test</span> |
| <span class="source-line-no">669</span><span id="line-669"> public void testDeleteFamilyBloomFilter() throws Exception {</span> |
| <span class="source-line-no">670</span><span id="line-670"> conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01);</span> |
| <span class="source-line-no">671</span><span id="line-671"> conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);</span> |
| <span class="source-line-no">672</span><span id="line-672"> float err = conf.getFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, 0);</span> |
| <span class="source-line-no">673</span><span id="line-673"></span> |
| <span class="source-line-no">674</span><span id="line-674"> // write the file</span> |
| <span class="source-line-no">675</span><span id="line-675"> if (!fs.exists(ROOT_DIR)) {</span> |
| <span class="source-line-no">676</span><span id="line-676"> fs.mkdirs(ROOT_DIR);</span> |
| <span class="source-line-no">677</span><span id="line-677"> }</span> |
| <span class="source-line-no">678</span><span id="line-678"> Path f = StoreFileWriter.getUniqueFile(fs, ROOT_DIR);</span> |
| <span class="source-line-no">679</span><span id="line-679"></span> |
| <span class="source-line-no">680</span><span id="line-680"> HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL)</span> |
| <span class="source-line-no">681</span><span id="line-681"> .withChecksumType(CKTYPE).withBytesPerCheckSum(CKBYTES).build();</span> |
| <span class="source-line-no">682</span><span id="line-682"> // Make a store file and write data to it.</span> |
| <span class="source-line-no">683</span><span id="line-683"> StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f)</span> |
| <span class="source-line-no">684</span><span id="line-684"> .withMaxKeyCount(2000).withFileContext(meta).build();</span> |
| <span class="source-line-no">685</span><span id="line-685"></span> |
| <span class="source-line-no">686</span><span id="line-686"> // add delete family</span> |
| <span class="source-line-no">687</span><span id="line-687"> long now = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">688</span><span id="line-688"> for (int i = 0; i < 2000; i += 2) {</span> |
| <span class="source-line-no">689</span><span id="line-689"> String row = String.format(localFormatter, i);</span> |
| <span class="source-line-no">690</span><span id="line-690"> KeyValue kv = new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("col"),</span> |
| <span class="source-line-no">691</span><span id="line-691"> now, KeyValue.Type.DeleteFamily, Bytes.toBytes("value"));</span> |
| <span class="source-line-no">692</span><span id="line-692"> writer.append(kv);</span> |
| <span class="source-line-no">693</span><span id="line-693"> }</span> |
| <span class="source-line-no">694</span><span id="line-694"> writer.close();</span> |
| <span class="source-line-no">695</span><span id="line-695"></span> |
| <span class="source-line-no">696</span><span id="line-696"> ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build();</span> |
| <span class="source-line-no">697</span><span id="line-697"> StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, f, true);</span> |
| <span class="source-line-no">698</span><span id="line-698"> storeFileInfo.initHFileInfo(context);</span> |
| <span class="source-line-no">699</span><span id="line-699"> StoreFileReader reader = storeFileInfo.createReader(context, cacheConf);</span> |
| <span class="source-line-no">700</span><span id="line-700"> storeFileInfo.getHFileInfo().initMetaAndIndex(reader.getHFileReader());</span> |
| <span class="source-line-no">701</span><span id="line-701"> reader.loadFileInfo();</span> |
| <span class="source-line-no">702</span><span id="line-702"> reader.loadBloomfilter();</span> |
| <span class="source-line-no">703</span><span id="line-703"></span> |
| <span class="source-line-no">704</span><span id="line-704"> // check false positives rate</span> |
| <span class="source-line-no">705</span><span id="line-705"> int falsePos = 0;</span> |
| <span class="source-line-no">706</span><span id="line-706"> int falseNeg = 0;</span> |
| <span class="source-line-no">707</span><span id="line-707"> for (int i = 0; i < 2000; i++) {</span> |
| <span class="source-line-no">708</span><span id="line-708"> String row = String.format(localFormatter, i);</span> |
| <span class="source-line-no">709</span><span id="line-709"> byte[] rowKey = Bytes.toBytes(row);</span> |
| <span class="source-line-no">710</span><span id="line-710"> boolean exists = reader.passesDeleteFamilyBloomFilter(rowKey, 0, rowKey.length);</span> |
| <span class="source-line-no">711</span><span id="line-711"> if (i % 2 == 0) {</span> |
| <span class="source-line-no">712</span><span id="line-712"> if (!exists) {</span> |
| <span class="source-line-no">713</span><span id="line-713"> falseNeg++;</span> |
| <span class="source-line-no">714</span><span id="line-714"> }</span> |
| <span class="source-line-no">715</span><span id="line-715"> } else {</span> |
| <span class="source-line-no">716</span><span id="line-716"> if (exists) {</span> |
| <span class="source-line-no">717</span><span id="line-717"> falsePos++;</span> |
| <span class="source-line-no">718</span><span id="line-718"> }</span> |
| <span class="source-line-no">719</span><span id="line-719"> }</span> |
| <span class="source-line-no">720</span><span id="line-720"> }</span> |
| <span class="source-line-no">721</span><span id="line-721"> assertEquals(1000, reader.getDeleteFamilyCnt());</span> |
| <span class="source-line-no">722</span><span id="line-722"> reader.close(true); // evict because we are about to delete the file</span> |
| <span class="source-line-no">723</span><span id="line-723"> fs.delete(f, true);</span> |
| <span class="source-line-no">724</span><span id="line-724"> assertEquals("False negatives: " + falseNeg, 0, falseNeg);</span> |
| <span class="source-line-no">725</span><span id="line-725"> int maxFalsePos = (int) (2 * 2000 * err);</span> |
| <span class="source-line-no">726</span><span id="line-726"> assertTrue("Too many false positives: " + falsePos + " (err=" + err + ", expected no more than "</span> |
| <span class="source-line-no">727</span><span id="line-727"> + maxFalsePos, falsePos <= maxFalsePos);</span> |
| <span class="source-line-no">728</span><span id="line-728"> }</span> |
| <span class="source-line-no">729</span><span id="line-729"></span> |
| <span class="source-line-no">730</span><span id="line-730"> /**</span> |
| <span class="source-line-no">731</span><span id="line-731"> * Test for HBASE-8012</span> |
| <span class="source-line-no">732</span><span id="line-732"> */</span> |
| <span class="source-line-no">733</span><span id="line-733"> @Test</span> |
| <span class="source-line-no">734</span><span id="line-734"> public void testReseek() throws Exception {</span> |
| <span class="source-line-no">735</span><span id="line-735"> // write the file</span> |
| <span class="source-line-no">736</span><span id="line-736"> if (!fs.exists(ROOT_DIR)) {</span> |
| <span class="source-line-no">737</span><span id="line-737"> fs.mkdirs(ROOT_DIR);</span> |
| <span class="source-line-no">738</span><span id="line-738"> }</span> |
| <span class="source-line-no">739</span><span id="line-739"> Path f = StoreFileWriter.getUniqueFile(fs, ROOT_DIR);</span> |
| <span class="source-line-no">740</span><span id="line-740"></span> |
| <span class="source-line-no">741</span><span id="line-741"> HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();</span> |
| <span class="source-line-no">742</span><span id="line-742"> // Make a store file and write data to it.</span> |
| <span class="source-line-no">743</span><span id="line-743"> StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f)</span> |
| <span class="source-line-no">744</span><span id="line-744"> .withFileContext(meta).build();</span> |
| <span class="source-line-no">745</span><span id="line-745"></span> |
| <span class="source-line-no">746</span><span id="line-746"> writeStoreFile(writer);</span> |
| <span class="source-line-no">747</span><span id="line-747"> writer.close();</span> |
| <span class="source-line-no">748</span><span id="line-748"></span> |
| <span class="source-line-no">749</span><span id="line-749"> ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build();</span> |
| <span class="source-line-no">750</span><span id="line-750"> StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, f, true);</span> |
| <span class="source-line-no">751</span><span id="line-751"> storeFileInfo.initHFileInfo(context);</span> |
| <span class="source-line-no">752</span><span id="line-752"> StoreFileReader reader = storeFileInfo.createReader(context, cacheConf);</span> |
| <span class="source-line-no">753</span><span id="line-753"> storeFileInfo.getHFileInfo().initMetaAndIndex(reader.getHFileReader());</span> |
| <span class="source-line-no">754</span><span id="line-754"></span> |
| <span class="source-line-no">755</span><span id="line-755"> // Now do reseek with empty KV to position to the beginning of the file</span> |
| <span class="source-line-no">756</span><span id="line-756"></span> |
| <span class="source-line-no">757</span><span id="line-757"> KeyValue k = KeyValueUtil.createFirstOnRow(HConstants.EMPTY_BYTE_ARRAY);</span> |
| <span class="source-line-no">758</span><span id="line-758"> StoreFileScanner s = getStoreFileScanner(reader, false, false);</span> |
| <span class="source-line-no">759</span><span id="line-759"> s.reseek(k);</span> |
| <span class="source-line-no">760</span><span id="line-760"></span> |
| <span class="source-line-no">761</span><span id="line-761"> assertNotNull("Intial reseek should position at the beginning of the file", s.peek());</span> |
| <span class="source-line-no">762</span><span id="line-762"> }</span> |
| <span class="source-line-no">763</span><span id="line-763"></span> |
| <span class="source-line-no">764</span><span id="line-764"> @Test</span> |
| <span class="source-line-no">765</span><span id="line-765"> public void testBloomTypes() throws Exception {</span> |
| <span class="source-line-no">766</span><span id="line-766"> float err = (float) 0.01;</span> |
| <span class="source-line-no">767</span><span id="line-767"> FileSystem fs = FileSystem.getLocal(conf);</span> |
| <span class="source-line-no">768</span><span id="line-768"> conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, err);</span> |
| <span class="source-line-no">769</span><span id="line-769"> conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);</span> |
| <span class="source-line-no">770</span><span id="line-770"></span> |
| <span class="source-line-no">771</span><span id="line-771"> int rowCount = 50;</span> |
| <span class="source-line-no">772</span><span id="line-772"> int colCount = 10;</span> |
| <span class="source-line-no">773</span><span id="line-773"> int versions = 2;</span> |
| <span class="source-line-no">774</span><span id="line-774"></span> |
| <span class="source-line-no">775</span><span id="line-775"> // run once using columns and once using rows</span> |
| <span class="source-line-no">776</span><span id="line-776"> BloomType[] bt = { BloomType.ROWCOL, BloomType.ROW };</span> |
| <span class="source-line-no">777</span><span id="line-777"> int[] expKeys = { rowCount * colCount, rowCount };</span> |
| <span class="source-line-no">778</span><span id="line-778"> // below line deserves commentary. it is expected bloom false positives</span> |
| <span class="source-line-no">779</span><span id="line-779"> // column = rowCount*2*colCount inserts</span> |
| <span class="source-line-no">780</span><span id="line-780"> // row-level = only rowCount*2 inserts, but failures will be magnified by</span> |
| <span class="source-line-no">781</span><span id="line-781"> // 2nd for loop for every column (2*colCount)</span> |
| <span class="source-line-no">782</span><span id="line-782"> float[] expErr = { 2 * rowCount * colCount * err, 2 * rowCount * 2 * colCount * err };</span> |
| <span class="source-line-no">783</span><span id="line-783"></span> |
| <span class="source-line-no">784</span><span id="line-784"> if (!fs.exists(ROOT_DIR)) {</span> |
| <span class="source-line-no">785</span><span id="line-785"> fs.mkdirs(ROOT_DIR);</span> |
| <span class="source-line-no">786</span><span id="line-786"> }</span> |
| <span class="source-line-no">787</span><span id="line-787"> for (int x : new int[] { 0, 1 }) {</span> |
| <span class="source-line-no">788</span><span id="line-788"> // write the file</span> |
| <span class="source-line-no">789</span><span id="line-789"> Path f = StoreFileWriter.getUniqueFile(fs, ROOT_DIR);</span> |
| <span class="source-line-no">790</span><span id="line-790"></span> |
| <span class="source-line-no">791</span><span id="line-791"> HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL)</span> |
| <span class="source-line-no">792</span><span id="line-792"> .withChecksumType(CKTYPE).withBytesPerCheckSum(CKBYTES).build();</span> |
| <span class="source-line-no">793</span><span id="line-793"> // Make a store file and write data to it.</span> |
| <span class="source-line-no">794</span><span id="line-794"> StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f)</span> |
| <span class="source-line-no">795</span><span id="line-795"> .withBloomType(bt[x]).withMaxKeyCount(expKeys[x]).withFileContext(meta).build();</span> |
| <span class="source-line-no">796</span><span id="line-796"></span> |
| <span class="source-line-no">797</span><span id="line-797"> long now = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">798</span><span id="line-798"> for (int i = 0; i < rowCount * 2; i += 2) { // rows</span> |
| <span class="source-line-no">799</span><span id="line-799"> for (int j = 0; j < colCount * 2; j += 2) { // column qualifiers</span> |
| <span class="source-line-no">800</span><span id="line-800"> String row = String.format(localFormatter, i);</span> |
| <span class="source-line-no">801</span><span id="line-801"> String col = String.format(localFormatter, j);</span> |
| <span class="source-line-no">802</span><span id="line-802"> for (int k = 0; k < versions; ++k) { // versions</span> |
| <span class="source-line-no">803</span><span id="line-803"> KeyValue kv = new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),</span> |
| <span class="source-line-no">804</span><span id="line-804"> Bytes.toBytes("col" + col), now - k, Bytes.toBytes(-1L));</span> |
| <span class="source-line-no">805</span><span id="line-805"> writer.append(kv);</span> |
| <span class="source-line-no">806</span><span id="line-806"> }</span> |
| <span class="source-line-no">807</span><span id="line-807"> }</span> |
| <span class="source-line-no">808</span><span id="line-808"> }</span> |
| <span class="source-line-no">809</span><span id="line-809"> writer.close();</span> |
| <span class="source-line-no">810</span><span id="line-810"></span> |
| <span class="source-line-no">811</span><span id="line-811"> ReaderContext context =</span> |
| <span class="source-line-no">812</span><span id="line-812"> new ReaderContextBuilder().withFilePath(f).withFileSize(fs.getFileStatus(f).getLen())</span> |
| <span class="source-line-no">813</span><span id="line-813"> .withFileSystem(fs).withInputStreamWrapper(new FSDataInputStreamWrapper(fs, f)).build();</span> |
| <span class="source-line-no">814</span><span id="line-814"> StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, f, true);</span> |
| <span class="source-line-no">815</span><span id="line-815"> storeFileInfo.initHFileInfo(context);</span> |
| <span class="source-line-no">816</span><span id="line-816"> StoreFileReader reader = storeFileInfo.createReader(context, cacheConf);</span> |
| <span class="source-line-no">817</span><span id="line-817"> storeFileInfo.getHFileInfo().initMetaAndIndex(reader.getHFileReader());</span> |
| <span class="source-line-no">818</span><span id="line-818"> reader.loadFileInfo();</span> |
| <span class="source-line-no">819</span><span id="line-819"> reader.loadBloomfilter();</span> |
| <span class="source-line-no">820</span><span id="line-820"> StoreFileScanner scanner = getStoreFileScanner(reader, false, false);</span> |
| <span class="source-line-no">821</span><span id="line-821"> assertEquals(expKeys[x], reader.getGeneralBloomFilter().getKeyCount());</span> |
| <span class="source-line-no">822</span><span id="line-822"></span> |
| <span class="source-line-no">823</span><span id="line-823"> HStore store = mock(HStore.class);</span> |
| <span class="source-line-no">824</span><span id="line-824"> when(store.getColumnFamilyDescriptor())</span> |
| <span class="source-line-no">825</span><span id="line-825"> .thenReturn(ColumnFamilyDescriptorBuilder.of("family"));</span> |
| <span class="source-line-no">826</span><span id="line-826"> // check false positives rate</span> |
| <span class="source-line-no">827</span><span id="line-827"> int falsePos = 0;</span> |
| <span class="source-line-no">828</span><span id="line-828"> int falseNeg = 0;</span> |
| <span class="source-line-no">829</span><span id="line-829"> for (int i = 0; i < rowCount * 2; ++i) { // rows</span> |
| <span class="source-line-no">830</span><span id="line-830"> for (int j = 0; j < colCount * 2; ++j) { // column qualifiers</span> |
| <span class="source-line-no">831</span><span id="line-831"> String row = String.format(localFormatter, i);</span> |
| <span class="source-line-no">832</span><span id="line-832"> String col = String.format(localFormatter, j);</span> |
| <span class="source-line-no">833</span><span id="line-833"> TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">834</span><span id="line-834"> columns.add(Bytes.toBytes("col" + col));</span> |
| <span class="source-line-no">835</span><span id="line-835"></span> |
| <span class="source-line-no">836</span><span id="line-836"> Scan scan =</span> |
| <span class="source-line-no">837</span><span id="line-837"> new Scan().withStartRow(Bytes.toBytes(row)).withStopRow(Bytes.toBytes(row), true);</span> |
| <span class="source-line-no">838</span><span id="line-838"> scan.addColumn(Bytes.toBytes("family"), Bytes.toBytes(("col" + col)));</span> |
| <span class="source-line-no">839</span><span id="line-839"></span> |
| <span class="source-line-no">840</span><span id="line-840"> boolean exists = scanner.shouldUseScanner(scan, store, Long.MIN_VALUE);</span> |
| <span class="source-line-no">841</span><span id="line-841"> boolean shouldRowExist = i % 2 == 0;</span> |
| <span class="source-line-no">842</span><span id="line-842"> boolean shouldColExist = j % 2 == 0;</span> |
| <span class="source-line-no">843</span><span id="line-843"> shouldColExist = shouldColExist || bt[x] == BloomType.ROW;</span> |
| <span class="source-line-no">844</span><span id="line-844"> if (shouldRowExist && shouldColExist) {</span> |
| <span class="source-line-no">845</span><span id="line-845"> if (!exists) {</span> |
| <span class="source-line-no">846</span><span id="line-846"> falseNeg++;</span> |
| <span class="source-line-no">847</span><span id="line-847"> }</span> |
| <span class="source-line-no">848</span><span id="line-848"> } else {</span> |
| <span class="source-line-no">849</span><span id="line-849"> if (exists) {</span> |
| <span class="source-line-no">850</span><span id="line-850"> falsePos++;</span> |
| <span class="source-line-no">851</span><span id="line-851"> }</span> |
| <span class="source-line-no">852</span><span id="line-852"> }</span> |
| <span class="source-line-no">853</span><span id="line-853"> }</span> |
| <span class="source-line-no">854</span><span id="line-854"> }</span> |
| <span class="source-line-no">855</span><span id="line-855"> reader.close(true); // evict because we are about to delete the file</span> |
| <span class="source-line-no">856</span><span id="line-856"> fs.delete(f, true);</span> |
| <span class="source-line-no">857</span><span id="line-857"> System.out.println(bt[x].toString());</span> |
| <span class="source-line-no">858</span><span id="line-858"> System.out.println(" False negatives: " + falseNeg);</span> |
| <span class="source-line-no">859</span><span id="line-859"> System.out.println(" False positives: " + falsePos);</span> |
| <span class="source-line-no">860</span><span id="line-860"> assertEquals(0, falseNeg);</span> |
| <span class="source-line-no">861</span><span id="line-861"> assertTrue(falsePos < 2 * expErr[x]);</span> |
| <span class="source-line-no">862</span><span id="line-862"> }</span> |
| <span class="source-line-no">863</span><span id="line-863"> }</span> |
| <span class="source-line-no">864</span><span id="line-864"></span> |
| <span class="source-line-no">865</span><span id="line-865"> @Test</span> |
| <span class="source-line-no">866</span><span id="line-866"> public void testSeqIdComparator() {</span> |
| <span class="source-line-no">867</span><span id="line-867"> assertOrdering(StoreFileComparators.SEQ_ID, mockStoreFile(true, 100, 1000, -1, "/foo/123"),</span> |
| <span class="source-line-no">868</span><span id="line-868"> mockStoreFile(true, 100, 1000, -1, "/foo/124"), mockStoreFile(true, 99, 1000, -1, "/foo/126"),</span> |
| <span class="source-line-no">869</span><span id="line-869"> mockStoreFile(true, 98, 2000, -1, "/foo/126"), mockStoreFile(false, 3453, -1, 1, "/foo/1"),</span> |
| <span class="source-line-no">870</span><span id="line-870"> mockStoreFile(false, 2, -1, 3, "/foo/2"), mockStoreFile(false, 1000, -1, 5, "/foo/2"),</span> |
| <span class="source-line-no">871</span><span id="line-871"> mockStoreFile(false, 76, -1, 5, "/foo/3"));</span> |
| <span class="source-line-no">872</span><span id="line-872"> }</span> |
| <span class="source-line-no">873</span><span id="line-873"></span> |
| <span class="source-line-no">874</span><span id="line-874"> /**</span> |
| <span class="source-line-no">875</span><span id="line-875"> * Assert that the given comparator orders the given storefiles in the same way that they're</span> |
| <span class="source-line-no">876</span><span id="line-876"> * passed.</span> |
| <span class="source-line-no">877</span><span id="line-877"> */</span> |
| <span class="source-line-no">878</span><span id="line-878"> private void assertOrdering(Comparator<? super HStoreFile> comparator, HStoreFile... sfs) {</span> |
| <span class="source-line-no">879</span><span id="line-879"> ArrayList<HStoreFile> sorted = Lists.newArrayList(sfs);</span> |
| <span class="source-line-no">880</span><span id="line-880"> Collections.shuffle(sorted);</span> |
| <span class="source-line-no">881</span><span id="line-881"> Collections.sort(sorted, comparator);</span> |
| <span class="source-line-no">882</span><span id="line-882"> LOG.debug("sfs: " + Joiner.on(",").join(sfs));</span> |
| <span class="source-line-no">883</span><span id="line-883"> LOG.debug("sorted: " + Joiner.on(",").join(sorted));</span> |
| <span class="source-line-no">884</span><span id="line-884"> assertTrue(Iterables.elementsEqual(Arrays.asList(sfs), sorted));</span> |
| <span class="source-line-no">885</span><span id="line-885"> }</span> |
| <span class="source-line-no">886</span><span id="line-886"></span> |
| <span class="source-line-no">887</span><span id="line-887"> /**</span> |
| <span class="source-line-no">888</span><span id="line-888"> * Create a mock StoreFile with the given attributes.</span> |
| <span class="source-line-no">889</span><span id="line-889"> */</span> |
| <span class="source-line-no">890</span><span id="line-890"> private HStoreFile mockStoreFile(boolean bulkLoad, long size, long bulkTimestamp, long seqId,</span> |
| <span class="source-line-no">891</span><span id="line-891"> String path) {</span> |
| <span class="source-line-no">892</span><span id="line-892"> HStoreFile mock = Mockito.mock(HStoreFile.class);</span> |
| <span class="source-line-no">893</span><span id="line-893"> StoreFileReader reader = Mockito.mock(StoreFileReader.class);</span> |
| <span class="source-line-no">894</span><span id="line-894"></span> |
| <span class="source-line-no">895</span><span id="line-895"> Mockito.doReturn(size).when(reader).length();</span> |
| <span class="source-line-no">896</span><span id="line-896"></span> |
| <span class="source-line-no">897</span><span id="line-897"> Mockito.doReturn(reader).when(mock).getReader();</span> |
| <span class="source-line-no">898</span><span id="line-898"> Mockito.doReturn(bulkLoad).when(mock).isBulkLoadResult();</span> |
| <span class="source-line-no">899</span><span id="line-899"> Mockito.doReturn(OptionalLong.of(bulkTimestamp)).when(mock).getBulkLoadTimestamp();</span> |
| <span class="source-line-no">900</span><span id="line-900"> Mockito.doReturn(seqId).when(mock).getMaxSequenceId();</span> |
| <span class="source-line-no">901</span><span id="line-901"> Mockito.doReturn(new Path(path)).when(mock).getPath();</span> |
| <span class="source-line-no">902</span><span id="line-902"> String name = "mock storefile, bulkLoad=" + bulkLoad + " bulkTimestamp=" + bulkTimestamp</span> |
| <span class="source-line-no">903</span><span id="line-903"> + " seqId=" + seqId + " path=" + path;</span> |
| <span class="source-line-no">904</span><span id="line-904"> Mockito.doReturn(name).when(mock).toString();</span> |
| <span class="source-line-no">905</span><span id="line-905"> return mock;</span> |
| <span class="source-line-no">906</span><span id="line-906"> }</span> |
| <span class="source-line-no">907</span><span id="line-907"></span> |
| <span class="source-line-no">908</span><span id="line-908"> /**</span> |
| <span class="source-line-no">909</span><span id="line-909"> * Generate a list of KeyValues for testing based on given parameters</span> |
| <span class="source-line-no">910</span><span id="line-910"> * @return the rows key-value list</span> |
| <span class="source-line-no">911</span><span id="line-911"> */</span> |
| <span class="source-line-no">912</span><span id="line-912"> List<KeyValue> getKeyValueSet(long[] timestamps, int numRows, byte[] qualifier, byte[] family) {</span> |
| <span class="source-line-no">913</span><span id="line-913"> List<KeyValue> kvList = new ArrayList<>();</span> |
| <span class="source-line-no">914</span><span id="line-914"> for (int i = 1; i <= numRows; i++) {</span> |
| <span class="source-line-no">915</span><span id="line-915"> byte[] b = Bytes.toBytes(i);</span> |
| <span class="source-line-no">916</span><span id="line-916"> LOG.info(Bytes.toString(b));</span> |
| <span class="source-line-no">917</span><span id="line-917"> LOG.info(Bytes.toString(b));</span> |
| <span class="source-line-no">918</span><span id="line-918"> for (long timestamp : timestamps) {</span> |
| <span class="source-line-no">919</span><span id="line-919"> kvList.add(new KeyValue(b, family, qualifier, timestamp, b));</span> |
| <span class="source-line-no">920</span><span id="line-920"> }</span> |
| <span class="source-line-no">921</span><span id="line-921"> }</span> |
| <span class="source-line-no">922</span><span id="line-922"> return kvList;</span> |
| <span class="source-line-no">923</span><span id="line-923"> }</span> |
| <span class="source-line-no">924</span><span id="line-924"></span> |
| <span class="source-line-no">925</span><span id="line-925"> /**</span> |
| <span class="source-line-no">926</span><span id="line-926"> * Test to ensure correctness when using StoreFile with multiple timestamps</span> |
| <span class="source-line-no">927</span><span id="line-927"> */</span> |
| <span class="source-line-no">928</span><span id="line-928"> @Test</span> |
| <span class="source-line-no">929</span><span id="line-929"> public void testMultipleTimestamps() throws IOException {</span> |
| <span class="source-line-no">930</span><span id="line-930"> byte[] family = Bytes.toBytes("familyname");</span> |
| <span class="source-line-no">931</span><span id="line-931"> byte[] qualifier = Bytes.toBytes("qualifier");</span> |
| <span class="source-line-no">932</span><span id="line-932"> int numRows = 10;</span> |
| <span class="source-line-no">933</span><span id="line-933"> long[] timestamps = new long[] { 20, 10, 5, 1 };</span> |
| <span class="source-line-no">934</span><span id="line-934"> Scan scan = new Scan();</span> |
| <span class="source-line-no">935</span><span id="line-935"></span> |
| <span class="source-line-no">936</span><span id="line-936"> // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname.</span> |
| <span class="source-line-no">937</span><span id="line-937"> Path storedir = new Path(new Path(testDir, "7e0102"), Bytes.toString(family));</span> |
| <span class="source-line-no">938</span><span id="line-938"> Path dir = new Path(storedir, "1234567890");</span> |
| <span class="source-line-no">939</span><span id="line-939"> HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();</span> |
| <span class="source-line-no">940</span><span id="line-940"> // Make a store file and write data to it.</span> |
| <span class="source-line-no">941</span><span id="line-941"> StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)</span> |
| <span class="source-line-no">942</span><span id="line-942"> .withOutputDir(dir).withFileContext(meta).build();</span> |
| <span class="source-line-no">943</span><span id="line-943"></span> |
| <span class="source-line-no">944</span><span id="line-944"> List<KeyValue> kvList = getKeyValueSet(timestamps, numRows, qualifier, family);</span> |
| <span class="source-line-no">945</span><span id="line-945"></span> |
| <span class="source-line-no">946</span><span id="line-946"> for (KeyValue kv : kvList) {</span> |
| <span class="source-line-no">947</span><span id="line-947"> writer.append(kv);</span> |
| <span class="source-line-no">948</span><span id="line-948"> }</span> |
| <span class="source-line-no">949</span><span id="line-949"> writer.appendMetadata(0, false);</span> |
| <span class="source-line-no">950</span><span id="line-950"> writer.close();</span> |
| <span class="source-line-no">951</span><span id="line-951"></span> |
| <span class="source-line-no">952</span><span id="line-952"> HStoreFile hsf =</span> |
| <span class="source-line-no">953</span><span id="line-953"> new HStoreFile(this.fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">954</span><span id="line-954"> HStore store = mock(HStore.class);</span> |
| <span class="source-line-no">955</span><span id="line-955"> when(store.getColumnFamilyDescriptor()).thenReturn(ColumnFamilyDescriptorBuilder.of(family));</span> |
| <span class="source-line-no">956</span><span id="line-956"> hsf.initReader();</span> |
| <span class="source-line-no">957</span><span id="line-957"> StoreFileReader reader = hsf.getReader();</span> |
| <span class="source-line-no">958</span><span id="line-958"> StoreFileScanner scanner = getStoreFileScanner(reader, false, false);</span> |
| <span class="source-line-no">959</span><span id="line-959"> TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR);</span> |
| <span class="source-line-no">960</span><span id="line-960"> columns.add(qualifier);</span> |
| <span class="source-line-no">961</span><span id="line-961"></span> |
| <span class="source-line-no">962</span><span id="line-962"> scan.setTimeRange(20, 100);</span> |
| <span class="source-line-no">963</span><span id="line-963"> assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE));</span> |
| <span class="source-line-no">964</span><span id="line-964"></span> |
| <span class="source-line-no">965</span><span id="line-965"> scan.setTimeRange(1, 2);</span> |
| <span class="source-line-no">966</span><span id="line-966"> assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE));</span> |
| <span class="source-line-no">967</span><span id="line-967"></span> |
| <span class="source-line-no">968</span><span id="line-968"> scan.setTimeRange(8, 10);</span> |
| <span class="source-line-no">969</span><span id="line-969"> assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE));</span> |
| <span class="source-line-no">970</span><span id="line-970"></span> |
| <span class="source-line-no">971</span><span id="line-971"> // lets make sure it still works with column family time ranges</span> |
| <span class="source-line-no">972</span><span id="line-972"> scan.setColumnFamilyTimeRange(family, 7, 50);</span> |
| <span class="source-line-no">973</span><span id="line-973"> assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE));</span> |
| <span class="source-line-no">974</span><span id="line-974"></span> |
| <span class="source-line-no">975</span><span id="line-975"> // This test relies on the timestamp range optimization</span> |
| <span class="source-line-no">976</span><span id="line-976"> scan = new Scan();</span> |
| <span class="source-line-no">977</span><span id="line-977"> scan.setTimeRange(27, 50);</span> |
| <span class="source-line-no">978</span><span id="line-978"> assertTrue(!scanner.shouldUseScanner(scan, store, Long.MIN_VALUE));</span> |
| <span class="source-line-no">979</span><span id="line-979"></span> |
| <span class="source-line-no">980</span><span id="line-980"> // should still use the scanner because we override the family time range</span> |
| <span class="source-line-no">981</span><span id="line-981"> scan = new Scan();</span> |
| <span class="source-line-no">982</span><span id="line-982"> scan.setTimeRange(27, 50);</span> |
| <span class="source-line-no">983</span><span id="line-983"> scan.setColumnFamilyTimeRange(family, 7, 50);</span> |
| <span class="source-line-no">984</span><span id="line-984"> assertTrue(scanner.shouldUseScanner(scan, store, Long.MIN_VALUE));</span> |
| <span class="source-line-no">985</span><span id="line-985"> }</span> |
| <span class="source-line-no">986</span><span id="line-986"></span> |
| <span class="source-line-no">987</span><span id="line-987"> @Test</span> |
| <span class="source-line-no">988</span><span id="line-988"> public void testCacheOnWriteEvictOnClose() throws Exception {</span> |
| <span class="source-line-no">989</span><span id="line-989"> Configuration conf = this.conf;</span> |
| <span class="source-line-no">990</span><span id="line-990"></span> |
| <span class="source-line-no">991</span><span id="line-991"> // Find a home for our files (regiondir ("7e0102") and familyname).</span> |
| <span class="source-line-no">992</span><span id="line-992"> Path baseDir = new Path(new Path(testDir, "7e0102"), "twoCOWEOC");</span> |
| <span class="source-line-no">993</span><span id="line-993"></span> |
| <span class="source-line-no">994</span><span id="line-994"> // Grab the block cache and get the initial hit/miss counts</span> |
| <span class="source-line-no">995</span><span id="line-995"> BlockCache bc = BlockCacheFactory.createBlockCache(conf);</span> |
| <span class="source-line-no">996</span><span id="line-996"> assertNotNull(bc);</span> |
| <span class="source-line-no">997</span><span id="line-997"> CacheStats cs = bc.getStats();</span> |
| <span class="source-line-no">998</span><span id="line-998"> long startHit = cs.getHitCount();</span> |
| <span class="source-line-no">999</span><span id="line-999"> long startMiss = cs.getMissCount();</span> |
| <span class="source-line-no">1000</span><span id="line-1000"> long startEvicted = cs.getEvictedCount();</span> |
| <span class="source-line-no">1001</span><span id="line-1001"></span> |
| <span class="source-line-no">1002</span><span id="line-1002"> // Let's write a StoreFile with three blocks, with cache on write off</span> |
| <span class="source-line-no">1003</span><span id="line-1003"> conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false);</span> |
| <span class="source-line-no">1004</span><span id="line-1004"> CacheConfig cacheConf = new CacheConfig(conf, bc);</span> |
| <span class="source-line-no">1005</span><span id="line-1005"> Path pathCowOff = new Path(baseDir, "123456789");</span> |
| <span class="source-line-no">1006</span><span id="line-1006"> StoreFileWriter writer = writeStoreFile(conf, cacheConf, pathCowOff, 3);</span> |
| <span class="source-line-no">1007</span><span id="line-1007"> HStoreFile hsf =</span> |
| <span class="source-line-no">1008</span><span id="line-1008"> new HStoreFile(this.fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">1009</span><span id="line-1009"> LOG.debug(hsf.getPath().toString());</span> |
| <span class="source-line-no">1010</span><span id="line-1010"></span> |
| <span class="source-line-no">1011</span><span id="line-1011"> // Read this file, we should see 3 misses</span> |
| <span class="source-line-no">1012</span><span id="line-1012"> hsf.initReader();</span> |
| <span class="source-line-no">1013</span><span id="line-1013"> StoreFileReader reader = hsf.getReader();</span> |
| <span class="source-line-no">1014</span><span id="line-1014"> reader.loadFileInfo();</span> |
| <span class="source-line-no">1015</span><span id="line-1015"> StoreFileScanner scanner = getStoreFileScanner(reader, true, true);</span> |
| <span class="source-line-no">1016</span><span id="line-1016"> scanner.seek(KeyValue.LOWESTKEY);</span> |
| <span class="source-line-no">1017</span><span id="line-1017"> while (scanner.next() != null) {</span> |
| <span class="source-line-no">1018</span><span id="line-1018"> continue;</span> |
| <span class="source-line-no">1019</span><span id="line-1019"> }</span> |
| <span class="source-line-no">1020</span><span id="line-1020"> assertEquals(startHit, cs.getHitCount());</span> |
| <span class="source-line-no">1021</span><span id="line-1021"> assertEquals(startMiss + 3, cs.getMissCount());</span> |
| <span class="source-line-no">1022</span><span id="line-1022"> assertEquals(startEvicted, cs.getEvictedCount());</span> |
| <span class="source-line-no">1023</span><span id="line-1023"> startMiss += 3;</span> |
| <span class="source-line-no">1024</span><span id="line-1024"> scanner.close();</span> |
| <span class="source-line-no">1025</span><span id="line-1025"> reader.close(cacheConf.shouldEvictOnClose());</span> |
| <span class="source-line-no">1026</span><span id="line-1026"></span> |
| <span class="source-line-no">1027</span><span id="line-1027"> // Now write a StoreFile with three blocks, with cache on write on</span> |
| <span class="source-line-no">1028</span><span id="line-1028"> conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true);</span> |
| <span class="source-line-no">1029</span><span id="line-1029"> cacheConf = new CacheConfig(conf, bc);</span> |
| <span class="source-line-no">1030</span><span id="line-1030"> Path pathCowOn = new Path(baseDir, "123456788");</span> |
| <span class="source-line-no">1031</span><span id="line-1031"> writer = writeStoreFile(conf, cacheConf, pathCowOn, 3);</span> |
| <span class="source-line-no">1032</span><span id="line-1032"> hsf = new HStoreFile(this.fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">1033</span><span id="line-1033"></span> |
| <span class="source-line-no">1034</span><span id="line-1034"> // Read this file, we should see 3 hits</span> |
| <span class="source-line-no">1035</span><span id="line-1035"> hsf.initReader();</span> |
| <span class="source-line-no">1036</span><span id="line-1036"> reader = hsf.getReader();</span> |
| <span class="source-line-no">1037</span><span id="line-1037"> scanner = getStoreFileScanner(reader, true, true);</span> |
| <span class="source-line-no">1038</span><span id="line-1038"> scanner.seek(KeyValue.LOWESTKEY);</span> |
| <span class="source-line-no">1039</span><span id="line-1039"> while (scanner.next() != null) {</span> |
| <span class="source-line-no">1040</span><span id="line-1040"> continue;</span> |
| <span class="source-line-no">1041</span><span id="line-1041"> }</span> |
| <span class="source-line-no">1042</span><span id="line-1042"> assertEquals(startHit + 3, cs.getHitCount());</span> |
| <span class="source-line-no">1043</span><span id="line-1043"> assertEquals(startMiss, cs.getMissCount());</span> |
| <span class="source-line-no">1044</span><span id="line-1044"> assertEquals(startEvicted, cs.getEvictedCount());</span> |
| <span class="source-line-no">1045</span><span id="line-1045"> startHit += 3;</span> |
| <span class="source-line-no">1046</span><span id="line-1046"> scanner.close();</span> |
| <span class="source-line-no">1047</span><span id="line-1047"> reader.close(cacheConf.shouldEvictOnClose());</span> |
| <span class="source-line-no">1048</span><span id="line-1048"></span> |
| <span class="source-line-no">1049</span><span id="line-1049"> // Let's read back the two files to ensure the blocks exactly match</span> |
| <span class="source-line-no">1050</span><span id="line-1050"> hsf = new HStoreFile(this.fs, pathCowOff, conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">1051</span><span id="line-1051"> hsf.initReader();</span> |
| <span class="source-line-no">1052</span><span id="line-1052"> StoreFileReader readerOne = hsf.getReader();</span> |
| <span class="source-line-no">1053</span><span id="line-1053"> readerOne.loadFileInfo();</span> |
| <span class="source-line-no">1054</span><span id="line-1054"> StoreFileScanner scannerOne = getStoreFileScanner(readerOne, true, true);</span> |
| <span class="source-line-no">1055</span><span id="line-1055"> scannerOne.seek(KeyValue.LOWESTKEY);</span> |
| <span class="source-line-no">1056</span><span id="line-1056"> hsf = new HStoreFile(this.fs, pathCowOn, conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">1057</span><span id="line-1057"> hsf.initReader();</span> |
| <span class="source-line-no">1058</span><span id="line-1058"> StoreFileReader readerTwo = hsf.getReader();</span> |
| <span class="source-line-no">1059</span><span id="line-1059"> readerTwo.loadFileInfo();</span> |
| <span class="source-line-no">1060</span><span id="line-1060"> StoreFileScanner scannerTwo = getStoreFileScanner(readerTwo, true, true);</span> |
| <span class="source-line-no">1061</span><span id="line-1061"> scannerTwo.seek(KeyValue.LOWESTKEY);</span> |
| <span class="source-line-no">1062</span><span id="line-1062"> ExtendedCell kv1 = null;</span> |
| <span class="source-line-no">1063</span><span id="line-1063"> ExtendedCell kv2 = null;</span> |
| <span class="source-line-no">1064</span><span id="line-1064"> while ((kv1 = scannerOne.next()) != null) {</span> |
| <span class="source-line-no">1065</span><span id="line-1065"> kv2 = scannerTwo.next();</span> |
| <span class="source-line-no">1066</span><span id="line-1066"> assertTrue(kv1.equals(kv2));</span> |
| <span class="source-line-no">1067</span><span id="line-1067"> KeyValue keyv1 = KeyValueUtil.ensureKeyValue(kv1);</span> |
| <span class="source-line-no">1068</span><span id="line-1068"> KeyValue keyv2 = KeyValueUtil.ensureKeyValue(kv2);</span> |
| <span class="source-line-no">1069</span><span id="line-1069"> assertTrue(Bytes.compareTo(keyv1.getBuffer(), keyv1.getKeyOffset(), keyv1.getKeyLength(),</span> |
| <span class="source-line-no">1070</span><span id="line-1070"> keyv2.getBuffer(), keyv2.getKeyOffset(), keyv2.getKeyLength()) == 0);</span> |
| <span class="source-line-no">1071</span><span id="line-1071"> assertTrue(Bytes.compareTo(kv1.getValueArray(), kv1.getValueOffset(), kv1.getValueLength(),</span> |
| <span class="source-line-no">1072</span><span id="line-1072"> kv2.getValueArray(), kv2.getValueOffset(), kv2.getValueLength()) == 0);</span> |
| <span class="source-line-no">1073</span><span id="line-1073"> }</span> |
| <span class="source-line-no">1074</span><span id="line-1074"> assertNull(scannerTwo.next());</span> |
| <span class="source-line-no">1075</span><span id="line-1075"> assertEquals(startHit + 6, cs.getHitCount());</span> |
| <span class="source-line-no">1076</span><span id="line-1076"> assertEquals(startMiss, cs.getMissCount());</span> |
| <span class="source-line-no">1077</span><span id="line-1077"> assertEquals(startEvicted, cs.getEvictedCount());</span> |
| <span class="source-line-no">1078</span><span id="line-1078"> startHit += 6;</span> |
| <span class="source-line-no">1079</span><span id="line-1079"> scannerOne.close();</span> |
| <span class="source-line-no">1080</span><span id="line-1080"> readerOne.close(cacheConf.shouldEvictOnClose());</span> |
| <span class="source-line-no">1081</span><span id="line-1081"> scannerTwo.close();</span> |
| <span class="source-line-no">1082</span><span id="line-1082"> readerTwo.close(cacheConf.shouldEvictOnClose());</span> |
| <span class="source-line-no">1083</span><span id="line-1083"></span> |
| <span class="source-line-no">1084</span><span id="line-1084"> // Let's close the first file with evict on close turned on</span> |
| <span class="source-line-no">1085</span><span id="line-1085"> conf.setBoolean("hbase.rs.evictblocksonclose", true);</span> |
| <span class="source-line-no">1086</span><span id="line-1086"> cacheConf = new CacheConfig(conf, bc);</span> |
| <span class="source-line-no">1087</span><span id="line-1087"> hsf = new HStoreFile(this.fs, pathCowOff, conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">1088</span><span id="line-1088"> hsf.initReader();</span> |
| <span class="source-line-no">1089</span><span id="line-1089"> reader = hsf.getReader();</span> |
| <span class="source-line-no">1090</span><span id="line-1090"> reader.close(cacheConf.shouldEvictOnClose());</span> |
| <span class="source-line-no">1091</span><span id="line-1091"></span> |
| <span class="source-line-no">1092</span><span id="line-1092"> // We should have 3 new evictions but the evict count stat should not change. Eviction because</span> |
| <span class="source-line-no">1093</span><span id="line-1093"> // of HFile invalidation is not counted along with normal evictions</span> |
| <span class="source-line-no">1094</span><span id="line-1094"> assertEquals(startHit, cs.getHitCount());</span> |
| <span class="source-line-no">1095</span><span id="line-1095"> assertEquals(startMiss, cs.getMissCount());</span> |
| <span class="source-line-no">1096</span><span id="line-1096"> assertEquals(startEvicted, cs.getEvictedCount());</span> |
| <span class="source-line-no">1097</span><span id="line-1097"></span> |
| <span class="source-line-no">1098</span><span id="line-1098"> // Let's close the second file with evict on close turned off</span> |
| <span class="source-line-no">1099</span><span id="line-1099"> conf.setBoolean("hbase.rs.evictblocksonclose", false);</span> |
| <span class="source-line-no">1100</span><span id="line-1100"> cacheConf = new CacheConfig(conf, bc);</span> |
| <span class="source-line-no">1101</span><span id="line-1101"> hsf = new HStoreFile(this.fs, pathCowOn, conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">1102</span><span id="line-1102"> hsf.initReader();</span> |
| <span class="source-line-no">1103</span><span id="line-1103"> reader = hsf.getReader();</span> |
| <span class="source-line-no">1104</span><span id="line-1104"> reader.close(cacheConf.shouldEvictOnClose());</span> |
| <span class="source-line-no">1105</span><span id="line-1105"></span> |
| <span class="source-line-no">1106</span><span id="line-1106"> // We expect no changes</span> |
| <span class="source-line-no">1107</span><span id="line-1107"> assertEquals(startHit, cs.getHitCount());</span> |
| <span class="source-line-no">1108</span><span id="line-1108"> assertEquals(startMiss, cs.getMissCount());</span> |
| <span class="source-line-no">1109</span><span id="line-1109"> assertEquals(startEvicted, cs.getEvictedCount());</span> |
| <span class="source-line-no">1110</span><span id="line-1110"> }</span> |
| <span class="source-line-no">1111</span><span id="line-1111"></span> |
| <span class="source-line-no">1112</span><span id="line-1112"> private Path splitStoreFile(final HRegionFileSystem regionFs, final RegionInfo hri,</span> |
| <span class="source-line-no">1113</span><span id="line-1113"> final String family, final HStoreFile sf, final byte[] splitKey, boolean isTopRef)</span> |
| <span class="source-line-no">1114</span><span id="line-1114"> throws IOException {</span> |
| <span class="source-line-no">1115</span><span id="line-1115"> Path path = regionFs.splitStoreFile(hri, family, sf, splitKey, isTopRef, null);</span> |
| <span class="source-line-no">1116</span><span id="line-1116"> if (null == path) {</span> |
| <span class="source-line-no">1117</span><span id="line-1117"> return null;</span> |
| <span class="source-line-no">1118</span><span id="line-1118"> }</span> |
| <span class="source-line-no">1119</span><span id="line-1119"> List<Path> splitFiles = new ArrayList<>();</span> |
| <span class="source-line-no">1120</span><span id="line-1120"> splitFiles.add(path);</span> |
| <span class="source-line-no">1121</span><span id="line-1121"> MasterProcedureEnv mockEnv = mock(MasterProcedureEnv.class);</span> |
| <span class="source-line-no">1122</span><span id="line-1122"> MasterServices mockServices = mock(MasterServices.class);</span> |
| <span class="source-line-no">1123</span><span id="line-1123"> when(mockEnv.getMasterServices()).thenReturn(mockServices);</span> |
| <span class="source-line-no">1124</span><span id="line-1124"> when(mockEnv.getMasterConfiguration()).thenReturn(new Configuration());</span> |
| <span class="source-line-no">1125</span><span id="line-1125"> TableDescriptors mockTblDescs = mock(TableDescriptors.class);</span> |
| <span class="source-line-no">1126</span><span id="line-1126"> when(mockServices.getTableDescriptors()).thenReturn(mockTblDescs);</span> |
| <span class="source-line-no">1127</span><span id="line-1127"> TableDescriptor mockTblDesc = TableDescriptorBuilder.newBuilder(hri.getTable())</span> |
| <span class="source-line-no">1128</span><span id="line-1128"> .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();</span> |
| <span class="source-line-no">1129</span><span id="line-1129"> when(mockTblDescs.get(any())).thenReturn(mockTblDesc);</span> |
| <span class="source-line-no">1130</span><span id="line-1130"> Path regionDir = regionFs.commitDaughterRegion(hri, splitFiles, mockEnv);</span> |
| <span class="source-line-no">1131</span><span id="line-1131"> return new Path(new Path(regionDir, family), path.getName());</span> |
| <span class="source-line-no">1132</span><span id="line-1132"> }</span> |
| <span class="source-line-no">1133</span><span id="line-1133"></span> |
| <span class="source-line-no">1134</span><span id="line-1134"> private StoreFileWriter writeStoreFile(Configuration conf, CacheConfig cacheConf, Path path,</span> |
| <span class="source-line-no">1135</span><span id="line-1135"> int numBlocks) throws IOException {</span> |
| <span class="source-line-no">1136</span><span id="line-1136"> // Let's put ~5 small KVs in each block, so let's make 5*numBlocks KVs</span> |
| <span class="source-line-no">1137</span><span id="line-1137"> int numKVs = 5 * numBlocks;</span> |
| <span class="source-line-no">1138</span><span id="line-1138"> List<KeyValue> kvs = new ArrayList<>(numKVs);</span> |
| <span class="source-line-no">1139</span><span id="line-1139"> byte[] b = Bytes.toBytes("x");</span> |
| <span class="source-line-no">1140</span><span id="line-1140"> int totalSize = 0;</span> |
| <span class="source-line-no">1141</span><span id="line-1141"> for (int i = numKVs; i > 0; i--) {</span> |
| <span class="source-line-no">1142</span><span id="line-1142"> KeyValue kv = new KeyValue(b, b, b, i, b);</span> |
| <span class="source-line-no">1143</span><span id="line-1143"> kvs.add(kv);</span> |
| <span class="source-line-no">1144</span><span id="line-1144"> // kv has memstoreTS 0, which takes 1 byte to store.</span> |
| <span class="source-line-no">1145</span><span id="line-1145"> totalSize += kv.getLength() + 1;</span> |
| <span class="source-line-no">1146</span><span id="line-1146"> }</span> |
| <span class="source-line-no">1147</span><span id="line-1147"> int blockSize = totalSize / numBlocks;</span> |
| <span class="source-line-no">1148</span><span id="line-1148"> HFileContext meta = new HFileContextBuilder().withBlockSize(blockSize).withChecksumType(CKTYPE)</span> |
| <span class="source-line-no">1149</span><span id="line-1149"> .withBytesPerCheckSum(CKBYTES).build();</span> |
| <span class="source-line-no">1150</span><span id="line-1150"> // Make a store file and write data to it.</span> |
| <span class="source-line-no">1151</span><span id="line-1151"> StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)</span> |
| <span class="source-line-no">1152</span><span id="line-1152"> .withFilePath(path).withMaxKeyCount(2000).withFileContext(meta).build();</span> |
| <span class="source-line-no">1153</span><span id="line-1153"> // We'll write N-1 KVs to ensure we don't write an extra block</span> |
| <span class="source-line-no">1154</span><span id="line-1154"> kvs.remove(kvs.size() - 1);</span> |
| <span class="source-line-no">1155</span><span id="line-1155"> for (KeyValue kv : kvs) {</span> |
| <span class="source-line-no">1156</span><span id="line-1156"> writer.append(kv);</span> |
| <span class="source-line-no">1157</span><span id="line-1157"> }</span> |
| <span class="source-line-no">1158</span><span id="line-1158"> writer.appendMetadata(0, false);</span> |
| <span class="source-line-no">1159</span><span id="line-1159"> writer.close();</span> |
| <span class="source-line-no">1160</span><span id="line-1160"> return writer;</span> |
| <span class="source-line-no">1161</span><span id="line-1161"> }</span> |
| <span class="source-line-no">1162</span><span id="line-1162"></span> |
| <span class="source-line-no">1163</span><span id="line-1163"> /**</span> |
| <span class="source-line-no">1164</span><span id="line-1164"> * Check if data block encoding information is saved correctly in HFile's file info.</span> |
| <span class="source-line-no">1165</span><span id="line-1165"> */</span> |
| <span class="source-line-no">1166</span><span id="line-1166"> @Test</span> |
| <span class="source-line-no">1167</span><span id="line-1167"> public void testDataBlockEncodingMetaData() throws IOException {</span> |
| <span class="source-line-no">1168</span><span id="line-1168"> // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname.</span> |
| <span class="source-line-no">1169</span><span id="line-1169"> Path dir = new Path(new Path(testDir, "7e0102"), "familyname");</span> |
| <span class="source-line-no">1170</span><span id="line-1170"> Path path = new Path(dir, "1234567890");</span> |
| <span class="source-line-no">1171</span><span id="line-1171"></span> |
| <span class="source-line-no">1172</span><span id="line-1172"> DataBlockEncoding dataBlockEncoderAlgo = DataBlockEncoding.FAST_DIFF;</span> |
| <span class="source-line-no">1173</span><span id="line-1173"> cacheConf = new CacheConfig(conf);</span> |
| <span class="source-line-no">1174</span><span id="line-1174"> HFileContext meta =</span> |
| <span class="source-line-no">1175</span><span id="line-1175"> new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).withChecksumType(CKTYPE)</span> |
| <span class="source-line-no">1176</span><span id="line-1176"> .withBytesPerCheckSum(CKBYTES).withDataBlockEncoding(dataBlockEncoderAlgo).build();</span> |
| <span class="source-line-no">1177</span><span id="line-1177"> // Make a store file and write data to it.</span> |
| <span class="source-line-no">1178</span><span id="line-1178"> StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)</span> |
| <span class="source-line-no">1179</span><span id="line-1179"> .withFilePath(path).withMaxKeyCount(2000).withFileContext(meta).build();</span> |
| <span class="source-line-no">1180</span><span id="line-1180"> writer.close();</span> |
| <span class="source-line-no">1181</span><span id="line-1181"></span> |
| <span class="source-line-no">1182</span><span id="line-1182"> HStoreFile storeFile =</span> |
| <span class="source-line-no">1183</span><span id="line-1183"> new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">1184</span><span id="line-1184"> storeFile.initReader();</span> |
| <span class="source-line-no">1185</span><span id="line-1185"> StoreFileReader reader = storeFile.getReader();</span> |
| <span class="source-line-no">1186</span><span id="line-1186"></span> |
| <span class="source-line-no">1187</span><span id="line-1187"> Map<byte[], byte[]> fileInfo = reader.loadFileInfo();</span> |
| <span class="source-line-no">1188</span><span id="line-1188"> byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING);</span> |
| <span class="source-line-no">1189</span><span id="line-1189"> assertArrayEquals(dataBlockEncoderAlgo.getNameInBytes(), value);</span> |
| <span class="source-line-no">1190</span><span id="line-1190"> }</span> |
| <span class="source-line-no">1191</span><span id="line-1191"></span> |
| <span class="source-line-no">1192</span><span id="line-1192"> @Test</span> |
| <span class="source-line-no">1193</span><span id="line-1193"> public void testDataBlockSizeEncoded() throws Exception {</span> |
| <span class="source-line-no">1194</span><span id="line-1194"> // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname.</span> |
| <span class="source-line-no">1195</span><span id="line-1195"> Path dir = new Path(new Path(this.testDir, "7e0102"), "familyname");</span> |
| <span class="source-line-no">1196</span><span id="line-1196"> Path path = new Path(dir, "1234567890");</span> |
| <span class="source-line-no">1197</span><span id="line-1197"></span> |
| <span class="source-line-no">1198</span><span id="line-1198"> DataBlockEncoding dataBlockEncoderAlgo = DataBlockEncoding.FAST_DIFF;</span> |
| <span class="source-line-no">1199</span><span id="line-1199"></span> |
| <span class="source-line-no">1200</span><span id="line-1200"> conf.setDouble("hbase.writer.unified.encoded.blocksize.ratio", 1);</span> |
| <span class="source-line-no">1201</span><span id="line-1201"></span> |
| <span class="source-line-no">1202</span><span id="line-1202"> cacheConf = new CacheConfig(conf);</span> |
| <span class="source-line-no">1203</span><span id="line-1203"> HFileContext meta =</span> |
| <span class="source-line-no">1204</span><span id="line-1204"> new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).withChecksumType(CKTYPE)</span> |
| <span class="source-line-no">1205</span><span id="line-1205"> .withBytesPerCheckSum(CKBYTES).withDataBlockEncoding(dataBlockEncoderAlgo).build();</span> |
| <span class="source-line-no">1206</span><span id="line-1206"> // Make a store file and write data to it.</span> |
| <span class="source-line-no">1207</span><span id="line-1207"> StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)</span> |
| <span class="source-line-no">1208</span><span id="line-1208"> .withFilePath(path).withMaxKeyCount(2000).withFileContext(meta).build();</span> |
| <span class="source-line-no">1209</span><span id="line-1209"> writeStoreFile(writer);</span> |
| <span class="source-line-no">1210</span><span id="line-1210"></span> |
| <span class="source-line-no">1211</span><span id="line-1211"> HStoreFile storeFile =</span> |
| <span class="source-line-no">1212</span><span id="line-1212"> new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">1213</span><span id="line-1213"> storeFile.initReader();</span> |
| <span class="source-line-no">1214</span><span id="line-1214"> StoreFileReader reader = storeFile.getReader();</span> |
| <span class="source-line-no">1215</span><span id="line-1215"></span> |
| <span class="source-line-no">1216</span><span id="line-1216"> Map<byte[], byte[]> fileInfo = reader.loadFileInfo();</span> |
| <span class="source-line-no">1217</span><span id="line-1217"> byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING);</span> |
| <span class="source-line-no">1218</span><span id="line-1218"> assertEquals(dataBlockEncoderAlgo.name(), Bytes.toString(value));</span> |
| <span class="source-line-no">1219</span><span id="line-1219"></span> |
| <span class="source-line-no">1220</span><span id="line-1220"> HFile.Reader fReader =</span> |
| <span class="source-line-no">1221</span><span id="line-1221"> HFile.createReader(fs, writer.getPath(), storeFile.getCacheConf(), true, conf);</span> |
| <span class="source-line-no">1222</span><span id="line-1222"></span> |
| <span class="source-line-no">1223</span><span id="line-1223"> FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, writer.getPath());</span> |
| <span class="source-line-no">1224</span><span id="line-1224"> long fileSize = fs.getFileStatus(writer.getPath()).getLen();</span> |
| <span class="source-line-no">1225</span><span id="line-1225"> FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize);</span> |
| <span class="source-line-no">1226</span><span id="line-1226"> long offset = trailer.getFirstDataBlockOffset(), max = trailer.getLastDataBlockOffset();</span> |
| <span class="source-line-no">1227</span><span id="line-1227"> HFileBlock block;</span> |
| <span class="source-line-no">1228</span><span id="line-1228"> while (offset <= max) {</span> |
| <span class="source-line-no">1229</span><span id="line-1229"> block = fReader.readBlock(offset, -1, /* cacheBlock */</span> |
| <span class="source-line-no">1230</span><span id="line-1230"> false, /* pread */ false, /* isCompaction */ false, /* updateCacheMetrics */</span> |
| <span class="source-line-no">1231</span><span id="line-1231"> false, null, null);</span> |
| <span class="source-line-no">1232</span><span id="line-1232"> offset += block.getOnDiskSizeWithHeader();</span> |
| <span class="source-line-no">1233</span><span id="line-1233"> double diff = block.getOnDiskSizeWithHeader() - BLOCKSIZE_SMALL;</span> |
| <span class="source-line-no">1234</span><span id="line-1234"> if (offset <= max) {</span> |
| <span class="source-line-no">1235</span><span id="line-1235"> assertTrue(diff >= 0 && diff < (BLOCKSIZE_SMALL * 0.05));</span> |
| <span class="source-line-no">1236</span><span id="line-1236"> }</span> |
| <span class="source-line-no">1237</span><span id="line-1237"> }</span> |
| <span class="source-line-no">1238</span><span id="line-1238"> }</span> |
| <span class="source-line-no">1239</span><span id="line-1239"></span> |
| <span class="source-line-no">1240</span><span id="line-1240"> @Test</span> |
| <span class="source-line-no">1241</span><span id="line-1241"> public void testDataBlockSizeCompressed() throws Exception {</span> |
| <span class="source-line-no">1242</span><span id="line-1242"> conf.set(BLOCK_COMPRESSED_SIZE_PREDICATOR,</span> |
| <span class="source-line-no">1243</span><span id="line-1243"> PreviousBlockCompressionRatePredicator.class.getName());</span> |
| <span class="source-line-no">1244</span><span id="line-1244"> testDataBlockSizeWithCompressionRatePredicator(12,</span> |
| <span class="source-line-no">1245</span><span id="line-1245"> (s, c) -> (c > 2 && c < 11) ? s >= BLOCKSIZE_SMALL * 10 : true);</span> |
| <span class="source-line-no">1246</span><span id="line-1246"> }</span> |
| <span class="source-line-no">1247</span><span id="line-1247"></span> |
| <span class="source-line-no">1248</span><span id="line-1248"> @Test</span> |
| <span class="source-line-no">1249</span><span id="line-1249"> public void testDataBlockSizeUnCompressed() throws Exception {</span> |
| <span class="source-line-no">1250</span><span id="line-1250"> conf.set(BLOCK_COMPRESSED_SIZE_PREDICATOR, UncompressedBlockSizePredicator.class.getName());</span> |
| <span class="source-line-no">1251</span><span id="line-1251"> testDataBlockSizeWithCompressionRatePredicator(200, (s, c) -> s < BLOCKSIZE_SMALL * 10);</span> |
| <span class="source-line-no">1252</span><span id="line-1252"> }</span> |
| <span class="source-line-no">1253</span><span id="line-1253"></span> |
| <span class="source-line-no">1254</span><span id="line-1254"> private void testDataBlockSizeWithCompressionRatePredicator(int expectedBlockCount,</span> |
| <span class="source-line-no">1255</span><span id="line-1255"> BiFunction<Integer, Integer, Boolean> validation) throws Exception {</span> |
| <span class="source-line-no">1256</span><span id="line-1256"> Path dir = new Path(new Path(this.testDir, "7e0102"), "familyname");</span> |
| <span class="source-line-no">1257</span><span id="line-1257"> Path path = new Path(dir, "1234567890");</span> |
| <span class="source-line-no">1258</span><span id="line-1258"> DataBlockEncoding dataBlockEncoderAlgo = DataBlockEncoding.FAST_DIFF;</span> |
| <span class="source-line-no">1259</span><span id="line-1259"> cacheConf = new CacheConfig(conf);</span> |
| <span class="source-line-no">1260</span><span id="line-1260"> HFileContext meta =</span> |
| <span class="source-line-no">1261</span><span id="line-1261"> new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).withChecksumType(CKTYPE)</span> |
| <span class="source-line-no">1262</span><span id="line-1262"> .withBytesPerCheckSum(CKBYTES).withDataBlockEncoding(dataBlockEncoderAlgo)</span> |
| <span class="source-line-no">1263</span><span id="line-1263"> .withCompression(Compression.Algorithm.GZ).build();</span> |
| <span class="source-line-no">1264</span><span id="line-1264"> // Make a store file and write data to it.</span> |
| <span class="source-line-no">1265</span><span id="line-1265"> StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs)</span> |
| <span class="source-line-no">1266</span><span id="line-1266"> .withFilePath(path).withMaxKeyCount(2000).withFileContext(meta).build();</span> |
| <span class="source-line-no">1267</span><span id="line-1267"> writeLargeStoreFile(writer, Bytes.toBytes(name.getMethodName()),</span> |
| <span class="source-line-no">1268</span><span id="line-1268"> Bytes.toBytes(name.getMethodName()), 200);</span> |
| <span class="source-line-no">1269</span><span id="line-1269"> writer.close();</span> |
| <span class="source-line-no">1270</span><span id="line-1270"> HStoreFile storeFile =</span> |
| <span class="source-line-no">1271</span><span id="line-1271"> new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true);</span> |
| <span class="source-line-no">1272</span><span id="line-1272"> storeFile.initReader();</span> |
| <span class="source-line-no">1273</span><span id="line-1273"> HFile.Reader fReader =</span> |
| <span class="source-line-no">1274</span><span id="line-1274"> HFile.createReader(fs, writer.getPath(), storeFile.getCacheConf(), true, conf);</span> |
| <span class="source-line-no">1275</span><span id="line-1275"> FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, writer.getPath());</span> |
| <span class="source-line-no">1276</span><span id="line-1276"> long fileSize = fs.getFileStatus(writer.getPath()).getLen();</span> |
| <span class="source-line-no">1277</span><span id="line-1277"> FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize);</span> |
| <span class="source-line-no">1278</span><span id="line-1278"> long offset = trailer.getFirstDataBlockOffset(), max = trailer.getLastDataBlockOffset();</span> |
| <span class="source-line-no">1279</span><span id="line-1279"> HFileBlock block;</span> |
| <span class="source-line-no">1280</span><span id="line-1280"> int blockCount = 0;</span> |
| <span class="source-line-no">1281</span><span id="line-1281"> while (offset <= max) {</span> |
| <span class="source-line-no">1282</span><span id="line-1282"> block = fReader.readBlock(offset, -1, /* cacheBlock */ false, /* pread */ false,</span> |
| <span class="source-line-no">1283</span><span id="line-1283"> /* isCompaction */ false, /* updateCacheMetrics */ false, null, null);</span> |
| <span class="source-line-no">1284</span><span id="line-1284"> offset += block.getOnDiskSizeWithHeader();</span> |
| <span class="source-line-no">1285</span><span id="line-1285"> blockCount++;</span> |
| <span class="source-line-no">1286</span><span id="line-1286"> assertTrue(validation.apply(block.getUncompressedSizeWithoutHeader(), blockCount));</span> |
| <span class="source-line-no">1287</span><span id="line-1287"> }</span> |
| <span class="source-line-no">1288</span><span id="line-1288"> assertEquals(expectedBlockCount, blockCount);</span> |
| <span class="source-line-no">1289</span><span id="line-1289"> }</span> |
| <span class="source-line-no">1290</span><span id="line-1290"></span> |
| <span class="source-line-no">1291</span><span id="line-1291">}</span> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </pre> |
| </div> |
| </main> |
| </body> |
| </html> |
