| <!DOCTYPE HTML> |
| <html lang="en"> |
| <head> |
| <!-- Generated by javadoc (17) --> |
| <title>Source code</title> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <meta name="description" content="source: package: org.apache.hadoop.hbase.mapreduce, class: TestHashTable"> |
| <meta name="generator" content="javadoc/SourceToHTMLConverter"> |
| <link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style"> |
| </head> |
| <body class="source-page"> |
| <main role="main"> |
| <div class="source-container"> |
| <pre><span class="source-line-no">001</span><span id="line-1">/*</span> |
| <span class="source-line-no">002</span><span id="line-2"> * Licensed to the Apache Software Foundation (ASF) under one</span> |
| <span class="source-line-no">003</span><span id="line-3"> * or more contributor license agreements. See the NOTICE file</span> |
| <span class="source-line-no">004</span><span id="line-4"> * distributed with this work for additional information</span> |
| <span class="source-line-no">005</span><span id="line-5"> * regarding copyright ownership. The ASF licenses this file</span> |
| <span class="source-line-no">006</span><span id="line-6"> * to you under the Apache License, Version 2.0 (the</span> |
| <span class="source-line-no">007</span><span id="line-7"> * "License"); you may not use this file except in compliance</span> |
| <span class="source-line-no">008</span><span id="line-8"> * with the License. You may obtain a copy of the License at</span> |
| <span class="source-line-no">009</span><span id="line-9"> *</span> |
| <span class="source-line-no">010</span><span id="line-10"> * http://www.apache.org/licenses/LICENSE-2.0</span> |
| <span class="source-line-no">011</span><span id="line-11"> *</span> |
| <span class="source-line-no">012</span><span id="line-12"> * Unless required by applicable law or agreed to in writing, software</span> |
| <span class="source-line-no">013</span><span id="line-13"> * distributed under the License is distributed on an "AS IS" BASIS,</span> |
| <span class="source-line-no">014</span><span id="line-14"> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> |
| <span class="source-line-no">015</span><span id="line-15"> * See the License for the specific language governing permissions and</span> |
| <span class="source-line-no">016</span><span id="line-16"> * limitations under the License.</span> |
| <span class="source-line-no">017</span><span id="line-17"> */</span> |
| <span class="source-line-no">018</span><span id="line-18">package org.apache.hadoop.hbase.mapreduce;</span> |
| <span class="source-line-no">019</span><span id="line-19"></span> |
| <span class="source-line-no">020</span><span id="line-20">import static org.junit.Assert.assertEquals;</span> |
| <span class="source-line-no">021</span><span id="line-21"></span> |
| <span class="source-line-no">022</span><span id="line-22">import java.util.HashMap;</span> |
| <span class="source-line-no">023</span><span id="line-23">import java.util.Map;</span> |
| <span class="source-line-no">024</span><span id="line-24">import org.apache.hadoop.fs.FileStatus;</span> |
| <span class="source-line-no">025</span><span id="line-25">import org.apache.hadoop.fs.FileSystem;</span> |
| <span class="source-line-no">026</span><span id="line-26">import org.apache.hadoop.fs.Path;</span> |
| <span class="source-line-no">027</span><span id="line-27">import org.apache.hadoop.hbase.HBaseClassTestRule;</span> |
| <span class="source-line-no">028</span><span id="line-28">import org.apache.hadoop.hbase.HBaseTestingUtil;</span> |
| <span class="source-line-no">029</span><span id="line-29">import org.apache.hadoop.hbase.TableName;</span> |
| <span class="source-line-no">030</span><span id="line-30">import org.apache.hadoop.hbase.client.Put;</span> |
| <span class="source-line-no">031</span><span id="line-31">import org.apache.hadoop.hbase.client.Table;</span> |
| <span class="source-line-no">032</span><span id="line-32">import org.apache.hadoop.hbase.io.ImmutableBytesWritable;</span> |
| <span class="source-line-no">033</span><span id="line-33">import org.apache.hadoop.hbase.testclassification.LargeTests;</span> |
| <span class="source-line-no">034</span><span id="line-34">import org.apache.hadoop.hbase.util.Bytes;</span> |
| <span class="source-line-no">035</span><span id="line-35">import org.apache.hadoop.io.MapFile;</span> |
| <span class="source-line-no">036</span><span id="line-36">import org.junit.AfterClass;</span> |
| <span class="source-line-no">037</span><span id="line-37">import org.junit.Assert;</span> |
| <span class="source-line-no">038</span><span id="line-38">import org.junit.BeforeClass;</span> |
| <span class="source-line-no">039</span><span id="line-39">import org.junit.ClassRule;</span> |
| <span class="source-line-no">040</span><span id="line-40">import org.junit.Rule;</span> |
| <span class="source-line-no">041</span><span id="line-41">import org.junit.Test;</span> |
| <span class="source-line-no">042</span><span id="line-42">import org.junit.experimental.categories.Category;</span> |
| <span class="source-line-no">043</span><span id="line-43">import org.junit.rules.TestName;</span> |
| <span class="source-line-no">044</span><span id="line-44">import org.slf4j.Logger;</span> |
| <span class="source-line-no">045</span><span id="line-45">import org.slf4j.LoggerFactory;</span> |
| <span class="source-line-no">046</span><span id="line-46"></span> |
| <span class="source-line-no">047</span><span id="line-47">import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;</span> |
| <span class="source-line-no">048</span><span id="line-48">import org.apache.hbase.thirdparty.com.google.common.collect.Maps;</span> |
| <span class="source-line-no">049</span><span id="line-49"></span> |
| <span class="source-line-no">050</span><span id="line-50">/**</span> |
| <span class="source-line-no">051</span><span id="line-51"> * Basic test for the HashTable M/R tool</span> |
| <span class="source-line-no">052</span><span id="line-52"> */</span> |
| <span class="source-line-no">053</span><span id="line-53">@Category(LargeTests.class)</span> |
| <span class="source-line-no">054</span><span id="line-54">public class TestHashTable {</span> |
| <span class="source-line-no">055</span><span id="line-55"> @ClassRule</span> |
| <span class="source-line-no">056</span><span id="line-56"> public static final HBaseClassTestRule CLASS_RULE =</span> |
| <span class="source-line-no">057</span><span id="line-57"> HBaseClassTestRule.forClass(TestHashTable.class);</span> |
| <span class="source-line-no">058</span><span id="line-58"></span> |
| <span class="source-line-no">059</span><span id="line-59"> private static final Logger LOG = LoggerFactory.getLogger(TestHashTable.class);</span> |
| <span class="source-line-no">060</span><span id="line-60"></span> |
| <span class="source-line-no">061</span><span id="line-61"> private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();</span> |
| <span class="source-line-no">062</span><span id="line-62"></span> |
| <span class="source-line-no">063</span><span id="line-63"> @Rule</span> |
| <span class="source-line-no">064</span><span id="line-64"> public TestName name = new TestName();</span> |
| <span class="source-line-no">065</span><span id="line-65"></span> |
| <span class="source-line-no">066</span><span id="line-66"> @BeforeClass</span> |
| <span class="source-line-no">067</span><span id="line-67"> public static void beforeClass() throws Exception {</span> |
| <span class="source-line-no">068</span><span id="line-68"> TEST_UTIL.startMiniCluster(3);</span> |
| <span class="source-line-no">069</span><span id="line-69"> }</span> |
| <span class="source-line-no">070</span><span id="line-70"></span> |
| <span class="source-line-no">071</span><span id="line-71"> @AfterClass</span> |
| <span class="source-line-no">072</span><span id="line-72"> public static void afterClass() throws Exception {</span> |
| <span class="source-line-no">073</span><span id="line-73"> TEST_UTIL.shutdownMiniCluster();</span> |
| <span class="source-line-no">074</span><span id="line-74"> }</span> |
| <span class="source-line-no">075</span><span id="line-75"></span> |
| <span class="source-line-no">076</span><span id="line-76"> @Test</span> |
| <span class="source-line-no">077</span><span id="line-77"> public void testHashTable() throws Exception {</span> |
| <span class="source-line-no">078</span><span id="line-78"> final TableName tableName = TableName.valueOf(name.getMethodName());</span> |
| <span class="source-line-no">079</span><span id="line-79"> final byte[] family = Bytes.toBytes("family");</span> |
| <span class="source-line-no">080</span><span id="line-80"> final byte[] column1 = Bytes.toBytes("c1");</span> |
| <span class="source-line-no">081</span><span id="line-81"> final byte[] column2 = Bytes.toBytes("c2");</span> |
| <span class="source-line-no">082</span><span id="line-82"> final byte[] column3 = Bytes.toBytes("c3");</span> |
| <span class="source-line-no">083</span><span id="line-83"></span> |
| <span class="source-line-no">084</span><span id="line-84"> int numRows = 100;</span> |
| <span class="source-line-no">085</span><span id="line-85"> int numRegions = 10;</span> |
| <span class="source-line-no">086</span><span id="line-86"> int numHashFiles = 3;</span> |
| <span class="source-line-no">087</span><span id="line-87"></span> |
| <span class="source-line-no">088</span><span id="line-88"> byte[][] splitRows = new byte[numRegions - 1][];</span> |
| <span class="source-line-no">089</span><span id="line-89"> for (int i = 1; i < numRegions; i++) {</span> |
| <span class="source-line-no">090</span><span id="line-90"> splitRows[i - 1] = Bytes.toBytes(numRows * i / numRegions);</span> |
| <span class="source-line-no">091</span><span id="line-91"> }</span> |
| <span class="source-line-no">092</span><span id="line-92"></span> |
| <span class="source-line-no">093</span><span id="line-93"> long timestamp = 1430764183454L;</span> |
| <span class="source-line-no">094</span><span id="line-94"> // put rows into the first table</span> |
| <span class="source-line-no">095</span><span id="line-95"> Table t1 = TEST_UTIL.createTable(tableName, family, splitRows);</span> |
| <span class="source-line-no">096</span><span id="line-96"> for (int i = 0; i < numRows; i++) {</span> |
| <span class="source-line-no">097</span><span id="line-97"> Put p = new Put(Bytes.toBytes(i), timestamp);</span> |
| <span class="source-line-no">098</span><span id="line-98"> p.addColumn(family, column1, column1);</span> |
| <span class="source-line-no">099</span><span id="line-99"> p.addColumn(family, column2, column2);</span> |
| <span class="source-line-no">100</span><span id="line-100"> p.addColumn(family, column3, column3);</span> |
| <span class="source-line-no">101</span><span id="line-101"> t1.put(p);</span> |
| <span class="source-line-no">102</span><span id="line-102"> }</span> |
| <span class="source-line-no">103</span><span id="line-103"> t1.close();</span> |
| <span class="source-line-no">104</span><span id="line-104"></span> |
| <span class="source-line-no">105</span><span id="line-105"> HashTable hashTable = new HashTable(TEST_UTIL.getConfiguration());</span> |
| <span class="source-line-no">106</span><span id="line-106"></span> |
| <span class="source-line-no">107</span><span id="line-107"> Path testDir = TEST_UTIL.getDataTestDirOnTestFS(tableName.getNameAsString());</span> |
| <span class="source-line-no">108</span><span id="line-108"></span> |
| <span class="source-line-no">109</span><span id="line-109"> long batchSize = 300;</span> |
| <span class="source-line-no">110</span><span id="line-110"> int code =</span> |
| <span class="source-line-no">111</span><span id="line-111"> hashTable.run(new String[] { "--batchsize=" + batchSize, "--numhashfiles=" + numHashFiles,</span> |
| <span class="source-line-no">112</span><span id="line-112"> "--scanbatch=2", tableName.getNameAsString(), testDir.toString() });</span> |
| <span class="source-line-no">113</span><span id="line-113"> assertEquals("test job failed", 0, code);</span> |
| <span class="source-line-no">114</span><span id="line-114"></span> |
| <span class="source-line-no">115</span><span id="line-115"> FileSystem fs = TEST_UTIL.getTestFileSystem();</span> |
| <span class="source-line-no">116</span><span id="line-116"></span> |
| <span class="source-line-no">117</span><span id="line-117"> HashTable.TableHash tableHash = HashTable.TableHash.read(fs.getConf(), testDir);</span> |
| <span class="source-line-no">118</span><span id="line-118"> assertEquals(tableName.getNameAsString(), tableHash.tableName);</span> |
| <span class="source-line-no">119</span><span id="line-119"> assertEquals(batchSize, tableHash.batchSize);</span> |
| <span class="source-line-no">120</span><span id="line-120"> assertEquals(numHashFiles, tableHash.numHashFiles);</span> |
| <span class="source-line-no">121</span><span id="line-121"> assertEquals(numHashFiles - 1, tableHash.partitions.size());</span> |
| <span class="source-line-no">122</span><span id="line-122"> for (ImmutableBytesWritable bytes : tableHash.partitions) {</span> |
| <span class="source-line-no">123</span><span id="line-123"> LOG.debug("partition: " + Bytes.toInt(bytes.get()));</span> |
| <span class="source-line-no">124</span><span id="line-124"> }</span> |
| <span class="source-line-no">125</span><span id="line-125"></span> |
| <span class="source-line-no">126</span><span id="line-126"> ImmutableMap<Integer, ImmutableBytesWritable> expectedHashes =</span> |
| <span class="source-line-no">127</span><span id="line-127"> ImmutableMap.<Integer, ImmutableBytesWritable> builder()</span> |
| <span class="source-line-no">128</span><span id="line-128"> .put(-1, new ImmutableBytesWritable(Bytes.fromHex("714cb10a9e3b5569852980edd8c6ca2f")))</span> |
| <span class="source-line-no">129</span><span id="line-129"> .put(5, new ImmutableBytesWritable(Bytes.fromHex("28d961d9252ce8f8d44a07b38d3e1d96")))</span> |
| <span class="source-line-no">130</span><span id="line-130"> .put(10, new ImmutableBytesWritable(Bytes.fromHex("f6bbc4a224d8fd929b783a92599eaffa")))</span> |
| <span class="source-line-no">131</span><span id="line-131"> .put(15, new ImmutableBytesWritable(Bytes.fromHex("522deb5d97f73a414ecc11457be46881")))</span> |
| <span class="source-line-no">132</span><span id="line-132"> .put(20, new ImmutableBytesWritable(Bytes.fromHex("b026f2611aaa46f7110116d807545352")))</span> |
| <span class="source-line-no">133</span><span id="line-133"> .put(25, new ImmutableBytesWritable(Bytes.fromHex("39ffc1a3094aa12a2e90ffd9cef2ce93")))</span> |
| <span class="source-line-no">134</span><span id="line-134"> .put(30, new ImmutableBytesWritable(Bytes.fromHex("f6b4d75727ce9a30ac29e4f08f601666")))</span> |
| <span class="source-line-no">135</span><span id="line-135"> .put(35, new ImmutableBytesWritable(Bytes.fromHex("422e2d2f1eb79a8f02171a705a42c090")))</span> |
| <span class="source-line-no">136</span><span id="line-136"> .put(40, new ImmutableBytesWritable(Bytes.fromHex("559ad61c900fffefea0a15abf8a97bc3")))</span> |
| <span class="source-line-no">137</span><span id="line-137"> .put(45, new ImmutableBytesWritable(Bytes.fromHex("23019084513eca41cee436b2a29611cb")))</span> |
| <span class="source-line-no">138</span><span id="line-138"> .put(50, new ImmutableBytesWritable(Bytes.fromHex("b40467d222ddb4949b142fe145ee9edc")))</span> |
| <span class="source-line-no">139</span><span id="line-139"> .put(55, new ImmutableBytesWritable(Bytes.fromHex("372bf89fcd8ca4b7ab3c1add9d07f7e4")))</span> |
| <span class="source-line-no">140</span><span id="line-140"> .put(60, new ImmutableBytesWritable(Bytes.fromHex("69ae0585e6255de27dce974e332b8f8b")))</span> |
| <span class="source-line-no">141</span><span id="line-141"> .put(65, new ImmutableBytesWritable(Bytes.fromHex("8029610044297aad0abdbecd485d8e59")))</span> |
| <span class="source-line-no">142</span><span id="line-142"> .put(70, new ImmutableBytesWritable(Bytes.fromHex("de5f784f7f78987b6e57ecfd81c8646f")))</span> |
| <span class="source-line-no">143</span><span id="line-143"> .put(75, new ImmutableBytesWritable(Bytes.fromHex("1cd757cc4e1715c8c3b1c24447a1ec56")))</span> |
| <span class="source-line-no">144</span><span id="line-144"> .put(80, new ImmutableBytesWritable(Bytes.fromHex("f9a53aacfeb6142b08066615e7038095")))</span> |
| <span class="source-line-no">145</span><span id="line-145"> .put(85, new ImmutableBytesWritable(Bytes.fromHex("89b872b7e639df32d3276b33928c0c91")))</span> |
| <span class="source-line-no">146</span><span id="line-146"> .put(90, new ImmutableBytesWritable(Bytes.fromHex("45eeac0646d46a474ea0484175faed38")))</span> |
| <span class="source-line-no">147</span><span id="line-147"> .put(95, new ImmutableBytesWritable(Bytes.fromHex("f57c447e32a08f4bf1abb2892839ac56")))</span> |
| <span class="source-line-no">148</span><span id="line-148"> .build();</span> |
| <span class="source-line-no">149</span><span id="line-149"></span> |
| <span class="source-line-no">150</span><span id="line-150"> Map<Integer, ImmutableBytesWritable> actualHashes = new HashMap<>();</span> |
| <span class="source-line-no">151</span><span id="line-151"> Path dataDir = new Path(testDir, HashTable.HASH_DATA_DIR);</span> |
| <span class="source-line-no">152</span><span id="line-152"> for (int i = 0; i < numHashFiles; i++) {</span> |
| <span class="source-line-no">153</span><span id="line-153"> Path hashPath = new Path(dataDir, HashTable.TableHash.getDataFileName(i));</span> |
| <span class="source-line-no">154</span><span id="line-154"></span> |
| <span class="source-line-no">155</span><span id="line-155"> MapFile.Reader reader = new MapFile.Reader(hashPath, fs.getConf());</span> |
| <span class="source-line-no">156</span><span id="line-156"> ImmutableBytesWritable key = new ImmutableBytesWritable();</span> |
| <span class="source-line-no">157</span><span id="line-157"> ImmutableBytesWritable hash = new ImmutableBytesWritable();</span> |
| <span class="source-line-no">158</span><span id="line-158"> while (reader.next(key, hash)) {</span> |
| <span class="source-line-no">159</span><span id="line-159"> String keyString = Bytes.toHex(key.get(), key.getOffset(), key.getLength());</span> |
| <span class="source-line-no">160</span><span id="line-160"> LOG.debug("Key: " + (keyString.isEmpty() ? "-1" : Integer.parseInt(keyString, 16))</span> |
| <span class="source-line-no">161</span><span id="line-161"> + " Hash: " + Bytes.toHex(hash.get(), hash.getOffset(), hash.getLength()));</span> |
| <span class="source-line-no">162</span><span id="line-162"></span> |
| <span class="source-line-no">163</span><span id="line-163"> int intKey = -1;</span> |
| <span class="source-line-no">164</span><span id="line-164"> if (key.getLength() > 0) {</span> |
| <span class="source-line-no">165</span><span id="line-165"> intKey = Bytes.toInt(key.get(), key.getOffset(), key.getLength());</span> |
| <span class="source-line-no">166</span><span id="line-166"> }</span> |
| <span class="source-line-no">167</span><span id="line-167"> if (actualHashes.containsKey(intKey)) {</span> |
| <span class="source-line-no">168</span><span id="line-168"> Assert.fail("duplicate key in data files: " + intKey);</span> |
| <span class="source-line-no">169</span><span id="line-169"> }</span> |
| <span class="source-line-no">170</span><span id="line-170"> actualHashes.put(intKey, new ImmutableBytesWritable(hash.copyBytes()));</span> |
| <span class="source-line-no">171</span><span id="line-171"> }</span> |
| <span class="source-line-no">172</span><span id="line-172"> reader.close();</span> |
| <span class="source-line-no">173</span><span id="line-173"> }</span> |
| <span class="source-line-no">174</span><span id="line-174"></span> |
| <span class="source-line-no">175</span><span id="line-175"> FileStatus[] files = fs.listStatus(testDir);</span> |
| <span class="source-line-no">176</span><span id="line-176"> for (FileStatus file : files) {</span> |
| <span class="source-line-no">177</span><span id="line-177"> LOG.debug("Output file: " + file.getPath());</span> |
| <span class="source-line-no">178</span><span id="line-178"> }</span> |
| <span class="source-line-no">179</span><span id="line-179"></span> |
| <span class="source-line-no">180</span><span id="line-180"> files = fs.listStatus(dataDir);</span> |
| <span class="source-line-no">181</span><span id="line-181"> for (FileStatus file : files) {</span> |
| <span class="source-line-no">182</span><span id="line-182"> LOG.debug("Data file: " + file.getPath());</span> |
| <span class="source-line-no">183</span><span id="line-183"> }</span> |
| <span class="source-line-no">184</span><span id="line-184"></span> |
| <span class="source-line-no">185</span><span id="line-185"> if (!expectedHashes.equals(actualHashes)) {</span> |
| <span class="source-line-no">186</span><span id="line-186"> LOG.error("Diff: " + Maps.difference(expectedHashes, actualHashes));</span> |
| <span class="source-line-no">187</span><span id="line-187"> }</span> |
| <span class="source-line-no">188</span><span id="line-188"> Assert.assertEquals(expectedHashes, actualHashes);</span> |
| <span class="source-line-no">189</span><span id="line-189"></span> |
| <span class="source-line-no">190</span><span id="line-190"> TEST_UTIL.deleteTable(tableName);</span> |
| <span class="source-line-no">191</span><span id="line-191"> TEST_UTIL.cleanupDataTestDirOnTestFS();</span> |
| <span class="source-line-no">192</span><span id="line-192"> }</span> |
| <span class="source-line-no">193</span><span id="line-193">}</span> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </pre> |
| </div> |
| </main> |
| </body> |
| </html> |
