| <!DOCTYPE HTML> |
| <html lang="en"> |
| <head> |
| <!-- Generated by javadoc (17) --> |
| <title>Source code</title> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <meta name="description" content="source: package: org.apache.hadoop.hbase.io.hfile, class: HFileBlock, interface: FSReader"> |
| <meta name="generator" content="javadoc/SourceToHTMLConverter"> |
| <link rel="stylesheet" type="text/css" href="../../../../../../../stylesheet.css" title="Style"> |
| </head> |
| <body class="source-page"> |
| <main role="main"> |
| <div class="source-container"> |
| <pre><span class="source-line-no">001</span><span id="line-1">/*</span> |
| <span class="source-line-no">002</span><span id="line-2"> * Licensed to the Apache Software Foundation (ASF) under one</span> |
| <span class="source-line-no">003</span><span id="line-3"> * or more contributor license agreements. See the NOTICE file</span> |
| <span class="source-line-no">004</span><span id="line-4"> * distributed with this work for additional information</span> |
| <span class="source-line-no">005</span><span id="line-5"> * regarding copyright ownership. The ASF licenses this file</span> |
| <span class="source-line-no">006</span><span id="line-6"> * to you under the Apache License, Version 2.0 (the</span> |
| <span class="source-line-no">007</span><span id="line-7"> * "License"); you may not use this file except in compliance</span> |
| <span class="source-line-no">008</span><span id="line-8"> * with the License. You may obtain a copy of the License at</span> |
| <span class="source-line-no">009</span><span id="line-9"> *</span> |
| <span class="source-line-no">010</span><span id="line-10"> * http://www.apache.org/licenses/LICENSE-2.0</span> |
| <span class="source-line-no">011</span><span id="line-11"> *</span> |
| <span class="source-line-no">012</span><span id="line-12"> * Unless required by applicable law or agreed to in writing, software</span> |
| <span class="source-line-no">013</span><span id="line-13"> * distributed under the License is distributed on an "AS IS" BASIS,</span> |
| <span class="source-line-no">014</span><span id="line-14"> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> |
| <span class="source-line-no">015</span><span id="line-15"> * See the License for the specific language governing permissions and</span> |
| <span class="source-line-no">016</span><span id="line-16"> * limitations under the License.</span> |
| <span class="source-line-no">017</span><span id="line-17"> */</span> |
| <span class="source-line-no">018</span><span id="line-18">package org.apache.hadoop.hbase.io.hfile;</span> |
| <span class="source-line-no">019</span><span id="line-19"></span> |
| <span class="source-line-no">020</span><span id="line-20">import static org.apache.hadoop.hbase.io.ByteBuffAllocator.HEAP;</span> |
| <span class="source-line-no">021</span><span id="line-21">import static org.apache.hadoop.hbase.io.hfile.BlockCompressedSizePredicator.BLOCK_COMPRESSED_SIZE_PREDICATOR;</span> |
| <span class="source-line-no">022</span><span id="line-22">import static org.apache.hadoop.hbase.io.hfile.trace.HFileContextAttributesBuilderConsumer.CONTEXT_KEY;</span> |
| <span class="source-line-no">023</span><span id="line-23"></span> |
| <span class="source-line-no">024</span><span id="line-24">import io.opentelemetry.api.common.Attributes;</span> |
| <span class="source-line-no">025</span><span id="line-25">import io.opentelemetry.api.common.AttributesBuilder;</span> |
| <span class="source-line-no">026</span><span id="line-26">import io.opentelemetry.api.trace.Span;</span> |
| <span class="source-line-no">027</span><span id="line-27">import io.opentelemetry.context.Context;</span> |
| <span class="source-line-no">028</span><span id="line-28">import io.opentelemetry.context.Scope;</span> |
| <span class="source-line-no">029</span><span id="line-29">import java.io.DataInputStream;</span> |
| <span class="source-line-no">030</span><span id="line-30">import java.io.DataOutput;</span> |
| <span class="source-line-no">031</span><span id="line-31">import java.io.DataOutputStream;</span> |
| <span class="source-line-no">032</span><span id="line-32">import java.io.IOException;</span> |
| <span class="source-line-no">033</span><span id="line-33">import java.nio.ByteBuffer;</span> |
| <span class="source-line-no">034</span><span id="line-34">import java.util.ArrayList;</span> |
| <span class="source-line-no">035</span><span id="line-35">import java.util.List;</span> |
| <span class="source-line-no">036</span><span id="line-36">import java.util.Optional;</span> |
| <span class="source-line-no">037</span><span id="line-37">import java.util.concurrent.atomic.AtomicReference;</span> |
| <span class="source-line-no">038</span><span id="line-38">import java.util.concurrent.locks.Lock;</span> |
| <span class="source-line-no">039</span><span id="line-39">import java.util.concurrent.locks.ReentrantLock;</span> |
| <span class="source-line-no">040</span><span id="line-40">import org.apache.hadoop.conf.Configuration;</span> |
| <span class="source-line-no">041</span><span id="line-41">import org.apache.hadoop.fs.FSDataInputStream;</span> |
| <span class="source-line-no">042</span><span id="line-42">import org.apache.hadoop.fs.FSDataOutputStream;</span> |
| <span class="source-line-no">043</span><span id="line-43">import org.apache.hadoop.hbase.ExtendedCell;</span> |
| <span class="source-line-no">044</span><span id="line-44">import org.apache.hadoop.hbase.HConstants;</span> |
| <span class="source-line-no">045</span><span id="line-45">import org.apache.hadoop.hbase.fs.HFileSystem;</span> |
| <span class="source-line-no">046</span><span id="line-46">import org.apache.hadoop.hbase.io.ByteArrayOutputStream;</span> |
| <span class="source-line-no">047</span><span id="line-47">import org.apache.hadoop.hbase.io.ByteBuffAllocator;</span> |
| <span class="source-line-no">048</span><span id="line-48">import org.apache.hadoop.hbase.io.ByteBuffInputStream;</span> |
| <span class="source-line-no">049</span><span id="line-49">import org.apache.hadoop.hbase.io.ByteBufferWriterDataOutputStream;</span> |
| <span class="source-line-no">050</span><span id="line-50">import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;</span> |
| <span class="source-line-no">051</span><span id="line-51">import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;</span> |
| <span class="source-line-no">052</span><span id="line-52">import org.apache.hadoop.hbase.io.encoding.EncodingState;</span> |
| <span class="source-line-no">053</span><span id="line-53">import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;</span> |
| <span class="source-line-no">054</span><span id="line-54">import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext;</span> |
| <span class="source-line-no">055</span><span id="line-55">import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;</span> |
| <span class="source-line-no">056</span><span id="line-56">import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;</span> |
| <span class="source-line-no">057</span><span id="line-57">import org.apache.hadoop.hbase.io.hfile.trace.HFileContextAttributesBuilderConsumer;</span> |
| <span class="source-line-no">058</span><span id="line-58">import org.apache.hadoop.hbase.io.util.BlockIOUtils;</span> |
| <span class="source-line-no">059</span><span id="line-59">import org.apache.hadoop.hbase.nio.ByteBuff;</span> |
| <span class="source-line-no">060</span><span id="line-60">import org.apache.hadoop.hbase.nio.MultiByteBuff;</span> |
| <span class="source-line-no">061</span><span id="line-61">import org.apache.hadoop.hbase.nio.SingleByteBuff;</span> |
| <span class="source-line-no">062</span><span id="line-62">import org.apache.hadoop.hbase.regionserver.ShipperListener;</span> |
| <span class="source-line-no">063</span><span id="line-63">import org.apache.hadoop.hbase.trace.HBaseSemanticAttributes.ReadType;</span> |
| <span class="source-line-no">064</span><span id="line-64">import org.apache.hadoop.hbase.util.Bytes;</span> |
| <span class="source-line-no">065</span><span id="line-65">import org.apache.hadoop.hbase.util.ChecksumType;</span> |
| <span class="source-line-no">066</span><span id="line-66">import org.apache.hadoop.hbase.util.ClassSize;</span> |
| <span class="source-line-no">067</span><span id="line-67">import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;</span> |
| <span class="source-line-no">068</span><span id="line-68">import org.apache.hadoop.util.ReflectionUtils;</span> |
| <span class="source-line-no">069</span><span id="line-69">import org.apache.yetus.audience.InterfaceAudience;</span> |
| <span class="source-line-no">070</span><span id="line-70">import org.slf4j.Logger;</span> |
| <span class="source-line-no">071</span><span id="line-71">import org.slf4j.LoggerFactory;</span> |
| <span class="source-line-no">072</span><span id="line-72"></span> |
| <span class="source-line-no">073</span><span id="line-73">import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;</span> |
| <span class="source-line-no">074</span><span id="line-74"></span> |
| <span class="source-line-no">075</span><span id="line-75">/**</span> |
| <span class="source-line-no">076</span><span id="line-76"> * Cacheable Blocks of an {@link HFile} version 2 file. Version 2 was introduced in hbase-0.92.0.</span> |
| <span class="source-line-no">077</span><span id="line-77"> * <p></span> |
| <span class="source-line-no">078</span><span id="line-78"> * Version 1 was the original file block. Version 2 was introduced when we changed the hbase file</span> |
| <span class="source-line-no">079</span><span id="line-79"> * format to support multi-level block indexes and compound bloom filters (HBASE-3857). Support for</span> |
| <span class="source-line-no">080</span><span id="line-80"> * Version 1 was removed in hbase-1.3.0.</span> |
| <span class="source-line-no">081</span><span id="line-81"> * <h3>HFileBlock: Version 2</h3> In version 2, a block is structured as follows:</span> |
| <span class="source-line-no">082</span><span id="line-82"> * <ul></span> |
| <span class="source-line-no">083</span><span id="line-83"> * <li><b>Header:</b> See Writer#putHeader() for where header is written; header total size is</span> |
| <span class="source-line-no">084</span><span id="line-84"> * HFILEBLOCK_HEADER_SIZE</span> |
| <span class="source-line-no">085</span><span id="line-85"> * <ul></span> |
| <span class="source-line-no">086</span><span id="line-86"> * <li>0. blockType: Magic record identifying the {@link BlockType} (8 bytes): e.g.</span> |
| <span class="source-line-no">087</span><span id="line-87"> * <code>DATABLK*</code></span> |
| <span class="source-line-no">088</span><span id="line-88"> * <li>1. onDiskSizeWithoutHeader: Compressed -- a.k.a 'on disk' -- block size, excluding header,</span> |
| <span class="source-line-no">089</span><span id="line-89"> * but including tailing checksum bytes (4 bytes)</span> |
| <span class="source-line-no">090</span><span id="line-90"> * <li>2. uncompressedSizeWithoutHeader: Uncompressed block size, excluding header, and excluding</span> |
| <span class="source-line-no">091</span><span id="line-91"> * checksum bytes (4 bytes)</span> |
| <span class="source-line-no">092</span><span id="line-92"> * <li>3. prevBlockOffset: The offset of the previous block of the same type (8 bytes). This is used</span> |
| <span class="source-line-no">093</span><span id="line-93"> * to navigate to the previous block without having to go to the block index</span> |
| <span class="source-line-no">094</span><span id="line-94"> * <li>4: For minorVersions &gt;=1, the ordinal describing checksum type (1 byte)</span> |
| <span class="source-line-no">095</span><span id="line-95"> * <li>5: For minorVersions &gt;=1, the number of data bytes/checksum chunk (4 bytes)</span> |
| <span class="source-line-no">096</span><span id="line-96"> * <li>6: onDiskDataSizeWithHeader: For minorVersions &gt;=1, the size of data 'on disk', including</span> |
| <span class="source-line-no">097</span><span id="line-97"> * header, excluding checksums (4 bytes)</span> |
| <span class="source-line-no">098</span><span id="line-98"> * </ul></span> |
| <span class="source-line-no">099</span><span id="line-99"> * </li></span> |
| <span class="source-line-no">100</span><span id="line-100"> * <li><b>Raw/Compressed/Encrypted/Encoded data:</b> The compression algorithm is the same for all</span> |
| <span class="source-line-no">101</span><span id="line-101"> * the blocks in an {@link HFile}. If compression is NONE, this is just raw, serialized Cells.</span> |
| <span class="source-line-no">102</span><span id="line-102"> * <li><b>Tail:</b> For minorVersions &gt;=1, a series of 4 byte checksums, one each for the number</span> |
| <span class="source-line-no">103</span><span id="line-103"> * of bytes specified by bytesPerChecksum.</span> |
| <span class="source-line-no">104</span><span id="line-104"> * </ul></span> |
| <span class="source-line-no">105</span><span id="line-105"> * <h3>Caching</h3> Caches cache whole blocks with trailing checksums if any. We then tag on some</span> |
| <span class="source-line-no">106</span><span id="line-106"> * metadata, the content of BLOCK_METADATA_SPACE which will be flag on if we are doing 'hbase'</span> |
| <span class="source-line-no">107</span><span id="line-107"> * checksums and then the offset into the file which is needed when we re-make a cache key when we</span> |
| <span class="source-line-no">108</span><span id="line-108"> * return the block to the cache as 'done'. See {@link Cacheable#serialize(ByteBuffer, boolean)} and</span> |
| <span class="source-line-no">109</span><span id="line-109"> * {@link Cacheable#getDeserializer()}.</span> |
| <span class="source-line-no">110</span><span id="line-110"> * <p></span> |
| <span class="source-line-no">111</span><span id="line-111"> * TODO: Should we cache the checksums? Down in Writer#getBlockForCaching(CacheConfig) where we make</span> |
| <span class="source-line-no">112</span><span id="line-112"> * a block to cache-on-write, there is an attempt at turning off checksums. This is not the only</span> |
| <span class="source-line-no">113</span><span id="line-113"> * place we get blocks to cache. We also will cache the raw return from an hdfs read. In this case,</span> |
| <span class="source-line-no">114</span><span id="line-114"> * the checksums may be present. If the cache is backed by something that doesn't do ECC, say an</span> |
| <span class="source-line-no">115</span><span id="line-115"> * SSD, we might want to preserve checksums. For now this is open question.</span> |
| <span class="source-line-no">116</span><span id="line-116"> * <p></span> |
| <span class="source-line-no">117</span><span id="line-117"> * TODO: Over in BucketCache, we save a block allocation by doing a custom serialization. Be sure to</span> |
| <span class="source-line-no">118</span><span id="line-118"> * change it if serialization changes in here. Could we add a method here that takes an IOEngine and</span> |
| <span class="source-line-no">119</span><span id="line-119"> * that then serializes to it rather than expose our internals over in BucketCache? IOEngine is in</span> |
| <span class="source-line-no">120</span><span id="line-120"> * the bucket subpackage. Pull it up? Then this class knows about bucketcache. Ugh.</span> |
| <span class="source-line-no">121</span><span id="line-121"> */</span> |
| <span class="source-line-no">122</span><span id="line-122">@InterfaceAudience.Private</span> |
| <span class="source-line-no">123</span><span id="line-123">public class HFileBlock implements Cacheable {</span> |
| <span class="source-line-no">124</span><span id="line-124"> private static final Logger LOG = LoggerFactory.getLogger(HFileBlock.class);</span> |
| <span class="source-line-no">125</span><span id="line-125"> public static final long FIXED_OVERHEAD = ClassSize.estimateBase(HFileBlock.class, false);</span> |
| <span class="source-line-no">126</span><span id="line-126"></span> |
| <span class="source-line-no">127</span><span id="line-127"> // Block Header fields.</span> |
| <span class="source-line-no">128</span><span id="line-128"></span> |
| <span class="source-line-no">129</span><span id="line-129"> // TODO: encapsulate Header related logic in this inner class.</span> |
| <span class="source-line-no">130</span><span id="line-130"> static class Header {</span> |
| <span class="source-line-no">131</span><span id="line-131"> // Format of header is:</span> |
| <span class="source-line-no">132</span><span id="line-132"> // 8 bytes - block magic</span> |
| <span class="source-line-no">133</span><span id="line-133"> // 4 bytes int - onDiskSizeWithoutHeader</span> |
| <span class="source-line-no">134</span><span id="line-134"> // 4 bytes int - uncompressedSizeWithoutHeader</span> |
| <span class="source-line-no">135</span><span id="line-135"> // 8 bytes long - prevBlockOffset</span> |
| <span class="source-line-no">136</span><span id="line-136"> // The following 3 are only present if header contains checksum information</span> |
| <span class="source-line-no">137</span><span id="line-137"> // 1 byte - checksum type</span> |
| <span class="source-line-no">138</span><span id="line-138"> // 4 byte int - bytes per checksum</span> |
| <span class="source-line-no">139</span><span id="line-139"> // 4 byte int - onDiskDataSizeWithHeader</span> |
| <span class="source-line-no">140</span><span id="line-140"> static int BLOCK_MAGIC_INDEX = 0;</span> |
| <span class="source-line-no">141</span><span id="line-141"> static int ON_DISK_SIZE_WITHOUT_HEADER_INDEX = 8;</span> |
| <span class="source-line-no">142</span><span id="line-142"> static int UNCOMPRESSED_SIZE_WITHOUT_HEADER_INDEX = 12;</span> |
| <span class="source-line-no">143</span><span id="line-143"> static int PREV_BLOCK_OFFSET_INDEX = 16;</span> |
| <span class="source-line-no">144</span><span id="line-144"> static int CHECKSUM_TYPE_INDEX = 24;</span> |
| <span class="source-line-no">145</span><span id="line-145"> static int BYTES_PER_CHECKSUM_INDEX = 25;</span> |
| <span class="source-line-no">146</span><span id="line-146"> static int ON_DISK_DATA_SIZE_WITH_HEADER_INDEX = 29;</span> |
| <span class="source-line-no">147</span><span id="line-147"> }</span> |
| <span class="source-line-no">148</span><span id="line-148"></span> |
| <span class="source-line-no">149</span><span id="line-149"> /** Type of block. Header field 0. */</span> |
| <span class="source-line-no">150</span><span id="line-150"> private BlockType blockType;</span> |
| <span class="source-line-no">151</span><span id="line-151"></span> |
| <span class="source-line-no">152</span><span id="line-152"> /**</span> |
| <span class="source-line-no">153</span><span id="line-153"> * Size on disk excluding header, including checksum. Header field 1.</span> |
| <span class="source-line-no">154</span><span id="line-154"> * @see Writer#putHeader(byte[], int, int, int, int)</span> |
| <span class="source-line-no">155</span><span id="line-155"> */</span> |
| <span class="source-line-no">156</span><span id="line-156"> private int onDiskSizeWithoutHeader;</span> |
| <span class="source-line-no">157</span><span id="line-157"></span> |
| <span class="source-line-no">158</span><span id="line-158"> /**</span> |
| <span class="source-line-no">159</span><span id="line-159"> * Size of pure data. Does not include header or checksums. Header field 2.</span> |
| <span class="source-line-no">160</span><span id="line-160"> * @see Writer#putHeader(byte[], int, int, int, int)</span> |
| <span class="source-line-no">161</span><span id="line-161"> */</span> |
| <span class="source-line-no">162</span><span id="line-162"> private int uncompressedSizeWithoutHeader;</span> |
| <span class="source-line-no">163</span><span id="line-163"></span> |
| <span class="source-line-no">164</span><span id="line-164"> /**</span> |
| <span class="source-line-no">165</span><span id="line-165"> * The offset of the previous block on disk. Header field 3.</span> |
| <span class="source-line-no">166</span><span id="line-166"> * @see Writer#putHeader(byte[], int, int, int, int)</span> |
| <span class="source-line-no">167</span><span id="line-167"> */</span> |
| <span class="source-line-no">168</span><span id="line-168"> private long prevBlockOffset;</span> |
| <span class="source-line-no">169</span><span id="line-169"></span> |
| <span class="source-line-no">170</span><span id="line-170"> /**</span> |
| <span class="source-line-no">171</span><span id="line-171"> * Size on disk of header + data. Excludes checksum. Header field 6, OR calculated from</span> |
| <span class="source-line-no">172</span><span id="line-172"> * {@link #onDiskSizeWithoutHeader} when using HDFS checksum.</span> |
| <span class="source-line-no">173</span><span id="line-173"> * @see Writer#putHeader(byte[], int, int, int, int)</span> |
| <span class="source-line-no">174</span><span id="line-174"> */</span> |
| <span class="source-line-no">175</span><span id="line-175"> private final int onDiskDataSizeWithHeader;</span> |
| <span class="source-line-no">176</span><span id="line-176"> // End of Block Header fields.</span> |
| <span class="source-line-no">177</span><span id="line-177"></span> |
| <span class="source-line-no">178</span><span id="line-178"> /**</span> |
| <span class="source-line-no">179</span><span id="line-179"> * The in-memory representation of the hfile block. Can be on or offheap. Can be backed by a</span> |
| <span class="source-line-no">180</span><span id="line-180"> * single ByteBuffer or by many. Make no assumptions.</span> |
| <span class="source-line-no">181</span><span id="line-181"> * <p></span> |
| <span class="source-line-no">182</span><span id="line-182"> * Be careful reading from this <code>buf</code>. Duplicate and work on the duplicate or if not,</span> |
| <span class="source-line-no">183</span><span id="line-183"> * be sure to reset position and limit else trouble down the road.</span> |
| <span class="source-line-no">184</span><span id="line-184"> * <p></span> |
| <span class="source-line-no">185</span><span id="line-185"> * TODO: Make this read-only once made.</span> |
| <span class="source-line-no">186</span><span id="line-186"> * <p></span> |
| <span class="source-line-no">187</span><span id="line-187"> * We are using the ByteBuff type. ByteBuffer is not extensible yet we need to be able to have a</span> |
| <span class="source-line-no">188</span><span id="line-188"> * ByteBuffer-like API across multiple ByteBuffers reading from a cache such as BucketCache. So,</span> |
| <span class="source-line-no">189</span><span id="line-189"> * we have this ByteBuff type. Unfortunately, it is spread all about HFileBlock. Would be good if</span> |
| <span class="source-line-no">190</span><span id="line-190"> * could be confined to cache-use only but hard-to-do.</span> |
| <span class="source-line-no">191</span><span id="line-191"> * <p></span> |
| <span class="source-line-no">192</span><span id="line-192"> * NOTE: this byteBuff including HFileBlock header and data, but excluding checksum.</span> |
| <span class="source-line-no">193</span><span id="line-193"> */</span> |
| <span class="source-line-no">194</span><span id="line-194"> private ByteBuff bufWithoutChecksum;</span> |
| <span class="source-line-no">195</span><span id="line-195"></span> |
| <span class="source-line-no">196</span><span id="line-196"> /**</span> |
| <span class="source-line-no">197</span><span id="line-197"> * Meta data that holds meta information on the hfileblock.</span> |
| <span class="source-line-no">198</span><span id="line-198"> */</span> |
| <span class="source-line-no">199</span><span id="line-199"> private final HFileContext fileContext;</span> |
| <span class="source-line-no">200</span><span id="line-200"></span> |
| <span class="source-line-no">201</span><span id="line-201"> /**</span> |
| <span class="source-line-no">202</span><span id="line-202"> * The offset of this block in the file. Populated by the reader for convenience of access. This</span> |
| <span class="source-line-no">203</span><span id="line-203"> * offset is not part of the block header.</span> |
| <span class="source-line-no">204</span><span id="line-204"> */</span> |
| <span class="source-line-no">205</span><span id="line-205"> private long offset = UNSET;</span> |
| <span class="source-line-no">206</span><span id="line-206"></span> |
| <span class="source-line-no">207</span><span id="line-207"> /**</span> |
| <span class="source-line-no">208</span><span id="line-208"> * The on-disk size of the next block, including the header and checksums if present. UNSET if</span> |
| <span class="source-line-no">209</span><span id="line-209"> * unknown. Blocks try to carry the size of the next block to read in this data member. Usually we</span> |
| <span class="source-line-no">210</span><span id="line-210"> * get block sizes from the hfile index but sometimes the index is not available: e.g. when we</span> |
| <span class="source-line-no">211</span><span id="line-211"> * read the indexes themselves (indexes are stored in blocks, we do not have an index for the</span> |
| <span class="source-line-no">212</span><span id="line-212"> * indexes). Saves seeks especially around file open when there is a flurry of reading in hfile</span> |
| <span class="source-line-no">213</span><span id="line-213"> * metadata.</span> |
| <span class="source-line-no">214</span><span id="line-214"> */</span> |
| <span class="source-line-no">215</span><span id="line-215"> private int nextBlockOnDiskSize = UNSET;</span> |
| <span class="source-line-no">216</span><span id="line-216"></span> |
| <span class="source-line-no">217</span><span id="line-217"> private ByteBuffAllocator allocator;</span> |
| <span class="source-line-no">218</span><span id="line-218"></span> |
| <span class="source-line-no">219</span><span id="line-219"> /**</span> |
| <span class="source-line-no">220</span><span id="line-220"> * On a checksum failure, do these many succeeding read requests using hdfs checksums before</span> |
| <span class="source-line-no">221</span><span id="line-221"> * auto-reenabling hbase checksum verification.</span> |
| <span class="source-line-no">222</span><span id="line-222"> */</span> |
| <span class="source-line-no">223</span><span id="line-223"> static final int CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD = 3;</span> |
| <span class="source-line-no">224</span><span id="line-224"></span> |
| <span class="source-line-no">225</span><span id="line-225"> private static int UNSET = -1;</span> |
| <span class="source-line-no">226</span><span id="line-226"> public static final boolean FILL_HEADER = true;</span> |
| <span class="source-line-no">227</span><span id="line-227"> public static final boolean DONT_FILL_HEADER = false;</span> |
| <span class="source-line-no">228</span><span id="line-228"></span> |
| <span class="source-line-no">229</span><span id="line-229"> // How to get the estimate correctly? if it is a singleBB?</span> |
| <span class="source-line-no">230</span><span id="line-230"> public static final int MULTI_BYTE_BUFFER_HEAP_SIZE =</span> |
| <span class="source-line-no">231</span><span id="line-231"> (int) ClassSize.estimateBase(MultiByteBuff.class, false);</span> |
| <span class="source-line-no">232</span><span id="line-232"></span> |
| <span class="source-line-no">233</span><span id="line-233"> /**</span> |
| <span class="source-line-no">234</span><span id="line-234"> * Space for metadata on a block that gets stored along with the block when we cache it. There are</span> |
| <span class="source-line-no">235</span><span id="line-235"> * a few bytes stuck on the end of the HFileBlock that we pull in from HDFS. 8 bytes are for the</span> |
| <span class="source-line-no">236</span><span id="line-236"> * offset of this block (long) in the file. Offset is important because is is used when we remake</span> |
| <span class="source-line-no">237</span><span id="line-237"> * the CacheKey when we return block to the cache when done. There is also a flag on whether</span> |
| <span class="source-line-no">238</span><span id="line-238"> * checksumming is being done by hbase or not. See class comment for note on uncertain state of</span> |
| <span class="source-line-no">239</span><span id="line-239"> * checksumming of blocks that come out of cache (should we or should we not?). Finally there are</span> |
| <span class="source-line-no">240</span><span id="line-240"> * 4 bytes to hold the length of the next block which can save a seek on occasion if available.</span> |
| <span class="source-line-no">241</span><span id="line-241"> * (This EXTRA info came in with original commit of the bucketcache, HBASE-7404. It was formerly</span> |
| <span class="source-line-no">242</span><span id="line-242"> * known as EXTRA_SERIALIZATION_SPACE).</span> |
| <span class="source-line-no">243</span><span id="line-243"> */</span> |
| <span class="source-line-no">244</span><span id="line-244"> public static final int BLOCK_METADATA_SPACE =</span> |
| <span class="source-line-no">245</span><span id="line-245"> Bytes.SIZEOF_BYTE + Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT;</span> |
| <span class="source-line-no">246</span><span id="line-246"></span> |
| <span class="source-line-no">247</span><span id="line-247"> /**</span> |
| <span class="source-line-no">248</span><span id="line-248"> * Each checksum value is an integer that can be stored in 4 bytes.</span> |
| <span class="source-line-no">249</span><span id="line-249"> */</span> |
| <span class="source-line-no">250</span><span id="line-250"> static final int CHECKSUM_SIZE = Bytes.SIZEOF_INT;</span> |
| <span class="source-line-no">251</span><span id="line-251"></span> |
| <span class="source-line-no">252</span><span id="line-252"> static final byte[] DUMMY_HEADER_NO_CHECKSUM =</span> |
| <span class="source-line-no">253</span><span id="line-253"> new byte[HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM];</span> |
| <span class="source-line-no">254</span><span id="line-254"></span> |
| <span class="source-line-no">255</span><span id="line-255"> /**</span> |
| <span class="source-line-no">256</span><span id="line-256"> * Used deserializing blocks from Cache. <code></span> |
| <span class="source-line-no">257</span><span id="line-257"> * ++++++++++++++</span> |
| <span class="source-line-no">258</span><span id="line-258"> * + HFileBlock +</span> |
| <span class="source-line-no">259</span><span id="line-259"> * ++++++++++++++</span> |
| <span class="source-line-no">260</span><span id="line-260"> * + Checksums + <= Optional</span> |
| <span class="source-line-no">261</span><span id="line-261"> * ++++++++++++++</span> |
| <span class="source-line-no">262</span><span id="line-262"> * + Metadata! + <= See note on BLOCK_METADATA_SPACE above.</span> |
| <span class="source-line-no">263</span><span id="line-263"> * ++++++++++++++</span> |
| <span class="source-line-no">264</span><span id="line-264"> * </code></span> |
| <span class="source-line-no">265</span><span id="line-265"> * @see #serialize(ByteBuffer, boolean)</span> |
| <span class="source-line-no">266</span><span id="line-266"> */</span> |
| <span class="source-line-no">267</span><span id="line-267"> public static final CacheableDeserializer<Cacheable> BLOCK_DESERIALIZER = new BlockDeserializer();</span> |
| <span class="source-line-no">268</span><span id="line-268"></span> |
| <span class="source-line-no">269</span><span id="line-269"> public static final class BlockDeserializer implements CacheableDeserializer<Cacheable> {</span> |
| <span class="source-line-no">270</span><span id="line-270"> private BlockDeserializer() {</span> |
| <span class="source-line-no">271</span><span id="line-271"> }</span> |
| <span class="source-line-no">272</span><span id="line-272"></span> |
| <span class="source-line-no">273</span><span id="line-273"> @Override</span> |
| <span class="source-line-no">274</span><span id="line-274"> public HFileBlock deserialize(ByteBuff buf, ByteBuffAllocator alloc) throws IOException {</span> |
| <span class="source-line-no">275</span><span id="line-275"> // The buf has the file block followed by block metadata.</span> |
| <span class="source-line-no">276</span><span id="line-276"> // Set limit to just before the BLOCK_METADATA_SPACE then rewind.</span> |
| <span class="source-line-no">277</span><span id="line-277"> buf.limit(buf.limit() - BLOCK_METADATA_SPACE).rewind();</span> |
| <span class="source-line-no">278</span><span id="line-278"> // Get a new buffer to pass the HFileBlock for it to 'own'.</span> |
| <span class="source-line-no">279</span><span id="line-279"> ByteBuff newByteBuff = buf.slice();</span> |
| <span class="source-line-no">280</span><span id="line-280"> // Read out the BLOCK_METADATA_SPACE content and shove into our HFileBlock.</span> |
| <span class="source-line-no">281</span><span id="line-281"> buf.position(buf.limit());</span> |
| <span class="source-line-no">282</span><span id="line-282"> buf.limit(buf.limit() + HFileBlock.BLOCK_METADATA_SPACE);</span> |
| <span class="source-line-no">283</span><span id="line-283"> boolean usesChecksum = buf.get() == (byte) 1;</span> |
| <span class="source-line-no">284</span><span id="line-284"> long offset = buf.getLong();</span> |
| <span class="source-line-no">285</span><span id="line-285"> int nextBlockOnDiskSize = buf.getInt();</span> |
| <span class="source-line-no">286</span><span id="line-286"> return createFromBuff(newByteBuff, usesChecksum, offset, nextBlockOnDiskSize, null, alloc);</span> |
| <span class="source-line-no">287</span><span id="line-287"> }</span> |
| <span class="source-line-no">288</span><span id="line-288"></span> |
| <span class="source-line-no">289</span><span id="line-289"> @Override</span> |
| <span class="source-line-no">290</span><span id="line-290"> public int getDeserializerIdentifier() {</span> |
| <span class="source-line-no">291</span><span id="line-291"> return DESERIALIZER_IDENTIFIER;</span> |
| <span class="source-line-no">292</span><span id="line-292"> }</span> |
| <span class="source-line-no">293</span><span id="line-293"> }</span> |
| <span class="source-line-no">294</span><span id="line-294"></span> |
| <span class="source-line-no">295</span><span id="line-295"> private static final int DESERIALIZER_IDENTIFIER;</span> |
| <span class="source-line-no">296</span><span id="line-296"> static {</span> |
| <span class="source-line-no">297</span><span id="line-297"> DESERIALIZER_IDENTIFIER =</span> |
| <span class="source-line-no">298</span><span id="line-298"> CacheableDeserializerIdManager.registerDeserializer(BLOCK_DESERIALIZER);</span> |
| <span class="source-line-no">299</span><span id="line-299"> }</span> |
| <span class="source-line-no">300</span><span id="line-300"></span> |
| <span class="source-line-no">301</span><span id="line-301"> private final int totalChecksumBytes;</span> |
| <span class="source-line-no">302</span><span id="line-302"></span> |
| <span class="source-line-no">303</span><span id="line-303"> /**</span> |
| <span class="source-line-no">304</span><span id="line-304"> * Creates a new {@link HFile} block from the given fields. This constructor is used only while</span> |
| <span class="source-line-no">305</span><span id="line-305"> * writing blocks and caching, and is sitting in a byte buffer and we want to stuff the block into</span> |
| <span class="source-line-no">306</span><span id="line-306"> * cache.</span> |
| <span class="source-line-no">307</span><span id="line-307"> * <p></span> |
| <span class="source-line-no">308</span><span id="line-308"> * TODO: The caller presumes no checksumming</span> |
| <span class="source-line-no">309</span><span id="line-309"> * <p></span> |
| <span class="source-line-no">310</span><span id="line-310"> * TODO: HFile block writer can also off-heap ?</span> |
| <span class="source-line-no">311</span><span id="line-311"> * </p></span> |
| <span class="source-line-no">312</span><span id="line-312"> * required of this block instance since going into cache; checksum already verified on underlying</span> |
| <span class="source-line-no">313</span><span id="line-313"> * block data pulled in from filesystem. Is that correct? What if cache is SSD?</span> |
| <span class="source-line-no">314</span><span id="line-314"> * @param blockType the type of this block, see {@link BlockType}</span> |
| <span class="source-line-no">315</span><span id="line-315"> * @param onDiskSizeWithoutHeader see {@link #onDiskSizeWithoutHeader}</span> |
| <span class="source-line-no">316</span><span id="line-316"> * @param uncompressedSizeWithoutHeader see {@link #uncompressedSizeWithoutHeader}</span> |
| <span class="source-line-no">317</span><span id="line-317"> * @param prevBlockOffset see {@link #prevBlockOffset}</span> |
| <span class="source-line-no">318</span><span id="line-318"> * @param buf block buffer with header</span> |
| <span class="source-line-no">319</span><span id="line-319"> * ({@link HConstants#HFILEBLOCK_HEADER_SIZE} bytes)</span> |
| <span class="source-line-no">320</span><span id="line-320"> * @param fillHeader when true, write the first 4 header fields into passed</span> |
| <span class="source-line-no">321</span><span id="line-321"> * buffer.</span> |
| <span class="source-line-no">322</span><span id="line-322"> * @param offset the file offset the block was read from</span> |
| <span class="source-line-no">323</span><span id="line-323"> * @param onDiskDataSizeWithHeader see {@link #onDiskDataSizeWithHeader}</span> |
| <span class="source-line-no">324</span><span id="line-324"> * @param fileContext HFile meta data</span> |
| <span class="source-line-no">325</span><span id="line-325"> */</span> |
| <span class="source-line-no">326</span><span id="line-326"> public HFileBlock(BlockType blockType, int onDiskSizeWithoutHeader,</span> |
| <span class="source-line-no">327</span><span id="line-327"> int uncompressedSizeWithoutHeader, long prevBlockOffset, ByteBuff buf, boolean fillHeader,</span> |
| <span class="source-line-no">328</span><span id="line-328"> long offset, int nextBlockOnDiskSize, int onDiskDataSizeWithHeader, HFileContext fileContext,</span> |
| <span class="source-line-no">329</span><span id="line-329"> ByteBuffAllocator allocator) {</span> |
| <span class="source-line-no">330</span><span id="line-330"> this.blockType = blockType;</span> |
| <span class="source-line-no">331</span><span id="line-331"> this.onDiskSizeWithoutHeader = onDiskSizeWithoutHeader;</span> |
| <span class="source-line-no">332</span><span id="line-332"> this.uncompressedSizeWithoutHeader = uncompressedSizeWithoutHeader;</span> |
| <span class="source-line-no">333</span><span id="line-333"> this.prevBlockOffset = prevBlockOffset;</span> |
| <span class="source-line-no">334</span><span id="line-334"> this.offset = offset;</span> |
| <span class="source-line-no">335</span><span id="line-335"> this.onDiskDataSizeWithHeader = onDiskDataSizeWithHeader;</span> |
| <span class="source-line-no">336</span><span id="line-336"> this.nextBlockOnDiskSize = nextBlockOnDiskSize;</span> |
| <span class="source-line-no">337</span><span id="line-337"> this.fileContext = fileContext;</span> |
| <span class="source-line-no">338</span><span id="line-338"> this.allocator = allocator;</span> |
| <span class="source-line-no">339</span><span id="line-339"> this.bufWithoutChecksum = buf;</span> |
| <span class="source-line-no">340</span><span id="line-340"> if (fillHeader) {</span> |
| <span class="source-line-no">341</span><span id="line-341"> overwriteHeader();</span> |
| <span class="source-line-no">342</span><span id="line-342"> }</span> |
| <span class="source-line-no">343</span><span id="line-343"> this.bufWithoutChecksum.rewind();</span> |
| <span class="source-line-no">344</span><span id="line-344"> this.totalChecksumBytes = computeTotalChecksumBytes();</span> |
| <span class="source-line-no">345</span><span id="line-345"> }</span> |
| <span class="source-line-no">346</span><span id="line-346"></span> |
| <span class="source-line-no">347</span><span id="line-347"> /**</span> |
| <span class="source-line-no">348</span><span id="line-348"> * Creates a block from an existing buffer starting with a header. Rewinds and takes ownership of</span> |
| <span class="source-line-no">349</span><span id="line-349"> * the buffer. By definition of rewind, ignores the buffer position, but if you slice the buffer</span> |
| <span class="source-line-no">350</span><span id="line-350"> * beforehand, it will rewind to that point.</span> |
| <span class="source-line-no">351</span><span id="line-351"> * @param buf Has header, content, and trailing checksums if present.</span> |
| <span class="source-line-no">352</span><span id="line-352"> */</span> |
| <span class="source-line-no">353</span><span id="line-353"> static HFileBlock createFromBuff(ByteBuff buf, boolean usesHBaseChecksum, final long offset,</span> |
| <span class="source-line-no">354</span><span id="line-354"> final int nextBlockOnDiskSize, HFileContext fileContext, ByteBuffAllocator allocator)</span> |
| <span class="source-line-no">355</span><span id="line-355"> throws IOException {</span> |
| <span class="source-line-no">356</span><span id="line-356"> buf.rewind();</span> |
| <span class="source-line-no">357</span><span id="line-357"> final BlockType blockType = BlockType.read(buf);</span> |
| <span class="source-line-no">358</span><span id="line-358"> final int onDiskSizeWithoutHeader = buf.getInt(Header.ON_DISK_SIZE_WITHOUT_HEADER_INDEX);</span> |
| <span class="source-line-no">359</span><span id="line-359"> final int uncompressedSizeWithoutHeader =</span> |
| <span class="source-line-no">360</span><span id="line-360"> buf.getInt(Header.UNCOMPRESSED_SIZE_WITHOUT_HEADER_INDEX);</span> |
| <span class="source-line-no">361</span><span id="line-361"> final long prevBlockOffset = buf.getLong(Header.PREV_BLOCK_OFFSET_INDEX);</span> |
| <span class="source-line-no">362</span><span id="line-362"> // This constructor is called when we deserialize a block from cache and when we read a block in</span> |
| <span class="source-line-no">363</span><span id="line-363"> // from the fs. fileCache is null when deserialized from cache so need to make up one.</span> |
| <span class="source-line-no">364</span><span id="line-364"> HFileContextBuilder fileContextBuilder =</span> |
| <span class="source-line-no">365</span><span id="line-365"> fileContext != null ? new HFileContextBuilder(fileContext) : new HFileContextBuilder();</span> |
| <span class="source-line-no">366</span><span id="line-366"> fileContextBuilder.withHBaseCheckSum(usesHBaseChecksum);</span> |
| <span class="source-line-no">367</span><span id="line-367"> int onDiskDataSizeWithHeader;</span> |
| <span class="source-line-no">368</span><span id="line-368"> if (usesHBaseChecksum) {</span> |
| <span class="source-line-no">369</span><span id="line-369"> byte checksumType = buf.get(Header.CHECKSUM_TYPE_INDEX);</span> |
| <span class="source-line-no">370</span><span id="line-370"> int bytesPerChecksum = buf.getInt(Header.BYTES_PER_CHECKSUM_INDEX);</span> |
| <span class="source-line-no">371</span><span id="line-371"> onDiskDataSizeWithHeader = buf.getInt(Header.ON_DISK_DATA_SIZE_WITH_HEADER_INDEX);</span> |
| <span class="source-line-no">372</span><span id="line-372"> // Use the checksum type and bytes per checksum from header, not from fileContext.</span> |
| <span class="source-line-no">373</span><span id="line-373"> fileContextBuilder.withChecksumType(ChecksumType.codeToType(checksumType));</span> |
| <span class="source-line-no">374</span><span id="line-374"> fileContextBuilder.withBytesPerCheckSum(bytesPerChecksum);</span> |
| <span class="source-line-no">375</span><span id="line-375"> } else {</span> |
| <span class="source-line-no">376</span><span id="line-376"> fileContextBuilder.withChecksumType(ChecksumType.NULL);</span> |
| <span class="source-line-no">377</span><span id="line-377"> fileContextBuilder.withBytesPerCheckSum(0);</span> |
| <span class="source-line-no">378</span><span id="line-378"> // Need to fix onDiskDataSizeWithHeader; there are not checksums after-block-data</span> |
| <span class="source-line-no">379</span><span id="line-379"> onDiskDataSizeWithHeader = onDiskSizeWithoutHeader + headerSize(usesHBaseChecksum);</span> |
| <span class="source-line-no">380</span><span id="line-380"> }</span> |
| <span class="source-line-no">381</span><span id="line-381"> fileContext = fileContextBuilder.build();</span> |
| <span class="source-line-no">382</span><span id="line-382"> assert usesHBaseChecksum == fileContext.isUseHBaseChecksum();</span> |
| <span class="source-line-no">383</span><span id="line-383"> return new HFileBlockBuilder().withBlockType(blockType)</span> |
| <span class="source-line-no">384</span><span id="line-384"> .withOnDiskSizeWithoutHeader(onDiskSizeWithoutHeader)</span> |
| <span class="source-line-no">385</span><span id="line-385"> .withUncompressedSizeWithoutHeader(uncompressedSizeWithoutHeader)</span> |
| <span class="source-line-no">386</span><span id="line-386"> .withPrevBlockOffset(prevBlockOffset).withOffset(offset)</span> |
| <span class="source-line-no">387</span><span id="line-387"> .withOnDiskDataSizeWithHeader(onDiskDataSizeWithHeader)</span> |
| <span class="source-line-no">388</span><span id="line-388"> .withNextBlockOnDiskSize(nextBlockOnDiskSize).withHFileContext(fileContext)</span> |
| <span class="source-line-no">389</span><span id="line-389"> .withByteBuffAllocator(allocator).withByteBuff(buf.rewind()).withShared(!buf.hasArray())</span> |
| <span class="source-line-no">390</span><span id="line-390"> .build();</span> |
| <span class="source-line-no">391</span><span id="line-391"> }</span> |
| <span class="source-line-no">392</span><span id="line-392"></span> |
| <span class="source-line-no">393</span><span id="line-393"> /**</span> |
| <span class="source-line-no">394</span><span id="line-394"> * Parse total on disk size including header and checksum.</span> |
| <span class="source-line-no">395</span><span id="line-395"> * @param headerBuf Header ByteBuffer. Presumed exact size of header.</span> |
| <span class="source-line-no">396</span><span id="line-396"> * @param checksumSupport true if checksum verification is in use.</span> |
| <span class="source-line-no">397</span><span id="line-397"> * @return Size of the block with header included.</span> |
| <span class="source-line-no">398</span><span id="line-398"> */</span> |
| <span class="source-line-no">399</span><span id="line-399"> private static int getOnDiskSizeWithHeader(final ByteBuff headerBuf, boolean checksumSupport) {</span> |
| <span class="source-line-no">400</span><span id="line-400"> return headerBuf.getInt(Header.ON_DISK_SIZE_WITHOUT_HEADER_INDEX) + headerSize(checksumSupport);</span> |
| <span class="source-line-no">401</span><span id="line-401"> }</span> |
| <span class="source-line-no">402</span><span id="line-402"></span> |
| <span class="source-line-no">403</span><span id="line-403"> /**</span> |
| <span class="source-line-no">404</span><span id="line-404"> * @return the on-disk size of the next block (including the header size and any checksums if</span> |
| <span class="source-line-no">405</span><span id="line-405"> * present) read by peeking into the next block's header; use as a hint when doing a read</span> |
| <span class="source-line-no">406</span><span id="line-406"> * of the next block when scanning or running over a file.</span> |
| <span class="source-line-no">407</span><span id="line-407"> */</span> |
| <span class="source-line-no">408</span><span id="line-408"> int getNextBlockOnDiskSize() {</span> |
| <span class="source-line-no">409</span><span id="line-409"> return nextBlockOnDiskSize;</span> |
| <span class="source-line-no">410</span><span id="line-410"> }</span> |
| <span class="source-line-no">411</span><span id="line-411"></span> |
| <span class="source-line-no">412</span><span id="line-412"> @Override</span> |
| <span class="source-line-no">413</span><span id="line-413"> public BlockType getBlockType() {</span> |
| <span class="source-line-no">414</span><span id="line-414"> return blockType;</span> |
| <span class="source-line-no">415</span><span id="line-415"> }</span> |
| <span class="source-line-no">416</span><span id="line-416"></span> |
| <span class="source-line-no">417</span><span id="line-417"> @Override</span> |
| <span class="source-line-no">418</span><span id="line-418"> public int refCnt() {</span> |
| <span class="source-line-no">419</span><span id="line-419"> return bufWithoutChecksum.refCnt();</span> |
| <span class="source-line-no">420</span><span id="line-420"> }</span> |
| <span class="source-line-no">421</span><span id="line-421"></span> |
| <span class="source-line-no">422</span><span id="line-422"> @Override</span> |
| <span class="source-line-no">423</span><span id="line-423"> public HFileBlock retain() {</span> |
| <span class="source-line-no">424</span><span id="line-424"> bufWithoutChecksum.retain();</span> |
| <span class="source-line-no">425</span><span id="line-425"> return this;</span> |
| <span class="source-line-no">426</span><span id="line-426"> }</span> |
| <span class="source-line-no">427</span><span id="line-427"></span> |
| <span class="source-line-no">428</span><span id="line-428"> /**</span> |
| <span class="source-line-no">429</span><span id="line-429"> * Call {@link ByteBuff#release()} to decrease the reference count, if no other reference, it will</span> |
| <span class="source-line-no">430</span><span id="line-430"> * return back the {@link ByteBuffer} to {@link org.apache.hadoop.hbase.io.ByteBuffAllocator}</span> |
| <span class="source-line-no">431</span><span id="line-431"> */</span> |
| <span class="source-line-no">432</span><span id="line-432"> @Override</span> |
| <span class="source-line-no">433</span><span id="line-433"> public boolean release() {</span> |
| <span class="source-line-no">434</span><span id="line-434"> return bufWithoutChecksum.release();</span> |
| <span class="source-line-no">435</span><span id="line-435"> }</span> |
| <span class="source-line-no">436</span><span id="line-436"></span> |
| <span class="source-line-no">437</span><span id="line-437"> /**</span> |
| <span class="source-line-no">438</span><span id="line-438"> * Calling this method in strategic locations where HFileBlocks are referenced may help diagnose</span> |
| <span class="source-line-no">439</span><span id="line-439"> * potential buffer leaks. We pass the block itself as a default hint, but one can use</span> |
| <span class="source-line-no">440</span><span id="line-440"> * {@link #touch(Object)} to pass their own hint as well.</span> |
| <span class="source-line-no">441</span><span id="line-441"> */</span> |
| <span class="source-line-no">442</span><span id="line-442"> @Override</span> |
| <span class="source-line-no">443</span><span id="line-443"> public HFileBlock touch() {</span> |
| <span class="source-line-no">444</span><span id="line-444"> return touch(this);</span> |
| <span class="source-line-no">445</span><span id="line-445"> }</span> |
| <span class="source-line-no">446</span><span id="line-446"></span> |
| <span class="source-line-no">447</span><span id="line-447"> @Override</span> |
| <span class="source-line-no">448</span><span id="line-448"> public HFileBlock touch(Object hint) {</span> |
| <span class="source-line-no">449</span><span id="line-449"> bufWithoutChecksum.touch(hint);</span> |
| <span class="source-line-no">450</span><span id="line-450"> return this;</span> |
| <span class="source-line-no">451</span><span id="line-451"> }</span> |
| <span class="source-line-no">452</span><span id="line-452"></span> |
| <span class="source-line-no">453</span><span id="line-453"> /** Returns get data block encoding id that was used to encode this block */</span> |
| <span class="source-line-no">454</span><span id="line-454"> short getDataBlockEncodingId() {</span> |
| <span class="source-line-no">455</span><span id="line-455"> if (blockType != BlockType.ENCODED_DATA) {</span> |
| <span class="source-line-no">456</span><span id="line-456"> throw new IllegalArgumentException("Querying encoder ID of a block " + "of type other than "</span> |
| <span class="source-line-no">457</span><span id="line-457"> + BlockType.ENCODED_DATA + ": " + blockType);</span> |
| <span class="source-line-no">458</span><span id="line-458"> }</span> |
| <span class="source-line-no">459</span><span id="line-459"> return bufWithoutChecksum.getShort(headerSize());</span> |
| <span class="source-line-no">460</span><span id="line-460"> }</span> |
| <span class="source-line-no">461</span><span id="line-461"></span> |
| <span class="source-line-no">462</span><span id="line-462"> /** Returns the on-disk size of header + data part + checksum. */</span> |
| <span class="source-line-no">463</span><span id="line-463"> public int getOnDiskSizeWithHeader() {</span> |
| <span class="source-line-no">464</span><span id="line-464"> return onDiskSizeWithoutHeader + headerSize();</span> |
| <span class="source-line-no">465</span><span id="line-465"> }</span> |
| <span class="source-line-no">466</span><span id="line-466"></span> |
| <span class="source-line-no">467</span><span id="line-467"> /** Returns the on-disk size of the data part + checksum (header excluded). */</span> |
| <span class="source-line-no">468</span><span id="line-468"> int getOnDiskSizeWithoutHeader() {</span> |
| <span class="source-line-no">469</span><span id="line-469"> return onDiskSizeWithoutHeader;</span> |
| <span class="source-line-no">470</span><span id="line-470"> }</span> |
| <span class="source-line-no">471</span><span id="line-471"></span> |
| <span class="source-line-no">472</span><span id="line-472"> /** Returns the uncompressed size of data part (header and checksum excluded). */</span> |
| <span class="source-line-no">473</span><span id="line-473"> public int getUncompressedSizeWithoutHeader() {</span> |
| <span class="source-line-no">474</span><span id="line-474"> return uncompressedSizeWithoutHeader;</span> |
| <span class="source-line-no">475</span><span id="line-475"> }</span> |
| <span class="source-line-no">476</span><span id="line-476"></span> |
| <span class="source-line-no">477</span><span id="line-477"> /** Returns the offset of the previous block of the same type in the file, or -1 if unknown */</span> |
| <span class="source-line-no">478</span><span id="line-478"> long getPrevBlockOffset() {</span> |
| <span class="source-line-no">479</span><span id="line-479"> return prevBlockOffset;</span> |
| <span class="source-line-no">480</span><span id="line-480"> }</span> |
| <span class="source-line-no">481</span><span id="line-481"></span> |
| <span class="source-line-no">482</span><span id="line-482"> /**</span> |
| <span class="source-line-no">483</span><span id="line-483"> * Rewinds {@code buf} and writes first 4 header fields. {@code buf} position is modified as</span> |
| <span class="source-line-no">484</span><span id="line-484"> * side-effect.</span> |
| <span class="source-line-no">485</span><span id="line-485"> */</span> |
| <span class="source-line-no">486</span><span id="line-486"> private void overwriteHeader() {</span> |
| <span class="source-line-no">487</span><span id="line-487"> bufWithoutChecksum.rewind();</span> |
| <span class="source-line-no">488</span><span id="line-488"> blockType.write(bufWithoutChecksum);</span> |
| <span class="source-line-no">489</span><span id="line-489"> bufWithoutChecksum.putInt(onDiskSizeWithoutHeader);</span> |
| <span class="source-line-no">490</span><span id="line-490"> bufWithoutChecksum.putInt(uncompressedSizeWithoutHeader);</span> |
| <span class="source-line-no">491</span><span id="line-491"> bufWithoutChecksum.putLong(prevBlockOffset);</span> |
| <span class="source-line-no">492</span><span id="line-492"> if (this.fileContext.isUseHBaseChecksum()) {</span> |
| <span class="source-line-no">493</span><span id="line-493"> bufWithoutChecksum.put(fileContext.getChecksumType().getCode());</span> |
| <span class="source-line-no">494</span><span id="line-494"> bufWithoutChecksum.putInt(fileContext.getBytesPerChecksum());</span> |
| <span class="source-line-no">495</span><span id="line-495"> bufWithoutChecksum.putInt(onDiskDataSizeWithHeader);</span> |
| <span class="source-line-no">496</span><span id="line-496"> }</span> |
| <span class="source-line-no">497</span><span id="line-497"> }</span> |
| <span class="source-line-no">498</span><span id="line-498"></span> |
| <span class="source-line-no">499</span><span id="line-499"> /**</span> |
| <span class="source-line-no">500</span><span id="line-500"> * Returns a buffer that does not include the header and checksum.</span> |
| <span class="source-line-no">501</span><span id="line-501"> * @return the buffer with header skipped and checksum omitted.</span> |
| <span class="source-line-no">502</span><span id="line-502"> */</span> |
| <span class="source-line-no">503</span><span id="line-503"> public ByteBuff getBufferWithoutHeader() {</span> |
| <span class="source-line-no">504</span><span id="line-504"> ByteBuff dup = getBufferReadOnly();</span> |
| <span class="source-line-no">505</span><span id="line-505"> return dup.position(headerSize()).slice();</span> |
| <span class="source-line-no">506</span><span id="line-506"> }</span> |
| <span class="source-line-no">507</span><span id="line-507"></span> |
| <span class="source-line-no">508</span><span id="line-508"> /**</span> |
| <span class="source-line-no">509</span><span id="line-509"> * Returns a read-only duplicate of the buffer this block stores internally ready to be read.</span> |
| <span class="source-line-no">510</span><span id="line-510"> * Clients must not modify the buffer object though they may set position and limit on the</span> |
| <span class="source-line-no">511</span><span id="line-511"> * returned buffer since we pass back a duplicate. This method has to be public because it is used</span> |
| <span class="source-line-no">512</span><span id="line-512"> * in {@link CompoundBloomFilter} to avoid object creation on every Bloom filter lookup, but has</span> |
| <span class="source-line-no">513</span><span id="line-513"> * to be used with caution. Buffer holds header, block content, and any follow-on checksums if</span> |
| <span class="source-line-no">514</span><span id="line-514"> * present.</span> |
| <span class="source-line-no">515</span><span id="line-515"> * @return the buffer of this block for read-only operations,the buffer includes header,but not</span> |
| <span class="source-line-no">516</span><span id="line-516"> * checksum.</span> |
| <span class="source-line-no">517</span><span id="line-517"> */</span> |
| <span class="source-line-no">518</span><span id="line-518"> public ByteBuff getBufferReadOnly() {</span> |
| <span class="source-line-no">519</span><span id="line-519"> // TODO: ByteBuf does not support asReadOnlyBuffer(). Fix.</span> |
| <span class="source-line-no">520</span><span id="line-520"> ByteBuff dup = this.bufWithoutChecksum.duplicate();</span> |
| <span class="source-line-no">521</span><span id="line-521"> assert dup.position() == 0;</span> |
| <span class="source-line-no">522</span><span id="line-522"> return dup;</span> |
| <span class="source-line-no">523</span><span id="line-523"> }</span> |
| <span class="source-line-no">524</span><span id="line-524"></span> |
| <span class="source-line-no">525</span><span id="line-525"> public ByteBuffAllocator getByteBuffAllocator() {</span> |
| <span class="source-line-no">526</span><span id="line-526"> return this.allocator;</span> |
| <span class="source-line-no">527</span><span id="line-527"> }</span> |
| <span class="source-line-no">528</span><span id="line-528"></span> |
| <span class="source-line-no">529</span><span id="line-529"> private void sanityCheckAssertion(long valueFromBuf, long valueFromField, String fieldName)</span> |
| <span class="source-line-no">530</span><span id="line-530"> throws IOException {</span> |
| <span class="source-line-no">531</span><span id="line-531"> if (valueFromBuf != valueFromField) {</span> |
| <span class="source-line-no">532</span><span id="line-532"> throw new AssertionError(fieldName + " in the buffer (" + valueFromBuf</span> |
| <span class="source-line-no">533</span><span id="line-533"> + ") is different from that in the field (" + valueFromField + ")");</span> |
| <span class="source-line-no">534</span><span id="line-534"> }</span> |
| <span class="source-line-no">535</span><span id="line-535"> }</span> |
| <span class="source-line-no">536</span><span id="line-536"></span> |
| <span class="source-line-no">537</span><span id="line-537"> private void sanityCheckAssertion(BlockType valueFromBuf, BlockType valueFromField)</span> |
| <span class="source-line-no">538</span><span id="line-538"> throws IOException {</span> |
| <span class="source-line-no">539</span><span id="line-539"> if (valueFromBuf != valueFromField) {</span> |
| <span class="source-line-no">540</span><span id="line-540"> throw new IOException("Block type stored in the buffer: " + valueFromBuf</span> |
| <span class="source-line-no">541</span><span id="line-541"> + ", block type field: " + valueFromField);</span> |
| <span class="source-line-no">542</span><span id="line-542"> }</span> |
| <span class="source-line-no">543</span><span id="line-543"> }</span> |
| <span class="source-line-no">544</span><span id="line-544"></span> |
| <span class="source-line-no">545</span><span id="line-545"> /**</span> |
| <span class="source-line-no">546</span><span id="line-546"> * Checks if the block is internally consistent, i.e. the first</span> |
| <span class="source-line-no">547</span><span id="line-547"> * {@link HConstants#HFILEBLOCK_HEADER_SIZE} bytes of the buffer contain a valid header consistent</span> |
| <span class="source-line-no">548</span><span id="line-548"> * with the fields. Assumes a packed block structure. This function is primary for testing and</span> |
| <span class="source-line-no">549</span><span id="line-549"> * debugging, and is not thread-safe, because it alters the internal buffer pointer. Used by tests</span> |
| <span class="source-line-no">550</span><span id="line-550"> * only.</span> |
| <span class="source-line-no">551</span><span id="line-551"> */</span> |
| <span class="source-line-no">552</span><span id="line-552"> void sanityCheck() throws IOException {</span> |
| <span class="source-line-no">553</span><span id="line-553"> // Duplicate so no side-effects</span> |
| <span class="source-line-no">554</span><span id="line-554"> ByteBuff dup = this.bufWithoutChecksum.duplicate().rewind();</span> |
| <span class="source-line-no">555</span><span id="line-555"> sanityCheckAssertion(BlockType.read(dup), blockType);</span> |
| <span class="source-line-no">556</span><span id="line-556"></span> |
| <span class="source-line-no">557</span><span id="line-557"> sanityCheckAssertion(dup.getInt(), onDiskSizeWithoutHeader, "onDiskSizeWithoutHeader");</span> |
| <span class="source-line-no">558</span><span id="line-558"></span> |
| <span class="source-line-no">559</span><span id="line-559"> sanityCheckAssertion(dup.getInt(), uncompressedSizeWithoutHeader,</span> |
| <span class="source-line-no">560</span><span id="line-560"> "uncompressedSizeWithoutHeader");</span> |
| <span class="source-line-no">561</span><span id="line-561"></span> |
| <span class="source-line-no">562</span><span id="line-562"> sanityCheckAssertion(dup.getLong(), prevBlockOffset, "prevBlockOffset");</span> |
| <span class="source-line-no">563</span><span id="line-563"> if (this.fileContext.isUseHBaseChecksum()) {</span> |
| <span class="source-line-no">564</span><span id="line-564"> sanityCheckAssertion(dup.get(), this.fileContext.getChecksumType().getCode(), "checksumType");</span> |
| <span class="source-line-no">565</span><span id="line-565"> sanityCheckAssertion(dup.getInt(), this.fileContext.getBytesPerChecksum(),</span> |
| <span class="source-line-no">566</span><span id="line-566"> "bytesPerChecksum");</span> |
| <span class="source-line-no">567</span><span id="line-567"> sanityCheckAssertion(dup.getInt(), onDiskDataSizeWithHeader, "onDiskDataSizeWithHeader");</span> |
| <span class="source-line-no">568</span><span id="line-568"> }</span> |
| <span class="source-line-no">569</span><span id="line-569"></span> |
| <span class="source-line-no">570</span><span id="line-570"> if (dup.limit() != onDiskDataSizeWithHeader) {</span> |
| <span class="source-line-no">571</span><span id="line-571"> throw new AssertionError(</span> |
| <span class="source-line-no">572</span><span id="line-572"> "Expected limit " + onDiskDataSizeWithHeader + ", got " + dup.limit());</span> |
| <span class="source-line-no">573</span><span id="line-573"> }</span> |
| <span class="source-line-no">574</span><span id="line-574"></span> |
| <span class="source-line-no">575</span><span id="line-575"> // We might optionally allocate HFILEBLOCK_HEADER_SIZE more bytes to read the next</span> |
| <span class="source-line-no">576</span><span id="line-576"> // block's header, so there are two sensible values for buffer capacity.</span> |
| <span class="source-line-no">577</span><span id="line-577"> int hdrSize = headerSize();</span> |
| <span class="source-line-no">578</span><span id="line-578"> dup.rewind();</span> |
| <span class="source-line-no">579</span><span id="line-579"> if (</span> |
| <span class="source-line-no">580</span><span id="line-580"> dup.remaining() != onDiskDataSizeWithHeader</span> |
| <span class="source-line-no">581</span><span id="line-581"> && dup.remaining() != onDiskDataSizeWithHeader + hdrSize</span> |
| <span class="source-line-no">582</span><span id="line-582"> ) {</span> |
| <span class="source-line-no">583</span><span id="line-583"> throw new AssertionError("Invalid buffer capacity: " + dup.remaining() + ", expected "</span> |
| <span class="source-line-no">584</span><span id="line-584"> + onDiskDataSizeWithHeader + " or " + (onDiskDataSizeWithHeader + hdrSize));</span> |
| <span class="source-line-no">585</span><span id="line-585"> }</span> |
| <span class="source-line-no">586</span><span id="line-586"> }</span> |
| <span class="source-line-no">587</span><span id="line-587"></span> |
| <span class="source-line-no">588</span><span id="line-588"> @Override</span> |
| <span class="source-line-no">589</span><span id="line-589"> public String toString() {</span> |
| <span class="source-line-no">590</span><span id="line-590"> StringBuilder sb = new StringBuilder().append("[").append("blockType=").append(blockType)</span> |
| <span class="source-line-no">591</span><span id="line-591"> .append(", fileOffset=").append(offset).append(", headerSize=").append(headerSize())</span> |
| <span class="source-line-no">592</span><span id="line-592"> .append(", onDiskSizeWithoutHeader=").append(onDiskSizeWithoutHeader)</span> |
| <span class="source-line-no">593</span><span id="line-593"> .append(", uncompressedSizeWithoutHeader=").append(uncompressedSizeWithoutHeader)</span> |
| <span class="source-line-no">594</span><span id="line-594"> .append(", prevBlockOffset=").append(prevBlockOffset).append(", isUseHBaseChecksum=")</span> |
| <span class="source-line-no">595</span><span id="line-595"> .append(fileContext.isUseHBaseChecksum());</span> |
| <span class="source-line-no">596</span><span id="line-596"> if (fileContext.isUseHBaseChecksum()) {</span> |
| <span class="source-line-no">597</span><span id="line-597"> sb.append(", checksumType=").append(ChecksumType.codeToType(this.bufWithoutChecksum.get(24)))</span> |
| <span class="source-line-no">598</span><span id="line-598"> .append(", bytesPerChecksum=").append(this.bufWithoutChecksum.getInt(24 + 1))</span> |
| <span class="source-line-no">599</span><span id="line-599"> .append(", onDiskDataSizeWithHeader=").append(onDiskDataSizeWithHeader);</span> |
| <span class="source-line-no">600</span><span id="line-600"> } else {</span> |
| <span class="source-line-no">601</span><span id="line-601"> sb.append(", onDiskDataSizeWithHeader=").append(onDiskDataSizeWithHeader).append("(")</span> |
| <span class="source-line-no">602</span><span id="line-602"> .append(onDiskSizeWithoutHeader).append("+")</span> |
| <span class="source-line-no">603</span><span id="line-603"> .append(HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM).append(")");</span> |
| <span class="source-line-no">604</span><span id="line-604"> }</span> |
| <span class="source-line-no">605</span><span id="line-605"> String dataBegin;</span> |
| <span class="source-line-no">606</span><span id="line-606"> if (bufWithoutChecksum.hasArray()) {</span> |
| <span class="source-line-no">607</span><span id="line-607"> dataBegin = Bytes.toStringBinary(bufWithoutChecksum.array(),</span> |
| <span class="source-line-no">608</span><span id="line-608"> bufWithoutChecksum.arrayOffset() + headerSize(),</span> |
| <span class="source-line-no">609</span><span id="line-609"> Math.min(32, bufWithoutChecksum.limit() - bufWithoutChecksum.arrayOffset() - headerSize()));</span> |
| <span class="source-line-no">610</span><span id="line-610"> } else {</span> |
| <span class="source-line-no">611</span><span id="line-611"> ByteBuff bufWithoutHeader = getBufferWithoutHeader();</span> |
| <span class="source-line-no">612</span><span id="line-612"> byte[] dataBeginBytes =</span> |
| <span class="source-line-no">613</span><span id="line-613"> new byte[Math.min(32, bufWithoutHeader.limit() - bufWithoutHeader.position())];</span> |
| <span class="source-line-no">614</span><span id="line-614"> bufWithoutHeader.get(dataBeginBytes);</span> |
| <span class="source-line-no">615</span><span id="line-615"> dataBegin = Bytes.toStringBinary(dataBeginBytes);</span> |
| <span class="source-line-no">616</span><span id="line-616"> }</span> |
| <span class="source-line-no">617</span><span id="line-617"> sb.append(", getOnDiskSizeWithHeader=").append(getOnDiskSizeWithHeader())</span> |
| <span class="source-line-no">618</span><span id="line-618"> .append(", totalChecksumBytes=").append(totalChecksumBytes()).append(", isUnpacked=")</span> |
| <span class="source-line-no">619</span><span id="line-619"> .append(isUnpacked()).append(", buf=[").append(bufWithoutChecksum).append("]")</span> |
| <span class="source-line-no">620</span><span id="line-620"> .append(", dataBeginsWith=").append(dataBegin).append(", fileContext=").append(fileContext)</span> |
| <span class="source-line-no">621</span><span id="line-621"> .append(", nextBlockOnDiskSize=").append(nextBlockOnDiskSize).append("]");</span> |
| <span class="source-line-no">622</span><span id="line-622"> return sb.toString();</span> |
| <span class="source-line-no">623</span><span id="line-623"> }</span> |
| <span class="source-line-no">624</span><span id="line-624"></span> |
| <span class="source-line-no">625</span><span id="line-625"> /**</span> |
| <span class="source-line-no">626</span><span id="line-626"> * Retrieves the decompressed/decrypted view of this block. An encoded block remains in its</span> |
| <span class="source-line-no">627</span><span id="line-627"> * encoded structure. Internal structures are shared between instances where applicable.</span> |
| <span class="source-line-no">628</span><span id="line-628"> */</span> |
| <span class="source-line-no">629</span><span id="line-629"> HFileBlock unpack(HFileContext fileContext, FSReader reader) throws IOException {</span> |
| <span class="source-line-no">630</span><span id="line-630"> if (!fileContext.isCompressedOrEncrypted()) {</span> |
| <span class="source-line-no">631</span><span id="line-631"> // TODO: cannot use our own fileContext here because HFileBlock(ByteBuffer, boolean),</span> |
| <span class="source-line-no">632</span><span id="line-632"> // which is used for block serialization to L2 cache, does not preserve encoding and</span> |
| <span class="source-line-no">633</span><span id="line-633"> // encryption details.</span> |
| <span class="source-line-no">634</span><span id="line-634"> return this;</span> |
| <span class="source-line-no">635</span><span id="line-635"> }</span> |
| <span class="source-line-no">636</span><span id="line-636"></span> |
| <span class="source-line-no">637</span><span id="line-637"> ByteBuff newBuf = allocateBufferForUnpacking(); // allocates space for the decompressed block</span> |
| <span class="source-line-no">638</span><span id="line-638"> HFileBlock unpacked = shallowClone(this, newBuf);</span> |
| <span class="source-line-no">639</span><span id="line-639"></span> |
| <span class="source-line-no">640</span><span id="line-640"> boolean succ = false;</span> |
| <span class="source-line-no">641</span><span id="line-641"> final Context context =</span> |
| <span class="source-line-no">642</span><span id="line-642"> Context.current().with(CONTEXT_KEY, new HFileContextAttributesBuilderConsumer(fileContext));</span> |
| <span class="source-line-no">643</span><span id="line-643"> try (Scope ignored = context.makeCurrent()) {</span> |
| <span class="source-line-no">644</span><span id="line-644"> HFileBlockDecodingContext ctx = blockType == BlockType.ENCODED_DATA</span> |
| <span class="source-line-no">645</span><span id="line-645"> ? reader.getBlockDecodingContext()</span> |
| <span class="source-line-no">646</span><span id="line-646"> : reader.getDefaultBlockDecodingContext();</span> |
| <span class="source-line-no">647</span><span id="line-647"> // Create a duplicated buffer without the header part.</span> |
| <span class="source-line-no">648</span><span id="line-648"> int headerSize = this.headerSize();</span> |
| <span class="source-line-no">649</span><span id="line-649"> ByteBuff dup = this.bufWithoutChecksum.duplicate();</span> |
| <span class="source-line-no">650</span><span id="line-650"> dup.position(headerSize);</span> |
| <span class="source-line-no">651</span><span id="line-651"> dup = dup.slice();</span> |
| <span class="source-line-no">652</span><span id="line-652"> // Decode the dup into unpacked#buf</span> |
| <span class="source-line-no">653</span><span id="line-653"> ctx.prepareDecoding(unpacked.getOnDiskDataSizeWithHeader() - headerSize,</span> |
| <span class="source-line-no">654</span><span id="line-654"> unpacked.getUncompressedSizeWithoutHeader(), unpacked.getBufferWithoutHeader(), dup);</span> |
| <span class="source-line-no">655</span><span id="line-655"> succ = true;</span> |
| <span class="source-line-no">656</span><span id="line-656"> return unpacked;</span> |
| <span class="source-line-no">657</span><span id="line-657"> } finally {</span> |
| <span class="source-line-no">658</span><span id="line-658"> if (!succ) {</span> |
| <span class="source-line-no">659</span><span id="line-659"> unpacked.release();</span> |
| <span class="source-line-no">660</span><span id="line-660"> }</span> |
| <span class="source-line-no">661</span><span id="line-661"> }</span> |
| <span class="source-line-no">662</span><span id="line-662"> }</span> |
| <span class="source-line-no">663</span><span id="line-663"></span> |
| <span class="source-line-no">664</span><span id="line-664"> /**</span> |
| <span class="source-line-no">665</span><span id="line-665"> * Always allocates a new buffer of the correct size. Copies header bytes from the existing</span> |
| <span class="source-line-no">666</span><span id="line-666"> * buffer. Does not change header fields. Reserve room to keep checksum bytes too.</span> |
| <span class="source-line-no">667</span><span id="line-667"> */</span> |
| <span class="source-line-no">668</span><span id="line-668"> private ByteBuff allocateBufferForUnpacking() {</span> |
| <span class="source-line-no">669</span><span id="line-669"> int headerSize = headerSize();</span> |
| <span class="source-line-no">670</span><span id="line-670"> int capacityNeeded = headerSize + uncompressedSizeWithoutHeader;</span> |
| <span class="source-line-no">671</span><span id="line-671"></span> |
| <span class="source-line-no">672</span><span id="line-672"> ByteBuff source = bufWithoutChecksum.duplicate();</span> |
| <span class="source-line-no">673</span><span id="line-673"> ByteBuff newBuf = allocator.allocate(capacityNeeded);</span> |
| <span class="source-line-no">674</span><span id="line-674"></span> |
| <span class="source-line-no">675</span><span id="line-675"> // Copy header bytes into newBuf.</span> |
| <span class="source-line-no">676</span><span id="line-676"> source.position(0);</span> |
| <span class="source-line-no">677</span><span id="line-677"> newBuf.put(0, source, 0, headerSize);</span> |
| <span class="source-line-no">678</span><span id="line-678"></span> |
| <span class="source-line-no">679</span><span id="line-679"> // set limit to exclude next block's header</span> |
| <span class="source-line-no">680</span><span id="line-680"> newBuf.limit(capacityNeeded);</span> |
| <span class="source-line-no">681</span><span id="line-681"> return newBuf;</span> |
| <span class="source-line-no">682</span><span id="line-682"> }</span> |
| <span class="source-line-no">683</span><span id="line-683"></span> |
| <span class="source-line-no">684</span><span id="line-684"> /**</span> |
| <span class="source-line-no">685</span><span id="line-685"> * Return true when this block's buffer has been unpacked, false otherwise. Note this is a</span> |
| <span class="source-line-no">686</span><span id="line-686"> * calculated heuristic, not tracked attribute of the block.</span> |
| <span class="source-line-no">687</span><span id="line-687"> */</span> |
| <span class="source-line-no">688</span><span id="line-688"> public boolean isUnpacked() {</span> |
| <span class="source-line-no">689</span><span id="line-689"> final int headerSize = headerSize();</span> |
| <span class="source-line-no">690</span><span id="line-690"> final int expectedCapacity = headerSize + uncompressedSizeWithoutHeader;</span> |
| <span class="source-line-no">691</span><span id="line-691"> final int bufCapacity = bufWithoutChecksum.remaining();</span> |
| <span class="source-line-no">692</span><span id="line-692"> return bufCapacity == expectedCapacity || bufCapacity == expectedCapacity + headerSize;</span> |
| <span class="source-line-no">693</span><span id="line-693"> }</span> |
| <span class="source-line-no">694</span><span id="line-694"></span> |
| <span class="source-line-no">695</span><span id="line-695"> /**</span> |
| <span class="source-line-no">696</span><span id="line-696"> * Cannot be {@link #UNSET}. Must be a legitimate value. Used re-making the {@link BlockCacheKey}</span> |
| <span class="source-line-no">697</span><span id="line-697"> * when block is returned to the cache.</span> |
| <span class="source-line-no">698</span><span id="line-698"> * @return the offset of this block in the file it was read from</span> |
| <span class="source-line-no">699</span><span id="line-699"> */</span> |
| <span class="source-line-no">700</span><span id="line-700"> public long getOffset() {</span> |
| <span class="source-line-no">701</span><span id="line-701"> if (offset < 0) {</span> |
| <span class="source-line-no">702</span><span id="line-702"> throw new IllegalStateException("HFile block offset not initialized properly");</span> |
| <span class="source-line-no">703</span><span id="line-703"> }</span> |
| <span class="source-line-no">704</span><span id="line-704"> return offset;</span> |
| <span class="source-line-no">705</span><span id="line-705"> }</span> |
| <span class="source-line-no">706</span><span id="line-706"></span> |
| <span class="source-line-no">707</span><span id="line-707"> /** Returns a byte stream reading the data(excluding header and checksum) of this block */</span> |
| <span class="source-line-no">708</span><span id="line-708"> DataInputStream getByteStream() {</span> |
| <span class="source-line-no">709</span><span id="line-709"> ByteBuff dup = this.bufWithoutChecksum.duplicate();</span> |
| <span class="source-line-no">710</span><span id="line-710"> dup.position(this.headerSize());</span> |
| <span class="source-line-no">711</span><span id="line-711"> return new DataInputStream(new ByteBuffInputStream(dup));</span> |
| <span class="source-line-no">712</span><span id="line-712"> }</span> |
| <span class="source-line-no">713</span><span id="line-713"></span> |
| <span class="source-line-no">714</span><span id="line-714"> @Override</span> |
| <span class="source-line-no">715</span><span id="line-715"> public long heapSize() {</span> |
| <span class="source-line-no">716</span><span id="line-716"> long size = FIXED_OVERHEAD;</span> |
| <span class="source-line-no">717</span><span id="line-717"> size += fileContext.heapSize();</span> |
| <span class="source-line-no">718</span><span id="line-718"> if (bufWithoutChecksum != null) {</span> |
| <span class="source-line-no">719</span><span id="line-719"> // Deep overhead of the byte buffer. Needs to be aligned separately.</span> |
| <span class="source-line-no">720</span><span id="line-720"> size += ClassSize.align(bufWithoutChecksum.capacity() + MULTI_BYTE_BUFFER_HEAP_SIZE);</span> |
| <span class="source-line-no">721</span><span id="line-721"> }</span> |
| <span class="source-line-no">722</span><span id="line-722"> return ClassSize.align(size);</span> |
| <span class="source-line-no">723</span><span id="line-723"> }</span> |
| <span class="source-line-no">724</span><span id="line-724"></span> |
| <span class="source-line-no">725</span><span id="line-725"> /**</span> |
| <span class="source-line-no">726</span><span id="line-726"> * Will be override by {@link SharedMemHFileBlock} or {@link ExclusiveMemHFileBlock}. Return true</span> |
| <span class="source-line-no">727</span><span id="line-727"> * by default.</span> |
| <span class="source-line-no">728</span><span id="line-728"> */</span> |
| <span class="source-line-no">729</span><span id="line-729"> public boolean isSharedMem() {</span> |
| <span class="source-line-no">730</span><span id="line-730"> return true;</span> |
| <span class="source-line-no">731</span><span id="line-731"> }</span> |
| <span class="source-line-no">732</span><span id="line-732"></span> |
| <span class="source-line-no">733</span><span id="line-733"> /**</span> |
| <span class="source-line-no">734</span><span id="line-734"> * Unified version 2 {@link HFile} block writer. The intended usage pattern is as follows:</span> |
| <span class="source-line-no">735</span><span id="line-735"> * <ol></span> |
| <span class="source-line-no">736</span><span id="line-736"> * <li>Construct an {@link HFileBlock.Writer}, providing a compression algorithm.</span> |
| <span class="source-line-no">737</span><span id="line-737"> * <li>Call {@link Writer#startWriting} and get a data stream to write to.</span> |
| <span class="source-line-no">738</span><span id="line-738"> * <li>Write your data into the stream.</span> |
| <span class="source-line-no">739</span><span id="line-739"> * <li>Call Writer#writeHeaderAndData(FSDataOutputStream) as many times as you need to. store the</span> |
| <span class="source-line-no">740</span><span id="line-740"> * serialized block into an external stream.</span> |
| <span class="source-line-no">741</span><span id="line-741"> * <li>Repeat to write more blocks.</span> |
| <span class="source-line-no">742</span><span id="line-742"> * </ol></span> |
| <span class="source-line-no">743</span><span id="line-743"> * <p></span> |
| <span class="source-line-no">744</span><span id="line-744"> */</span> |
| <span class="source-line-no">745</span><span id="line-745"> static class Writer implements ShipperListener {</span> |
| <span class="source-line-no">746</span><span id="line-746"> private enum State {</span> |
| <span class="source-line-no">747</span><span id="line-747"> INIT,</span> |
| <span class="source-line-no">748</span><span id="line-748"> WRITING,</span> |
| <span class="source-line-no">749</span><span id="line-749"> BLOCK_READY</span> |
| <span class="source-line-no">750</span><span id="line-750"> }</span> |
| <span class="source-line-no">751</span><span id="line-751"></span> |
| <span class="source-line-no">752</span><span id="line-752"> private int maxSizeUnCompressed;</span> |
| <span class="source-line-no">753</span><span id="line-753"></span> |
| <span class="source-line-no">754</span><span id="line-754"> private BlockCompressedSizePredicator compressedSizePredicator;</span> |
| <span class="source-line-no">755</span><span id="line-755"></span> |
| <span class="source-line-no">756</span><span id="line-756"> /** Writer state. Used to ensure the correct usage protocol. */</span> |
| <span class="source-line-no">757</span><span id="line-757"> private State state = State.INIT;</span> |
| <span class="source-line-no">758</span><span id="line-758"></span> |
| <span class="source-line-no">759</span><span id="line-759"> /** Data block encoder used for data blocks */</span> |
| <span class="source-line-no">760</span><span id="line-760"> private final HFileDataBlockEncoder dataBlockEncoder;</span> |
| <span class="source-line-no">761</span><span id="line-761"></span> |
| <span class="source-line-no">762</span><span id="line-762"> private HFileBlockEncodingContext dataBlockEncodingCtx;</span> |
| <span class="source-line-no">763</span><span id="line-763"></span> |
| <span class="source-line-no">764</span><span id="line-764"> /** block encoding context for non-data blocks */</span> |
| <span class="source-line-no">765</span><span id="line-765"> private HFileBlockDefaultEncodingContext defaultBlockEncodingCtx;</span> |
| <span class="source-line-no">766</span><span id="line-766"></span> |
| <span class="source-line-no">767</span><span id="line-767"> /**</span> |
| <span class="source-line-no">768</span><span id="line-768"> * The stream we use to accumulate data into a block in an uncompressed format. We reset this</span> |
| <span class="source-line-no">769</span><span id="line-769"> * stream at the end of each block and reuse it. The header is written as the first</span> |
| <span class="source-line-no">770</span><span id="line-770"> * {@link HConstants#HFILEBLOCK_HEADER_SIZE} bytes into this stream.</span> |
| <span class="source-line-no">771</span><span id="line-771"> */</span> |
| <span class="source-line-no">772</span><span id="line-772"> private ByteArrayOutputStream baosInMemory;</span> |
| <span class="source-line-no">773</span><span id="line-773"></span> |
| <span class="source-line-no">774</span><span id="line-774"> /**</span> |
| <span class="source-line-no">775</span><span id="line-775"> * Current block type. Set in {@link #startWriting(BlockType)}. Could be changed in</span> |
| <span class="source-line-no">776</span><span id="line-776"> * {@link #finishBlock()} from {@link BlockType#DATA} to {@link BlockType#ENCODED_DATA}.</span> |
| <span class="source-line-no">777</span><span id="line-777"> */</span> |
| <span class="source-line-no">778</span><span id="line-778"> private BlockType blockType;</span> |
| <span class="source-line-no">779</span><span id="line-779"></span> |
| <span class="source-line-no">780</span><span id="line-780"> /**</span> |
| <span class="source-line-no">781</span><span id="line-781"> * A stream that we write uncompressed bytes to, which compresses them and writes them to</span> |
| <span class="source-line-no">782</span><span id="line-782"> * {@link #baosInMemory}.</span> |
| <span class="source-line-no">783</span><span id="line-783"> */</span> |
| <span class="source-line-no">784</span><span id="line-784"> private DataOutputStream userDataStream;</span> |
| <span class="source-line-no">785</span><span id="line-785"></span> |
| <span class="source-line-no">786</span><span id="line-786"> /**</span> |
| <span class="source-line-no">787</span><span id="line-787"> * Bytes to be written to the file system, including the header. Compressed if compression is</span> |
| <span class="source-line-no">788</span><span id="line-788"> * turned on. It also includes the checksum data that immediately follows the block data.</span> |
| <span class="source-line-no">789</span><span id="line-789"> * (header + data + checksums)</span> |
| <span class="source-line-no">790</span><span id="line-790"> */</span> |
| <span class="source-line-no">791</span><span id="line-791"> private ByteArrayOutputStream onDiskBlockBytesWithHeader;</span> |
| <span class="source-line-no">792</span><span id="line-792"></span> |
| <span class="source-line-no">793</span><span id="line-793"> /**</span> |
| <span class="source-line-no">794</span><span id="line-794"> * The size of the checksum data on disk. It is used only if data is not compressed. If data is</span> |
| <span class="source-line-no">795</span><span id="line-795"> * compressed, then the checksums are already part of onDiskBytesWithHeader. If data is</span> |
| <span class="source-line-no">796</span><span id="line-796"> * uncompressed, then this variable stores the checksum data for this block.</span> |
| <span class="source-line-no">797</span><span id="line-797"> */</span> |
| <span class="source-line-no">798</span><span id="line-798"> private byte[] onDiskChecksum = HConstants.EMPTY_BYTE_ARRAY;</span> |
| <span class="source-line-no">799</span><span id="line-799"></span> |
| <span class="source-line-no">800</span><span id="line-800"> /**</span> |
| <span class="source-line-no">801</span><span id="line-801"> * Current block's start offset in the {@link HFile}. Set in</span> |
| <span class="source-line-no">802</span><span id="line-802"> * {@link #writeHeaderAndData(FSDataOutputStream)}.</span> |
| <span class="source-line-no">803</span><span id="line-803"> */</span> |
| <span class="source-line-no">804</span><span id="line-804"> private long startOffset;</span> |
| <span class="source-line-no">805</span><span id="line-805"></span> |
| <span class="source-line-no">806</span><span id="line-806"> /**</span> |
| <span class="source-line-no">807</span><span id="line-807"> * Offset of previous block by block type. Updated when the next block is started.</span> |
| <span class="source-line-no">808</span><span id="line-808"> */</span> |
| <span class="source-line-no">809</span><span id="line-809"> private long[] prevOffsetByType;</span> |
| <span class="source-line-no">810</span><span id="line-810"></span> |
| <span class="source-line-no">811</span><span id="line-811"> /** The offset of the previous block of the same type */</span> |
| <span class="source-line-no">812</span><span id="line-812"> private long prevOffset;</span> |
| <span class="source-line-no">813</span><span id="line-813"> /** Meta data that holds information about the hfileblock **/</span> |
| <span class="source-line-no">814</span><span id="line-814"> private HFileContext fileContext;</span> |
| <span class="source-line-no">815</span><span id="line-815"></span> |
| <span class="source-line-no">816</span><span id="line-816"> private final ByteBuffAllocator allocator;</span> |
| <span class="source-line-no">817</span><span id="line-817"></span> |
| <span class="source-line-no">818</span><span id="line-818"> @Override</span> |
| <span class="source-line-no">819</span><span id="line-819"> public void beforeShipped() {</span> |
| <span class="source-line-no">820</span><span id="line-820"> if (getEncodingState() != null) {</span> |
| <span class="source-line-no">821</span><span id="line-821"> getEncodingState().beforeShipped();</span> |
| <span class="source-line-no">822</span><span id="line-822"> }</span> |
| <span class="source-line-no">823</span><span id="line-823"> }</span> |
| <span class="source-line-no">824</span><span id="line-824"></span> |
| <span class="source-line-no">825</span><span id="line-825"> EncodingState getEncodingState() {</span> |
| <span class="source-line-no">826</span><span id="line-826"> return dataBlockEncodingCtx.getEncodingState();</span> |
| <span class="source-line-no">827</span><span id="line-827"> }</span> |
| <span class="source-line-no">828</span><span id="line-828"></span> |
| <span class="source-line-no">829</span><span id="line-829"> /**</span> |
| <span class="source-line-no">830</span><span id="line-830"> * @param dataBlockEncoder data block encoding algorithm to use</span> |
| <span class="source-line-no">831</span><span id="line-831"> */</span> |
| <span class="source-line-no">832</span><span id="line-832"> public Writer(Configuration conf, HFileDataBlockEncoder dataBlockEncoder,</span> |
| <span class="source-line-no">833</span><span id="line-833"> HFileContext fileContext) {</span> |
| <span class="source-line-no">834</span><span id="line-834"> this(conf, dataBlockEncoder, fileContext, ByteBuffAllocator.HEAP, fileContext.getBlocksize());</span> |
| <span class="source-line-no">835</span><span id="line-835"> }</span> |
| <span class="source-line-no">836</span><span id="line-836"></span> |
| <span class="source-line-no">837</span><span id="line-837"> public Writer(Configuration conf, HFileDataBlockEncoder dataBlockEncoder,</span> |
| <span class="source-line-no">838</span><span id="line-838"> HFileContext fileContext, ByteBuffAllocator allocator, int maxSizeUnCompressed) {</span> |
| <span class="source-line-no">839</span><span id="line-839"> if (fileContext.getBytesPerChecksum() < HConstants.HFILEBLOCK_HEADER_SIZE) {</span> |
| <span class="source-line-no">840</span><span id="line-840"> throw new RuntimeException("Unsupported value of bytesPerChecksum. " + " Minimum is "</span> |
| <span class="source-line-no">841</span><span id="line-841"> + HConstants.HFILEBLOCK_HEADER_SIZE + " but the configured value is "</span> |
| <span class="source-line-no">842</span><span id="line-842"> + fileContext.getBytesPerChecksum());</span> |
| <span class="source-line-no">843</span><span id="line-843"> }</span> |
| <span class="source-line-no">844</span><span id="line-844"> this.allocator = allocator;</span> |
| <span class="source-line-no">845</span><span id="line-845"> this.dataBlockEncoder =</span> |
| <span class="source-line-no">846</span><span id="line-846"> dataBlockEncoder != null ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;</span> |
| <span class="source-line-no">847</span><span id="line-847"> this.dataBlockEncodingCtx = this.dataBlockEncoder.newDataBlockEncodingContext(conf,</span> |
| <span class="source-line-no">848</span><span id="line-848"> HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext);</span> |
| <span class="source-line-no">849</span><span id="line-849"> // TODO: This should be lazily instantiated</span> |
| <span class="source-line-no">850</span><span id="line-850"> this.defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(conf, null,</span> |
| <span class="source-line-no">851</span><span id="line-851"> HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext);</span> |
| <span class="source-line-no">852</span><span id="line-852"> // TODO: Set BAOS initial size. Use fileContext.getBlocksize() and add for header/checksum</span> |
| <span class="source-line-no">853</span><span id="line-853"> baosInMemory = new ByteArrayOutputStream();</span> |
| <span class="source-line-no">854</span><span id="line-854"> prevOffsetByType = new long[BlockType.values().length];</span> |
| <span class="source-line-no">855</span><span id="line-855"> for (int i = 0; i < prevOffsetByType.length; ++i) {</span> |
| <span class="source-line-no">856</span><span id="line-856"> prevOffsetByType[i] = UNSET;</span> |
| <span class="source-line-no">857</span><span id="line-857"> }</span> |
| <span class="source-line-no">858</span><span id="line-858"> // TODO: Why fileContext saved away when we have dataBlockEncoder and/or</span> |
| <span class="source-line-no">859</span><span id="line-859"> // defaultDataBlockEncoder?</span> |
| <span class="source-line-no">860</span><span id="line-860"> this.fileContext = fileContext;</span> |
| <span class="source-line-no">861</span><span id="line-861"> this.compressedSizePredicator = (BlockCompressedSizePredicator) ReflectionUtils.newInstance(</span> |
| <span class="source-line-no">862</span><span id="line-862"> conf.getClass(BLOCK_COMPRESSED_SIZE_PREDICATOR, UncompressedBlockSizePredicator.class),</span> |
| <span class="source-line-no">863</span><span id="line-863"> new Configuration(conf));</span> |
| <span class="source-line-no">864</span><span id="line-864"> this.maxSizeUnCompressed = maxSizeUnCompressed;</span> |
| <span class="source-line-no">865</span><span id="line-865"> }</span> |
| <span class="source-line-no">866</span><span id="line-866"></span> |
| <span class="source-line-no">867</span><span id="line-867"> /**</span> |
| <span class="source-line-no">868</span><span id="line-868"> * Starts writing into the block. The previous block's data is discarded.</span> |
| <span class="source-line-no">869</span><span id="line-869"> * @return the stream the user can write their data into</span> |
| <span class="source-line-no">870</span><span id="line-870"> */</span> |
| <span class="source-line-no">871</span><span id="line-871"> DataOutputStream startWriting(BlockType newBlockType) throws IOException {</span> |
| <span class="source-line-no">872</span><span id="line-872"> if (state == State.BLOCK_READY && startOffset != -1) {</span> |
| <span class="source-line-no">873</span><span id="line-873"> // We had a previous block that was written to a stream at a specific</span> |
| <span class="source-line-no">874</span><span id="line-874"> // offset. Save that offset as the last offset of a block of that type.</span> |
| <span class="source-line-no">875</span><span id="line-875"> prevOffsetByType[blockType.getId()] = startOffset;</span> |
| <span class="source-line-no">876</span><span id="line-876"> }</span> |
| <span class="source-line-no">877</span><span id="line-877"></span> |
| <span class="source-line-no">878</span><span id="line-878"> startOffset = -1;</span> |
| <span class="source-line-no">879</span><span id="line-879"> blockType = newBlockType;</span> |
| <span class="source-line-no">880</span><span id="line-880"></span> |
| <span class="source-line-no">881</span><span id="line-881"> baosInMemory.reset();</span> |
| <span class="source-line-no">882</span><span id="line-882"> baosInMemory.write(HConstants.HFILEBLOCK_DUMMY_HEADER);</span> |
| <span class="source-line-no">883</span><span id="line-883"></span> |
| <span class="source-line-no">884</span><span id="line-884"> state = State.WRITING;</span> |
| <span class="source-line-no">885</span><span id="line-885"></span> |
| <span class="source-line-no">886</span><span id="line-886"> // We will compress it later in finishBlock()</span> |
| <span class="source-line-no">887</span><span id="line-887"> userDataStream = new ByteBufferWriterDataOutputStream(baosInMemory);</span> |
| <span class="source-line-no">888</span><span id="line-888"> if (newBlockType == BlockType.DATA) {</span> |
| <span class="source-line-no">889</span><span id="line-889"> this.dataBlockEncoder.startBlockEncoding(dataBlockEncodingCtx, userDataStream);</span> |
| <span class="source-line-no">890</span><span id="line-890"> }</span> |
| <span class="source-line-no">891</span><span id="line-891"> return userDataStream;</span> |
| <span class="source-line-no">892</span><span id="line-892"> }</span> |
| <span class="source-line-no">893</span><span id="line-893"></span> |
| <span class="source-line-no">894</span><span id="line-894"> /**</span> |
| <span class="source-line-no">895</span><span id="line-895"> * Writes the Cell to this block</span> |
| <span class="source-line-no">896</span><span id="line-896"> */</span> |
| <span class="source-line-no">897</span><span id="line-897"> void write(ExtendedCell cell) throws IOException {</span> |
| <span class="source-line-no">898</span><span id="line-898"> expectState(State.WRITING);</span> |
| <span class="source-line-no">899</span><span id="line-899"> this.dataBlockEncoder.encode(cell, dataBlockEncodingCtx, this.userDataStream);</span> |
| <span class="source-line-no">900</span><span id="line-900"> }</span> |
| <span class="source-line-no">901</span><span id="line-901"></span> |
| <span class="source-line-no">902</span><span id="line-902"> /**</span> |
| <span class="source-line-no">903</span><span id="line-903"> * Transitions the block writer from the "writing" state to the "block ready" state. Does</span> |
| <span class="source-line-no">904</span><span id="line-904"> * nothing if a block is already finished.</span> |
| <span class="source-line-no">905</span><span id="line-905"> */</span> |
| <span class="source-line-no">906</span><span id="line-906"> void ensureBlockReady() throws IOException {</span> |
| <span class="source-line-no">907</span><span id="line-907"> Preconditions.checkState(state != State.INIT, "Unexpected state: " + state);</span> |
| <span class="source-line-no">908</span><span id="line-908"></span> |
| <span class="source-line-no">909</span><span id="line-909"> if (state == State.BLOCK_READY) {</span> |
| <span class="source-line-no">910</span><span id="line-910"> return;</span> |
| <span class="source-line-no">911</span><span id="line-911"> }</span> |
| <span class="source-line-no">912</span><span id="line-912"></span> |
| <span class="source-line-no">913</span><span id="line-913"> // This will set state to BLOCK_READY.</span> |
| <span class="source-line-no">914</span><span id="line-914"> finishBlock();</span> |
| <span class="source-line-no">915</span><span id="line-915"> }</span> |
| <span class="source-line-no">916</span><span id="line-916"></span> |
| <span class="source-line-no">917</span><span id="line-917"> public boolean checkBoundariesWithPredicate() {</span> |
| <span class="source-line-no">918</span><span id="line-918"> int rawBlockSize = encodedBlockSizeWritten();</span> |
| <span class="source-line-no">919</span><span id="line-919"> if (rawBlockSize >= maxSizeUnCompressed) {</span> |
| <span class="source-line-no">920</span><span id="line-920"> return true;</span> |
| <span class="source-line-no">921</span><span id="line-921"> } else {</span> |
| <span class="source-line-no">922</span><span id="line-922"> return compressedSizePredicator.shouldFinishBlock(rawBlockSize);</span> |
| <span class="source-line-no">923</span><span id="line-923"> }</span> |
| <span class="source-line-no">924</span><span id="line-924"> }</span> |
| <span class="source-line-no">925</span><span id="line-925"></span> |
| <span class="source-line-no">926</span><span id="line-926"> /**</span> |
| <span class="source-line-no">927</span><span id="line-927"> * Finish up writing of the block. Flushes the compressing stream (if using compression), fills</span> |
| <span class="source-line-no">928</span><span id="line-928"> * out the header, does any compression/encryption of bytes to flush out to disk, and manages</span> |
| <span class="source-line-no">929</span><span id="line-929"> * the cache on write content, if applicable. Sets block write state to "block ready".</span> |
| <span class="source-line-no">930</span><span id="line-930"> */</span> |
| <span class="source-line-no">931</span><span id="line-931"> private void finishBlock() throws IOException {</span> |
| <span class="source-line-no">932</span><span id="line-932"> if (blockType == BlockType.DATA) {</span> |
| <span class="source-line-no">933</span><span id="line-933"> this.dataBlockEncoder.endBlockEncoding(dataBlockEncodingCtx, userDataStream,</span> |
| <span class="source-line-no">934</span><span id="line-934"> baosInMemory.getBuffer(), blockType);</span> |
| <span class="source-line-no">935</span><span id="line-935"> blockType = dataBlockEncodingCtx.getBlockType();</span> |
| <span class="source-line-no">936</span><span id="line-936"> }</span> |
| <span class="source-line-no">937</span><span id="line-937"> userDataStream.flush();</span> |
| <span class="source-line-no">938</span><span id="line-938"> prevOffset = prevOffsetByType[blockType.getId()];</span> |
| <span class="source-line-no">939</span><span id="line-939"></span> |
| <span class="source-line-no">940</span><span id="line-940"> // We need to cache the unencoded/uncompressed size before changing the block state</span> |
| <span class="source-line-no">941</span><span id="line-941"> int rawBlockSize = 0;</span> |
| <span class="source-line-no">942</span><span id="line-942"> if (this.getEncodingState() != null) {</span> |
| <span class="source-line-no">943</span><span id="line-943"> rawBlockSize = encodedBlockSizeWritten();</span> |
| <span class="source-line-no">944</span><span id="line-944"> }</span> |
| <span class="source-line-no">945</span><span id="line-945"> // We need to set state before we can package the block up for cache-on-write. In a way, the</span> |
| <span class="source-line-no">946</span><span id="line-946"> // block is ready, but not yet encoded or compressed.</span> |
| <span class="source-line-no">947</span><span id="line-947"> state = State.BLOCK_READY;</span> |
| <span class="source-line-no">948</span><span id="line-948"> Bytes compressAndEncryptDat;</span> |
| <span class="source-line-no">949</span><span id="line-949"> if (blockType == BlockType.DATA || blockType == BlockType.ENCODED_DATA) {</span> |
| <span class="source-line-no">950</span><span id="line-950"> compressAndEncryptDat =</span> |
| <span class="source-line-no">951</span><span id="line-951"> dataBlockEncodingCtx.compressAndEncrypt(baosInMemory.getBuffer(), 0, baosInMemory.size());</span> |
| <span class="source-line-no">952</span><span id="line-952"> } else {</span> |
| <span class="source-line-no">953</span><span id="line-953"> compressAndEncryptDat = defaultBlockEncodingCtx.compressAndEncrypt(baosInMemory.getBuffer(),</span> |
| <span class="source-line-no">954</span><span id="line-954"> 0, baosInMemory.size());</span> |
| <span class="source-line-no">955</span><span id="line-955"> }</span> |
| <span class="source-line-no">956</span><span id="line-956"> if (compressAndEncryptDat == null) {</span> |
| <span class="source-line-no">957</span><span id="line-957"> compressAndEncryptDat = new Bytes(baosInMemory.getBuffer(), 0, baosInMemory.size());</span> |
| <span class="source-line-no">958</span><span id="line-958"> }</span> |
| <span class="source-line-no">959</span><span id="line-959"> if (onDiskBlockBytesWithHeader == null) {</span> |
| <span class="source-line-no">960</span><span id="line-960"> onDiskBlockBytesWithHeader = new ByteArrayOutputStream(compressAndEncryptDat.getLength());</span> |
| <span class="source-line-no">961</span><span id="line-961"> }</span> |
| <span class="source-line-no">962</span><span id="line-962"> onDiskBlockBytesWithHeader.reset();</span> |
| <span class="source-line-no">963</span><span id="line-963"> onDiskBlockBytesWithHeader.write(compressAndEncryptDat.get(),</span> |
| <span class="source-line-no">964</span><span id="line-964"> compressAndEncryptDat.getOffset(), compressAndEncryptDat.getLength());</span> |
| <span class="source-line-no">965</span><span id="line-965"> // Update raw and compressed sizes in the predicate</span> |
| <span class="source-line-no">966</span><span id="line-966"> compressedSizePredicator.updateLatestBlockSizes(fileContext, rawBlockSize,</span> |
| <span class="source-line-no">967</span><span id="line-967"> onDiskBlockBytesWithHeader.size());</span> |
| <span class="source-line-no">968</span><span id="line-968"></span> |
| <span class="source-line-no">969</span><span id="line-969"> // Calculate how many bytes we need for checksum on the tail of the block.</span> |
| <span class="source-line-no">970</span><span id="line-970"> int numBytes = (int) ChecksumUtil.numBytes(onDiskBlockBytesWithHeader.size(),</span> |
| <span class="source-line-no">971</span><span id="line-971"> fileContext.getBytesPerChecksum());</span> |
| <span class="source-line-no">972</span><span id="line-972"></span> |
| <span class="source-line-no">973</span><span id="line-973"> // Put the header for the on disk bytes; header currently is unfilled-out</span> |
| <span class="source-line-no">974</span><span id="line-974"> putHeader(onDiskBlockBytesWithHeader, onDiskBlockBytesWithHeader.size() + numBytes,</span> |
| <span class="source-line-no">975</span><span id="line-975"> baosInMemory.size(), onDiskBlockBytesWithHeader.size());</span> |
| <span class="source-line-no">976</span><span id="line-976"></span> |
| <span class="source-line-no">977</span><span id="line-977"> if (onDiskChecksum.length != numBytes) {</span> |
| <span class="source-line-no">978</span><span id="line-978"> onDiskChecksum = new byte[numBytes];</span> |
| <span class="source-line-no">979</span><span id="line-979"> }</span> |
| <span class="source-line-no">980</span><span id="line-980"> ChecksumUtil.generateChecksums(onDiskBlockBytesWithHeader.getBuffer(), 0,</span> |
| <span class="source-line-no">981</span><span id="line-981"> onDiskBlockBytesWithHeader.size(), onDiskChecksum, 0, fileContext.getChecksumType(),</span> |
| <span class="source-line-no">982</span><span id="line-982"> fileContext.getBytesPerChecksum());</span> |
| <span class="source-line-no">983</span><span id="line-983"> }</span> |
| <span class="source-line-no">984</span><span id="line-984"></span> |
| <span class="source-line-no">985</span><span id="line-985"> /**</span> |
| <span class="source-line-no">986</span><span id="line-986"> * Put the header into the given byte array at the given offset.</span> |
| <span class="source-line-no">987</span><span id="line-987"> * @param onDiskSize size of the block on disk header + data + checksum</span> |
| <span class="source-line-no">988</span><span id="line-988"> * @param uncompressedSize size of the block after decompression (but before optional data block</span> |
| <span class="source-line-no">989</span><span id="line-989"> * decoding) including header</span> |
| <span class="source-line-no">990</span><span id="line-990"> * @param onDiskDataSize size of the block on disk with header and data but not including the</span> |
| <span class="source-line-no">991</span><span id="line-991"> * checksums</span> |
| <span class="source-line-no">992</span><span id="line-992"> */</span> |
| <span class="source-line-no">993</span><span id="line-993"> private void putHeader(byte[] dest, int offset, int onDiskSize, int uncompressedSize,</span> |
| <span class="source-line-no">994</span><span id="line-994"> int onDiskDataSize) {</span> |
| <span class="source-line-no">995</span><span id="line-995"> offset = blockType.put(dest, offset);</span> |
| <span class="source-line-no">996</span><span id="line-996"> offset = Bytes.putInt(dest, offset, onDiskSize - HConstants.HFILEBLOCK_HEADER_SIZE);</span> |
| <span class="source-line-no">997</span><span id="line-997"> offset = Bytes.putInt(dest, offset, uncompressedSize - HConstants.HFILEBLOCK_HEADER_SIZE);</span> |
| <span class="source-line-no">998</span><span id="line-998"> offset = Bytes.putLong(dest, offset, prevOffset);</span> |
| <span class="source-line-no">999</span><span id="line-999"> offset = Bytes.putByte(dest, offset, fileContext.getChecksumType().getCode());</span> |
| <span class="source-line-no">1000</span><span id="line-1000"> offset = Bytes.putInt(dest, offset, fileContext.getBytesPerChecksum());</span> |
| <span class="source-line-no">1001</span><span id="line-1001"> Bytes.putInt(dest, offset, onDiskDataSize);</span> |
| <span class="source-line-no">1002</span><span id="line-1002"> }</span> |
| <span class="source-line-no">1003</span><span id="line-1003"></span> |
| <span class="source-line-no">1004</span><span id="line-1004"> private void putHeader(ByteBuff buff, int onDiskSize, int uncompressedSize,</span> |
| <span class="source-line-no">1005</span><span id="line-1005"> int onDiskDataSize) {</span> |
| <span class="source-line-no">1006</span><span id="line-1006"> buff.rewind();</span> |
| <span class="source-line-no">1007</span><span id="line-1007"> blockType.write(buff);</span> |
| <span class="source-line-no">1008</span><span id="line-1008"> buff.putInt(onDiskSize - HConstants.HFILEBLOCK_HEADER_SIZE);</span> |
| <span class="source-line-no">1009</span><span id="line-1009"> buff.putInt(uncompressedSize - HConstants.HFILEBLOCK_HEADER_SIZE);</span> |
| <span class="source-line-no">1010</span><span id="line-1010"> buff.putLong(prevOffset);</span> |
| <span class="source-line-no">1011</span><span id="line-1011"> buff.put(fileContext.getChecksumType().getCode());</span> |
| <span class="source-line-no">1012</span><span id="line-1012"> buff.putInt(fileContext.getBytesPerChecksum());</span> |
| <span class="source-line-no">1013</span><span id="line-1013"> buff.putInt(onDiskDataSize);</span> |
| <span class="source-line-no">1014</span><span id="line-1014"> }</span> |
| <span class="source-line-no">1015</span><span id="line-1015"></span> |
| <span class="source-line-no">1016</span><span id="line-1016"> private void putHeader(ByteArrayOutputStream dest, int onDiskSize, int uncompressedSize,</span> |
| <span class="source-line-no">1017</span><span id="line-1017"> int onDiskDataSize) {</span> |
| <span class="source-line-no">1018</span><span id="line-1018"> putHeader(dest.getBuffer(), 0, onDiskSize, uncompressedSize, onDiskDataSize);</span> |
| <span class="source-line-no">1019</span><span id="line-1019"> }</span> |
| <span class="source-line-no">1020</span><span id="line-1020"></span> |
| <span class="source-line-no">1021</span><span id="line-1021"> /**</span> |
| <span class="source-line-no">1022</span><span id="line-1022"> * Similar to {@link #writeHeaderAndData(FSDataOutputStream)}, but records the offset of this</span> |
| <span class="source-line-no">1023</span><span id="line-1023"> * block so that it can be referenced in the next block of the same type.</span> |
| <span class="source-line-no">1024</span><span id="line-1024"> */</span> |
| <span class="source-line-no">1025</span><span id="line-1025"> void writeHeaderAndData(FSDataOutputStream out) throws IOException {</span> |
| <span class="source-line-no">1026</span><span id="line-1026"> long offset = out.getPos();</span> |
| <span class="source-line-no">1027</span><span id="line-1027"> if (startOffset != UNSET && offset != startOffset) {</span> |
| <span class="source-line-no">1028</span><span id="line-1028"> throw new IOException("A " + blockType + " block written to a "</span> |
| <span class="source-line-no">1029</span><span id="line-1029"> + "stream twice, first at offset " + startOffset + ", then at " + offset);</span> |
| <span class="source-line-no">1030</span><span id="line-1030"> }</span> |
| <span class="source-line-no">1031</span><span id="line-1031"> startOffset = offset;</span> |
| <span class="source-line-no">1032</span><span id="line-1032"> finishBlockAndWriteHeaderAndData(out);</span> |
| <span class="source-line-no">1033</span><span id="line-1033"> }</span> |
| <span class="source-line-no">1034</span><span id="line-1034"></span> |
| <span class="source-line-no">1035</span><span id="line-1035"> /**</span> |
| <span class="source-line-no">1036</span><span id="line-1036"> * Writes the header and the compressed data of this block (or uncompressed data when not using</span> |
| <span class="source-line-no">1037</span><span id="line-1037"> * compression) into the given stream. Can be called in the "writing" state or in the "block</span> |
| <span class="source-line-no">1038</span><span id="line-1038"> * ready" state. If called in the "writing" state, transitions the writer to the "block ready"</span> |
| <span class="source-line-no">1039</span><span id="line-1039"> * state.</span> |
| <span class="source-line-no">1040</span><span id="line-1040"> * @param out the output stream to write the</span> |
| <span class="source-line-no">1041</span><span id="line-1041"> */</span> |
| <span class="source-line-no">1042</span><span id="line-1042"> protected void finishBlockAndWriteHeaderAndData(DataOutputStream out) throws IOException {</span> |
| <span class="source-line-no">1043</span><span id="line-1043"> ensureBlockReady();</span> |
| <span class="source-line-no">1044</span><span id="line-1044"> long startTime = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">1045</span><span id="line-1045"> out.write(onDiskBlockBytesWithHeader.getBuffer(), 0, onDiskBlockBytesWithHeader.size());</span> |
| <span class="source-line-no">1046</span><span id="line-1046"> out.write(onDiskChecksum);</span> |
| <span class="source-line-no">1047</span><span id="line-1047"> HFile.updateWriteLatency(EnvironmentEdgeManager.currentTime() - startTime);</span> |
| <span class="source-line-no">1048</span><span id="line-1048"> }</span> |
| <span class="source-line-no">1049</span><span id="line-1049"></span> |
| <span class="source-line-no">1050</span><span id="line-1050"> /**</span> |
| <span class="source-line-no">1051</span><span id="line-1051"> * Returns the header or the compressed data (or uncompressed data when not using compression)</span> |
| <span class="source-line-no">1052</span><span id="line-1052"> * as a byte array. Can be called in the "writing" state or in the "block ready" state. If</span> |
| <span class="source-line-no">1053</span><span id="line-1053"> * called in the "writing" state, transitions the writer to the "block ready" state. This</span> |
| <span class="source-line-no">1054</span><span id="line-1054"> * returns the header + data + checksums stored on disk.</span> |
| <span class="source-line-no">1055</span><span id="line-1055"> * @return header and data as they would be stored on disk in a byte array</span> |
| <span class="source-line-no">1056</span><span id="line-1056"> */</span> |
| <span class="source-line-no">1057</span><span id="line-1057"> byte[] getHeaderAndDataForTest() throws IOException {</span> |
| <span class="source-line-no">1058</span><span id="line-1058"> ensureBlockReady();</span> |
| <span class="source-line-no">1059</span><span id="line-1059"> // This is not very optimal, because we are doing an extra copy.</span> |
| <span class="source-line-no">1060</span><span id="line-1060"> // But this method is used only by unit tests.</span> |
| <span class="source-line-no">1061</span><span id="line-1061"> byte[] output = new byte[onDiskBlockBytesWithHeader.size() + onDiskChecksum.length];</span> |
| <span class="source-line-no">1062</span><span id="line-1062"> System.arraycopy(onDiskBlockBytesWithHeader.getBuffer(), 0, output, 0,</span> |
| <span class="source-line-no">1063</span><span id="line-1063"> onDiskBlockBytesWithHeader.size());</span> |
| <span class="source-line-no">1064</span><span id="line-1064"> System.arraycopy(onDiskChecksum, 0, output, onDiskBlockBytesWithHeader.size(),</span> |
| <span class="source-line-no">1065</span><span id="line-1065"> onDiskChecksum.length);</span> |
| <span class="source-line-no">1066</span><span id="line-1066"> return output;</span> |
| <span class="source-line-no">1067</span><span id="line-1067"> }</span> |
| <span class="source-line-no">1068</span><span id="line-1068"></span> |
| <span class="source-line-no">1069</span><span id="line-1069"> /**</span> |
| <span class="source-line-no">1070</span><span id="line-1070"> * Releases resources used by this writer.</span> |
| <span class="source-line-no">1071</span><span id="line-1071"> */</span> |
| <span class="source-line-no">1072</span><span id="line-1072"> void release() {</span> |
| <span class="source-line-no">1073</span><span id="line-1073"> if (dataBlockEncodingCtx != null) {</span> |
| <span class="source-line-no">1074</span><span id="line-1074"> dataBlockEncodingCtx.close();</span> |
| <span class="source-line-no">1075</span><span id="line-1075"> dataBlockEncodingCtx = null;</span> |
| <span class="source-line-no">1076</span><span id="line-1076"> }</span> |
| <span class="source-line-no">1077</span><span id="line-1077"> if (defaultBlockEncodingCtx != null) {</span> |
| <span class="source-line-no">1078</span><span id="line-1078"> defaultBlockEncodingCtx.close();</span> |
| <span class="source-line-no">1079</span><span id="line-1079"> defaultBlockEncodingCtx = null;</span> |
| <span class="source-line-no">1080</span><span id="line-1080"> }</span> |
| <span class="source-line-no">1081</span><span id="line-1081"> }</span> |
| <span class="source-line-no">1082</span><span id="line-1082"></span> |
| <span class="source-line-no">1083</span><span id="line-1083"> /**</span> |
| <span class="source-line-no">1084</span><span id="line-1084"> * Returns the on-disk size of the data portion of the block. This is the compressed size if</span> |
| <span class="source-line-no">1085</span><span id="line-1085"> * compression is enabled. Can only be called in the "block ready" state. Header is not</span> |
| <span class="source-line-no">1086</span><span id="line-1086"> * compressed, and its size is not included in the return value.</span> |
| <span class="source-line-no">1087</span><span id="line-1087"> * @return the on-disk size of the block, not including the header.</span> |
| <span class="source-line-no">1088</span><span id="line-1088"> */</span> |
| <span class="source-line-no">1089</span><span id="line-1089"> int getOnDiskSizeWithoutHeader() {</span> |
| <span class="source-line-no">1090</span><span id="line-1090"> expectState(State.BLOCK_READY);</span> |
| <span class="source-line-no">1091</span><span id="line-1091"> return onDiskBlockBytesWithHeader.size() + onDiskChecksum.length</span> |
| <span class="source-line-no">1092</span><span id="line-1092"> - HConstants.HFILEBLOCK_HEADER_SIZE;</span> |
| <span class="source-line-no">1093</span><span id="line-1093"> }</span> |
| <span class="source-line-no">1094</span><span id="line-1094"></span> |
| <span class="source-line-no">1095</span><span id="line-1095"> /**</span> |
| <span class="source-line-no">1096</span><span id="line-1096"> * Returns the on-disk size of the block. Can only be called in the "block ready" state.</span> |
| <span class="source-line-no">1097</span><span id="line-1097"> * @return the on-disk size of the block ready to be written, including the header size, the</span> |
| <span class="source-line-no">1098</span><span id="line-1098"> * data and the checksum data.</span> |
| <span class="source-line-no">1099</span><span id="line-1099"> */</span> |
| <span class="source-line-no">1100</span><span id="line-1100"> int getOnDiskSizeWithHeader() {</span> |
| <span class="source-line-no">1101</span><span id="line-1101"> expectState(State.BLOCK_READY);</span> |
| <span class="source-line-no">1102</span><span id="line-1102"> return onDiskBlockBytesWithHeader.size() + onDiskChecksum.length;</span> |
| <span class="source-line-no">1103</span><span id="line-1103"> }</span> |
| <span class="source-line-no">1104</span><span id="line-1104"></span> |
| <span class="source-line-no">1105</span><span id="line-1105"> /**</span> |
| <span class="source-line-no">1106</span><span id="line-1106"> * The uncompressed size of the block data. Does not include header size.</span> |
| <span class="source-line-no">1107</span><span id="line-1107"> */</span> |
| <span class="source-line-no">1108</span><span id="line-1108"> int getUncompressedSizeWithoutHeader() {</span> |
| <span class="source-line-no">1109</span><span id="line-1109"> expectState(State.BLOCK_READY);</span> |
| <span class="source-line-no">1110</span><span id="line-1110"> return baosInMemory.size() - HConstants.HFILEBLOCK_HEADER_SIZE;</span> |
| <span class="source-line-no">1111</span><span id="line-1111"> }</span> |
| <span class="source-line-no">1112</span><span id="line-1112"></span> |
| <span class="source-line-no">1113</span><span id="line-1113"> /**</span> |
| <span class="source-line-no">1114</span><span id="line-1114"> * The uncompressed size of the block data, including header size.</span> |
| <span class="source-line-no">1115</span><span id="line-1115"> */</span> |
| <span class="source-line-no">1116</span><span id="line-1116"> public int getUncompressedSizeWithHeader() {</span> |
| <span class="source-line-no">1117</span><span id="line-1117"> expectState(State.BLOCK_READY);</span> |
| <span class="source-line-no">1118</span><span id="line-1118"> return baosInMemory.size();</span> |
| <span class="source-line-no">1119</span><span id="line-1119"> }</span> |
| <span class="source-line-no">1120</span><span id="line-1120"></span> |
| <span class="source-line-no">1121</span><span id="line-1121"> /** Returns true if a block is being written */</span> |
| <span class="source-line-no">1122</span><span id="line-1122"> boolean isWriting() {</span> |
| <span class="source-line-no">1123</span><span id="line-1123"> return state == State.WRITING;</span> |
| <span class="source-line-no">1124</span><span id="line-1124"> }</span> |
| <span class="source-line-no">1125</span><span id="line-1125"></span> |
| <span class="source-line-no">1126</span><span id="line-1126"> /**</span> |
| <span class="source-line-no">1127</span><span id="line-1127"> * Returns the number of bytes written into the current block so far, or zero if not writing the</span> |
| <span class="source-line-no">1128</span><span id="line-1128"> * block at the moment. Note that this will return zero in the "block ready" state as well.</span> |
| <span class="source-line-no">1129</span><span id="line-1129"> * @return the number of bytes written</span> |
| <span class="source-line-no">1130</span><span id="line-1130"> */</span> |
| <span class="source-line-no">1131</span><span id="line-1131"> public int encodedBlockSizeWritten() {</span> |
| <span class="source-line-no">1132</span><span id="line-1132"> return state != State.WRITING ? 0 : this.getEncodingState().getEncodedDataSizeWritten();</span> |
| <span class="source-line-no">1133</span><span id="line-1133"> }</span> |
| <span class="source-line-no">1134</span><span id="line-1134"></span> |
| <span class="source-line-no">1135</span><span id="line-1135"> /**</span> |
| <span class="source-line-no">1136</span><span id="line-1136"> * Returns the number of bytes written into the current block so far, or zero if not writing the</span> |
| <span class="source-line-no">1137</span><span id="line-1137"> * block at the moment. Note that this will return zero in the "block ready" state as well.</span> |
| <span class="source-line-no">1138</span><span id="line-1138"> * @return the number of bytes written</span> |
| <span class="source-line-no">1139</span><span id="line-1139"> */</span> |
| <span class="source-line-no">1140</span><span id="line-1140"> public int blockSizeWritten() {</span> |
| <span class="source-line-no">1141</span><span id="line-1141"> return state != State.WRITING ? 0 : this.getEncodingState().getUnencodedDataSizeWritten();</span> |
| <span class="source-line-no">1142</span><span id="line-1142"> }</span> |
| <span class="source-line-no">1143</span><span id="line-1143"></span> |
| <span class="source-line-no">1144</span><span id="line-1144"> /**</span> |
| <span class="source-line-no">1145</span><span id="line-1145"> * Clones the header followed by the uncompressed data, even if using compression. This is</span> |
| <span class="source-line-no">1146</span><span id="line-1146"> * needed for storing uncompressed blocks in the block cache. Can be called in the "writing"</span> |
| <span class="source-line-no">1147</span><span id="line-1147"> * state or the "block ready" state. Returns only the header and data, does not include checksum</span> |
| <span class="source-line-no">1148</span><span id="line-1148"> * data.</span> |
| <span class="source-line-no">1149</span><span id="line-1149"> * @return Returns an uncompressed block ByteBuff for caching on write</span> |
| <span class="source-line-no">1150</span><span id="line-1150"> */</span> |
| <span class="source-line-no">1151</span><span id="line-1151"> ByteBuff cloneUncompressedBufferWithHeader() {</span> |
| <span class="source-line-no">1152</span><span id="line-1152"> expectState(State.BLOCK_READY);</span> |
| <span class="source-line-no">1153</span><span id="line-1153"> ByteBuff bytebuff = allocator.allocate(baosInMemory.size());</span> |
| <span class="source-line-no">1154</span><span id="line-1154"> baosInMemory.toByteBuff(bytebuff);</span> |
| <span class="source-line-no">1155</span><span id="line-1155"> int numBytes = (int) ChecksumUtil.numBytes(onDiskBlockBytesWithHeader.size(),</span> |
| <span class="source-line-no">1156</span><span id="line-1156"> fileContext.getBytesPerChecksum());</span> |
| <span class="source-line-no">1157</span><span id="line-1157"> putHeader(bytebuff, onDiskBlockBytesWithHeader.size() + numBytes, baosInMemory.size(),</span> |
| <span class="source-line-no">1158</span><span id="line-1158"> onDiskBlockBytesWithHeader.size());</span> |
| <span class="source-line-no">1159</span><span id="line-1159"> bytebuff.rewind();</span> |
| <span class="source-line-no">1160</span><span id="line-1160"> return bytebuff;</span> |
| <span class="source-line-no">1161</span><span id="line-1161"> }</span> |
| <span class="source-line-no">1162</span><span id="line-1162"></span> |
| <span class="source-line-no">1163</span><span id="line-1163"> /**</span> |
| <span class="source-line-no">1164</span><span id="line-1164"> * Clones the header followed by the on-disk (compressed/encoded/encrypted) data. This is needed</span> |
| <span class="source-line-no">1165</span><span id="line-1165"> * for storing packed blocks in the block cache. Returns only the header and data, Does not</span> |
| <span class="source-line-no">1166</span><span id="line-1166"> * include checksum data.</span> |
| <span class="source-line-no">1167</span><span id="line-1167"> * @return Returns a copy of block bytes for caching on write</span> |
| <span class="source-line-no">1168</span><span id="line-1168"> */</span> |
| <span class="source-line-no">1169</span><span id="line-1169"> private ByteBuff cloneOnDiskBufferWithHeader() {</span> |
| <span class="source-line-no">1170</span><span id="line-1170"> expectState(State.BLOCK_READY);</span> |
| <span class="source-line-no">1171</span><span id="line-1171"> ByteBuff bytebuff = allocator.allocate(onDiskBlockBytesWithHeader.size());</span> |
| <span class="source-line-no">1172</span><span id="line-1172"> onDiskBlockBytesWithHeader.toByteBuff(bytebuff);</span> |
| <span class="source-line-no">1173</span><span id="line-1173"> bytebuff.rewind();</span> |
| <span class="source-line-no">1174</span><span id="line-1174"> return bytebuff;</span> |
| <span class="source-line-no">1175</span><span id="line-1175"> }</span> |
| <span class="source-line-no">1176</span><span id="line-1176"></span> |
| <span class="source-line-no">1177</span><span id="line-1177"> private void expectState(State expectedState) {</span> |
| <span class="source-line-no">1178</span><span id="line-1178"> if (state != expectedState) {</span> |
| <span class="source-line-no">1179</span><span id="line-1179"> throw new IllegalStateException(</span> |
| <span class="source-line-no">1180</span><span id="line-1180"> "Expected state: " + expectedState + ", actual state: " + state);</span> |
| <span class="source-line-no">1181</span><span id="line-1181"> }</span> |
| <span class="source-line-no">1182</span><span id="line-1182"> }</span> |
| <span class="source-line-no">1183</span><span id="line-1183"></span> |
| <span class="source-line-no">1184</span><span id="line-1184"> /**</span> |
| <span class="source-line-no">1185</span><span id="line-1185"> * Takes the given {@link BlockWritable} instance, creates a new block of its appropriate type,</span> |
| <span class="source-line-no">1186</span><span id="line-1186"> * writes the writable into this block, and flushes the block into the output stream. The writer</span> |
| <span class="source-line-no">1187</span><span id="line-1187"> * is instructed not to buffer uncompressed bytes for cache-on-write.</span> |
| <span class="source-line-no">1188</span><span id="line-1188"> * @param bw the block-writable object to write as a block</span> |
| <span class="source-line-no">1189</span><span id="line-1189"> * @param out the file system output stream</span> |
| <span class="source-line-no">1190</span><span id="line-1190"> */</span> |
| <span class="source-line-no">1191</span><span id="line-1191"> void writeBlock(BlockWritable bw, FSDataOutputStream out) throws IOException {</span> |
| <span class="source-line-no">1192</span><span id="line-1192"> bw.writeToBlock(startWriting(bw.getBlockType()));</span> |
| <span class="source-line-no">1193</span><span id="line-1193"> writeHeaderAndData(out);</span> |
| <span class="source-line-no">1194</span><span id="line-1194"> }</span> |
| <span class="source-line-no">1195</span><span id="line-1195"></span> |
| <span class="source-line-no">1196</span><span id="line-1196"> /**</span> |
| <span class="source-line-no">1197</span><span id="line-1197"> * Creates a new HFileBlock. Checksums have already been validated, so the byte buffer passed</span> |
| <span class="source-line-no">1198</span><span id="line-1198"> * into the constructor of this newly created block does not have checksum data even though the</span> |
| <span class="source-line-no">1199</span><span id="line-1199"> * header minor version is MINOR_VERSION_WITH_CHECKSUM. This is indicated by setting a 0 value</span> |
| <span class="source-line-no">1200</span><span id="line-1200"> * in bytesPerChecksum. This method copies the on-disk or uncompressed data to build the</span> |
| <span class="source-line-no">1201</span><span id="line-1201"> * HFileBlock which is used only while writing blocks and caching.</span> |
| <span class="source-line-no">1202</span><span id="line-1202"> * <p></span> |
| <span class="source-line-no">1203</span><span id="line-1203"> * TODO: Should there be an option where a cache can ask that hbase preserve block checksums for</span> |
| <span class="source-line-no">1204</span><span id="line-1204"> * checking after a block comes out of the cache? Otehrwise, cache is responsible for blocks</span> |
| <span class="source-line-no">1205</span><span id="line-1205"> * being wholesome (ECC memory or if file-backed, it does checksumming).</span> |
| <span class="source-line-no">1206</span><span id="line-1206"> */</span> |
| <span class="source-line-no">1207</span><span id="line-1207"> HFileBlock getBlockForCaching(CacheConfig cacheConf) {</span> |
| <span class="source-line-no">1208</span><span id="line-1208"> HFileContext newContext = BlockCacheUtil.cloneContext(fileContext);</span> |
| <span class="source-line-no">1209</span><span id="line-1209"> // Build the HFileBlock.</span> |
| <span class="source-line-no">1210</span><span id="line-1210"> HFileBlockBuilder builder = new HFileBlockBuilder();</span> |
| <span class="source-line-no">1211</span><span id="line-1211"> ByteBuff buff;</span> |
| <span class="source-line-no">1212</span><span id="line-1212"> if (cacheConf.shouldCacheCompressed(blockType.getCategory())) {</span> |
| <span class="source-line-no">1213</span><span id="line-1213"> buff = cloneOnDiskBufferWithHeader();</span> |
| <span class="source-line-no">1214</span><span id="line-1214"> } else {</span> |
| <span class="source-line-no">1215</span><span id="line-1215"> buff = cloneUncompressedBufferWithHeader();</span> |
| <span class="source-line-no">1216</span><span id="line-1216"> }</span> |
| <span class="source-line-no">1217</span><span id="line-1217"> return builder.withBlockType(blockType)</span> |
| <span class="source-line-no">1218</span><span id="line-1218"> .withOnDiskSizeWithoutHeader(getOnDiskSizeWithoutHeader())</span> |
| <span class="source-line-no">1219</span><span id="line-1219"> .withUncompressedSizeWithoutHeader(getUncompressedSizeWithoutHeader())</span> |
| <span class="source-line-no">1220</span><span id="line-1220"> .withPrevBlockOffset(prevOffset).withByteBuff(buff).withFillHeader(FILL_HEADER)</span> |
| <span class="source-line-no">1221</span><span id="line-1221"> .withOffset(startOffset).withNextBlockOnDiskSize(UNSET)</span> |
| <span class="source-line-no">1222</span><span id="line-1222"> .withOnDiskDataSizeWithHeader(onDiskBlockBytesWithHeader.size() + onDiskChecksum.length)</span> |
| <span class="source-line-no">1223</span><span id="line-1223"> .withHFileContext(newContext).withByteBuffAllocator(cacheConf.getByteBuffAllocator())</span> |
| <span class="source-line-no">1224</span><span id="line-1224"> .withShared(!buff.hasArray()).build();</span> |
| <span class="source-line-no">1225</span><span id="line-1225"> }</span> |
| <span class="source-line-no">1226</span><span id="line-1226"> }</span> |
| <span class="source-line-no">1227</span><span id="line-1227"></span> |
| <span class="source-line-no">1228</span><span id="line-1228"> /** Something that can be written into a block. */</span> |
| <span class="source-line-no">1229</span><span id="line-1229"> interface BlockWritable {</span> |
| <span class="source-line-no">1230</span><span id="line-1230"> /** The type of block this data should use. */</span> |
| <span class="source-line-no">1231</span><span id="line-1231"> BlockType getBlockType();</span> |
| <span class="source-line-no">1232</span><span id="line-1232"></span> |
| <span class="source-line-no">1233</span><span id="line-1233"> /**</span> |
| <span class="source-line-no">1234</span><span id="line-1234"> * Writes the block to the provided stream. Must not write any magic records.</span> |
| <span class="source-line-no">1235</span><span id="line-1235"> * @param out a stream to write uncompressed data into</span> |
| <span class="source-line-no">1236</span><span id="line-1236"> */</span> |
| <span class="source-line-no">1237</span><span id="line-1237"> void writeToBlock(DataOutput out) throws IOException;</span> |
| <span class="source-line-no">1238</span><span id="line-1238"> }</span> |
| <span class="source-line-no">1239</span><span id="line-1239"></span> |
| <span class="source-line-no">1240</span><span id="line-1240"> /**</span> |
| <span class="source-line-no">1241</span><span id="line-1241"> * Iterator for reading {@link HFileBlock}s in load-on-open-section, such as root data index</span> |
| <span class="source-line-no">1242</span><span id="line-1242"> * block, meta index block, file info block etc.</span> |
| <span class="source-line-no">1243</span><span id="line-1243"> */</span> |
| <span class="source-line-no">1244</span><span id="line-1244"> interface BlockIterator {</span> |
| <span class="source-line-no">1245</span><span id="line-1245"> /**</span> |
| <span class="source-line-no">1246</span><span id="line-1246"> * Get the next block, or null if there are no more blocks to iterate.</span> |
| <span class="source-line-no">1247</span><span id="line-1247"> */</span> |
| <span class="source-line-no">1248</span><span id="line-1248"> HFileBlock nextBlock() throws IOException;</span> |
| <span class="source-line-no">1249</span><span id="line-1249"></span> |
| <span class="source-line-no">1250</span><span id="line-1250"> /**</span> |
| <span class="source-line-no">1251</span><span id="line-1251"> * Similar to {@link #nextBlock()} but checks block type, throws an exception if incorrect, and</span> |
| <span class="source-line-no">1252</span><span id="line-1252"> * returns the HFile block</span> |
| <span class="source-line-no">1253</span><span id="line-1253"> */</span> |
| <span class="source-line-no">1254</span><span id="line-1254"> HFileBlock nextBlockWithBlockType(BlockType blockType) throws IOException;</span> |
| <span class="source-line-no">1255</span><span id="line-1255"></span> |
| <span class="source-line-no">1256</span><span id="line-1256"> /**</span> |
| <span class="source-line-no">1257</span><span id="line-1257"> * Now we use the {@link ByteBuffAllocator} to manage the nio ByteBuffers for HFileBlocks, so we</span> |
| <span class="source-line-no">1258</span><span id="line-1258"> * must deallocate all of the ByteBuffers in the end life. the BlockIterator's life cycle is</span> |
| <span class="source-line-no">1259</span><span id="line-1259"> * starting from opening an HFileReader and stopped when the HFileReader#close, so we will keep</span> |
| <span class="source-line-no">1260</span><span id="line-1260"> * track all the read blocks until we call {@link BlockIterator#freeBlocks()} when closing the</span> |
| <span class="source-line-no">1261</span><span id="line-1261"> * HFileReader. Sum bytes of those blocks in load-on-open section should be quite small, so</span> |
| <span class="source-line-no">1262</span><span id="line-1262"> * tracking them should be OK.</span> |
| <span class="source-line-no">1263</span><span id="line-1263"> */</span> |
| <span class="source-line-no">1264</span><span id="line-1264"> void freeBlocks();</span> |
| <span class="source-line-no">1265</span><span id="line-1265"> }</span> |
| <span class="source-line-no">1266</span><span id="line-1266"></span> |
| <span class="source-line-no">1267</span><span id="line-1267"> /** An HFile block reader with iteration ability. */</span> |
| <span class="source-line-no">1268</span><span id="line-1268"> interface FSReader {</span> |
| <span class="source-line-no">1269</span><span id="line-1269"> /**</span> |
| <span class="source-line-no">1270</span><span id="line-1270"> * Reads the block at the given offset in the file with the given on-disk size and uncompressed</span> |
| <span class="source-line-no">1271</span><span id="line-1271"> * size.</span> |
| <span class="source-line-no">1272</span><span id="line-1272"> * @param offset of the file to read</span> |
| <span class="source-line-no">1273</span><span id="line-1273"> * @param onDiskSize the on-disk size of the entire block, including all applicable headers,</span> |
| <span class="source-line-no">1274</span><span id="line-1274"> * or -1 if unknown</span> |
| <span class="source-line-no">1275</span><span id="line-1275"> * @param pread true to use pread, otherwise use the stream read.</span> |
| <span class="source-line-no">1276</span><span id="line-1276"> * @param updateMetrics update the metrics or not.</span> |
| <span class="source-line-no">1277</span><span id="line-1277"> * @param intoHeap allocate the block's ByteBuff by {@link ByteBuffAllocator} or JVM heap.</span> |
| <span class="source-line-no">1278</span><span id="line-1278"> * For LRUBlockCache, we must ensure that the block to cache is an heap</span> |
| <span class="source-line-no">1279</span><span id="line-1279"> * one, because the memory occupation is based on heap now, also for</span> |
| <span class="source-line-no">1280</span><span id="line-1280"> * {@link CombinedBlockCache}, we use the heap LRUBlockCache as L1 cache to</span> |
| <span class="source-line-no">1281</span><span id="line-1281"> * cache small blocks such as IndexBlock or MetaBlock for faster access. So</span> |
| <span class="source-line-no">1282</span><span id="line-1282"> * introduce an flag here to decide whether allocate from JVM heap or not</span> |
| <span class="source-line-no">1283</span><span id="line-1283"> * so that we can avoid an extra off-heap to heap memory copy when using</span> |
| <span class="source-line-no">1284</span><span id="line-1284"> * LRUBlockCache. For most cases, we known what's the expected block type</span> |
| <span class="source-line-no">1285</span><span id="line-1285"> * we'll read, while for some special case (Example:</span> |
| <span class="source-line-no">1286</span><span id="line-1286"> * HFileReaderImpl#readNextDataBlock()), we cannot pre-decide what's the</span> |
| <span class="source-line-no">1287</span><span id="line-1287"> * expected block type, then we can only allocate block's ByteBuff from</span> |
| <span class="source-line-no">1288</span><span id="line-1288"> * {@link ByteBuffAllocator} firstly, and then when caching it in</span> |
| <span class="source-line-no">1289</span><span id="line-1289"> * {@link LruBlockCache} we'll check whether the ByteBuff is from heap or</span> |
| <span class="source-line-no">1290</span><span id="line-1290"> * not, if not then we'll clone it to an heap one and cache it.</span> |
| <span class="source-line-no">1291</span><span id="line-1291"> * @return the newly read block</span> |
| <span class="source-line-no">1292</span><span id="line-1292"> */</span> |
| <span class="source-line-no">1293</span><span id="line-1293"> HFileBlock readBlockData(long offset, long onDiskSize, boolean pread, boolean updateMetrics,</span> |
| <span class="source-line-no">1294</span><span id="line-1294"> boolean intoHeap) throws IOException;</span> |
| <span class="source-line-no">1295</span><span id="line-1295"></span> |
| <span class="source-line-no">1296</span><span id="line-1296"> /**</span> |
| <span class="source-line-no">1297</span><span id="line-1297"> * Creates a block iterator over the given portion of the {@link HFile}. The iterator returns</span> |
| <span class="source-line-no">1298</span><span id="line-1298"> * blocks starting with offset such that offset &lt;= startOffset &lt; endOffset. Returned</span> |
| <span class="source-line-no">1299</span><span id="line-1299"> * blocks are always unpacked. Used when no hfile index available; e.g. reading in the hfile</span> |
| <span class="source-line-no">1300</span><span id="line-1300"> * index blocks themselves on file open.</span> |
| <span class="source-line-no">1301</span><span id="line-1301"> * @param startOffset the offset of the block to start iteration with</span> |
| <span class="source-line-no">1302</span><span id="line-1302"> * @param endOffset the offset to end iteration at (exclusive)</span> |
| <span class="source-line-no">1303</span><span id="line-1303"> * @return an iterator of blocks between the two given offsets</span> |
| <span class="source-line-no">1304</span><span id="line-1304"> */</span> |
| <span class="source-line-no">1305</span><span id="line-1305"> BlockIterator blockRange(long startOffset, long endOffset);</span> |
| <span class="source-line-no">1306</span><span id="line-1306"></span> |
| <span class="source-line-no">1307</span><span id="line-1307"> /** Closes the backing streams */</span> |
| <span class="source-line-no">1308</span><span id="line-1308"> void closeStreams() throws IOException;</span> |
| <span class="source-line-no">1309</span><span id="line-1309"></span> |
| <span class="source-line-no">1310</span><span id="line-1310"> /** Get a decoder for {@link BlockType#ENCODED_DATA} blocks from this file. */</span> |
| <span class="source-line-no">1311</span><span id="line-1311"> HFileBlockDecodingContext getBlockDecodingContext();</span> |
| <span class="source-line-no">1312</span><span id="line-1312"></span> |
| <span class="source-line-no">1313</span><span id="line-1313"> /** Get the default decoder for blocks from this file. */</span> |
| <span class="source-line-no">1314</span><span id="line-1314"> HFileBlockDecodingContext getDefaultBlockDecodingContext();</span> |
| <span class="source-line-no">1315</span><span id="line-1315"></span> |
| <span class="source-line-no">1316</span><span id="line-1316"> void setIncludesMemStoreTS(boolean includesMemstoreTS);</span> |
| <span class="source-line-no">1317</span><span id="line-1317"></span> |
| <span class="source-line-no">1318</span><span id="line-1318"> void setDataBlockEncoder(HFileDataBlockEncoder encoder, Configuration conf);</span> |
| <span class="source-line-no">1319</span><span id="line-1319"></span> |
| <span class="source-line-no">1320</span><span id="line-1320"> /**</span> |
| <span class="source-line-no">1321</span><span id="line-1321"> * To close the stream's socket. Note: This can be concurrently called from multiple threads and</span> |
| <span class="source-line-no">1322</span><span id="line-1322"> * implementation should take care of thread safety.</span> |
| <span class="source-line-no">1323</span><span id="line-1323"> */</span> |
| <span class="source-line-no">1324</span><span id="line-1324"> void unbufferStream();</span> |
| <span class="source-line-no">1325</span><span id="line-1325"> }</span> |
| <span class="source-line-no">1326</span><span id="line-1326"></span> |
| <span class="source-line-no">1327</span><span id="line-1327"> /**</span> |
| <span class="source-line-no">1328</span><span id="line-1328"> * Data-structure to use caching the header of the NEXT block. Only works if next read that comes</span> |
| <span class="source-line-no">1329</span><span id="line-1329"> * in here is next in sequence in this block. When we read, we read current block and the next</span> |
| <span class="source-line-no">1330</span><span id="line-1330"> * blocks' header. We do this so we have the length of the next block to read if the hfile index</span> |
| <span class="source-line-no">1331</span><span id="line-1331"> * is not available (rare, at hfile open only).</span> |
| <span class="source-line-no">1332</span><span id="line-1332"> */</span> |
| <span class="source-line-no">1333</span><span id="line-1333"> private static class PrefetchedHeader {</span> |
| <span class="source-line-no">1334</span><span id="line-1334"> long offset = -1;</span> |
| <span class="source-line-no">1335</span><span id="line-1335"> byte[] header = new byte[HConstants.HFILEBLOCK_HEADER_SIZE];</span> |
| <span class="source-line-no">1336</span><span id="line-1336"> final ByteBuff buf = new SingleByteBuff(ByteBuffer.wrap(header, 0, header.length));</span> |
| <span class="source-line-no">1337</span><span id="line-1337"></span> |
| <span class="source-line-no">1338</span><span id="line-1338"> @Override</span> |
| <span class="source-line-no">1339</span><span id="line-1339"> public String toString() {</span> |
| <span class="source-line-no">1340</span><span id="line-1340"> return "offset=" + this.offset + ", header=" + Bytes.toStringBinary(header);</span> |
| <span class="source-line-no">1341</span><span id="line-1341"> }</span> |
| <span class="source-line-no">1342</span><span id="line-1342"> }</span> |
| <span class="source-line-no">1343</span><span id="line-1343"></span> |
| <span class="source-line-no">1344</span><span id="line-1344"> /**</span> |
| <span class="source-line-no">1345</span><span id="line-1345"> * Reads version 2 HFile blocks from the filesystem.</span> |
| <span class="source-line-no">1346</span><span id="line-1346"> */</span> |
| <span class="source-line-no">1347</span><span id="line-1347"> static class FSReaderImpl implements FSReader {</span> |
| <span class="source-line-no">1348</span><span id="line-1348"> /**</span> |
| <span class="source-line-no">1349</span><span id="line-1349"> * The file system stream of the underlying {@link HFile} that does or doesn't do checksum</span> |
| <span class="source-line-no">1350</span><span id="line-1350"> * validations in the filesystem</span> |
| <span class="source-line-no">1351</span><span id="line-1351"> */</span> |
| <span class="source-line-no">1352</span><span id="line-1352"> private FSDataInputStreamWrapper streamWrapper;</span> |
| <span class="source-line-no">1353</span><span id="line-1353"></span> |
| <span class="source-line-no">1354</span><span id="line-1354"> private HFileBlockDecodingContext encodedBlockDecodingCtx;</span> |
| <span class="source-line-no">1355</span><span id="line-1355"></span> |
| <span class="source-line-no">1356</span><span id="line-1356"> /** Default context used when BlockType != {@link BlockType#ENCODED_DATA}. */</span> |
| <span class="source-line-no">1357</span><span id="line-1357"> private final HFileBlockDefaultDecodingContext defaultDecodingCtx;</span> |
| <span class="source-line-no">1358</span><span id="line-1358"></span> |
| <span class="source-line-no">1359</span><span id="line-1359"> /**</span> |
| <span class="source-line-no">1360</span><span id="line-1360"> * Cache of the NEXT header after this. Check it is indeed next blocks header before using it.</span> |
| <span class="source-line-no">1361</span><span id="line-1361"> * TODO: Review. This overread into next block to fetch next blocks header seems unnecessary</span> |
| <span class="source-line-no">1362</span><span id="line-1362"> * given we usually get the block size from the hfile index. Review!</span> |
| <span class="source-line-no">1363</span><span id="line-1363"> */</span> |
| <span class="source-line-no">1364</span><span id="line-1364"> private AtomicReference<PrefetchedHeader> prefetchedHeader =</span> |
| <span class="source-line-no">1365</span><span id="line-1365"> new AtomicReference<>(new PrefetchedHeader());</span> |
| <span class="source-line-no">1366</span><span id="line-1366"></span> |
| <span class="source-line-no">1367</span><span id="line-1367"> /** The size of the file we are reading from, or -1 if unknown. */</span> |
| <span class="source-line-no">1368</span><span id="line-1368"> private long fileSize;</span> |
| <span class="source-line-no">1369</span><span id="line-1369"></span> |
| <span class="source-line-no">1370</span><span id="line-1370"> /** The size of the header */</span> |
| <span class="source-line-no">1371</span><span id="line-1371"> protected final int hdrSize;</span> |
| <span class="source-line-no">1372</span><span id="line-1372"></span> |
| <span class="source-line-no">1373</span><span id="line-1373"> /** The filesystem used to access data */</span> |
| <span class="source-line-no">1374</span><span id="line-1374"> private HFileSystem hfs;</span> |
| <span class="source-line-no">1375</span><span id="line-1375"></span> |
| <span class="source-line-no">1376</span><span id="line-1376"> private HFileContext fileContext;</span> |
| <span class="source-line-no">1377</span><span id="line-1377"> // Cache the fileName</span> |
| <span class="source-line-no">1378</span><span id="line-1378"> private String pathName;</span> |
| <span class="source-line-no">1379</span><span id="line-1379"></span> |
| <span class="source-line-no">1380</span><span id="line-1380"> private final ByteBuffAllocator allocator;</span> |
| <span class="source-line-no">1381</span><span id="line-1381"></span> |
| <span class="source-line-no">1382</span><span id="line-1382"> private final Lock streamLock = new ReentrantLock();</span> |
| <span class="source-line-no">1383</span><span id="line-1383"></span> |
| <span class="source-line-no">1384</span><span id="line-1384"> private final boolean isPreadAllBytes;</span> |
| <span class="source-line-no">1385</span><span id="line-1385"></span> |
| <span class="source-line-no">1386</span><span id="line-1386"> private final long readWarnTime;</span> |
| <span class="source-line-no">1387</span><span id="line-1387"></span> |
| <span class="source-line-no">1388</span><span id="line-1388"> /**</span> |
| <span class="source-line-no">1389</span><span id="line-1389"> * If reading block cost time in milliseconds more than the threshold, a warning will be logged.</span> |
| <span class="source-line-no">1390</span><span id="line-1390"> */</span> |
| <span class="source-line-no">1391</span><span id="line-1391"> public static final String FS_READER_WARN_TIME_MS = "hbase.fs.reader.warn.time.ms";</span> |
| <span class="source-line-no">1392</span><span id="line-1392"></span> |
| <span class="source-line-no">1393</span><span id="line-1393"> FSReaderImpl(ReaderContext readerContext, HFileContext fileContext, ByteBuffAllocator allocator,</span> |
| <span class="source-line-no">1394</span><span id="line-1394"> Configuration conf) throws IOException {</span> |
| <span class="source-line-no">1395</span><span id="line-1395"> this.fileSize = readerContext.getFileSize();</span> |
| <span class="source-line-no">1396</span><span id="line-1396"> this.hfs = readerContext.getFileSystem();</span> |
| <span class="source-line-no">1397</span><span id="line-1397"> if (readerContext.getFilePath() != null) {</span> |
| <span class="source-line-no">1398</span><span id="line-1398"> this.pathName = readerContext.getFilePath().toString();</span> |
| <span class="source-line-no">1399</span><span id="line-1399"> }</span> |
| <span class="source-line-no">1400</span><span id="line-1400"> this.fileContext = fileContext;</span> |
| <span class="source-line-no">1401</span><span id="line-1401"> this.hdrSize = headerSize(fileContext.isUseHBaseChecksum());</span> |
| <span class="source-line-no">1402</span><span id="line-1402"> this.allocator = allocator;</span> |
| <span class="source-line-no">1403</span><span id="line-1403"></span> |
| <span class="source-line-no">1404</span><span id="line-1404"> this.streamWrapper = readerContext.getInputStreamWrapper();</span> |
| <span class="source-line-no">1405</span><span id="line-1405"> // Older versions of HBase didn't support checksum.</span> |
| <span class="source-line-no">1406</span><span id="line-1406"> this.streamWrapper.prepareForBlockReader(!fileContext.isUseHBaseChecksum());</span> |
| <span class="source-line-no">1407</span><span id="line-1407"> defaultDecodingCtx = new HFileBlockDefaultDecodingContext(conf, fileContext);</span> |
| <span class="source-line-no">1408</span><span id="line-1408"> encodedBlockDecodingCtx = defaultDecodingCtx;</span> |
| <span class="source-line-no">1409</span><span id="line-1409"> isPreadAllBytes = readerContext.isPreadAllBytes();</span> |
| <span class="source-line-no">1410</span><span id="line-1410"> // Default warn threshold set to -1, it means skipping record the read block slow warning log.</span> |
| <span class="source-line-no">1411</span><span id="line-1411"> readWarnTime = conf.getLong(FS_READER_WARN_TIME_MS, -1L);</span> |
| <span class="source-line-no">1412</span><span id="line-1412"> }</span> |
| <span class="source-line-no">1413</span><span id="line-1413"></span> |
| <span class="source-line-no">1414</span><span id="line-1414"> @Override</span> |
| <span class="source-line-no">1415</span><span id="line-1415"> public BlockIterator blockRange(final long startOffset, final long endOffset) {</span> |
| <span class="source-line-no">1416</span><span id="line-1416"> final FSReader owner = this; // handle for inner class</span> |
| <span class="source-line-no">1417</span><span id="line-1417"> return new BlockIterator() {</span> |
| <span class="source-line-no">1418</span><span id="line-1418"> private volatile boolean freed = false;</span> |
| <span class="source-line-no">1419</span><span id="line-1419"> // Tracking all read blocks until we call freeBlocks.</span> |
| <span class="source-line-no">1420</span><span id="line-1420"> private List<HFileBlock> blockTracker = new ArrayList<>();</span> |
| <span class="source-line-no">1421</span><span id="line-1421"> private long offset = startOffset;</span> |
| <span class="source-line-no">1422</span><span id="line-1422"> // Cache length of next block. Current block has the length of next block in it.</span> |
| <span class="source-line-no">1423</span><span id="line-1423"> private long length = -1;</span> |
| <span class="source-line-no">1424</span><span id="line-1424"></span> |
| <span class="source-line-no">1425</span><span id="line-1425"> @Override</span> |
| <span class="source-line-no">1426</span><span id="line-1426"> public HFileBlock nextBlock() throws IOException {</span> |
| <span class="source-line-no">1427</span><span id="line-1427"> if (offset >= endOffset) {</span> |
| <span class="source-line-no">1428</span><span id="line-1428"> return null;</span> |
| <span class="source-line-no">1429</span><span id="line-1429"> }</span> |
| <span class="source-line-no">1430</span><span id="line-1430"> HFileBlock b = readBlockData(offset, length, false, false, true);</span> |
| <span class="source-line-no">1431</span><span id="line-1431"> offset += b.getOnDiskSizeWithHeader();</span> |
| <span class="source-line-no">1432</span><span id="line-1432"> length = b.getNextBlockOnDiskSize();</span> |
| <span class="source-line-no">1433</span><span id="line-1433"> HFileBlock uncompressed = b.unpack(fileContext, owner);</span> |
| <span class="source-line-no">1434</span><span id="line-1434"> if (uncompressed != b) {</span> |
| <span class="source-line-no">1435</span><span id="line-1435"> b.release(); // Need to release the compressed Block now.</span> |
| <span class="source-line-no">1436</span><span id="line-1436"> }</span> |
| <span class="source-line-no">1437</span><span id="line-1437"> blockTracker.add(uncompressed);</span> |
| <span class="source-line-no">1438</span><span id="line-1438"> return uncompressed;</span> |
| <span class="source-line-no">1439</span><span id="line-1439"> }</span> |
| <span class="source-line-no">1440</span><span id="line-1440"></span> |
| <span class="source-line-no">1441</span><span id="line-1441"> @Override</span> |
| <span class="source-line-no">1442</span><span id="line-1442"> public HFileBlock nextBlockWithBlockType(BlockType blockType) throws IOException {</span> |
| <span class="source-line-no">1443</span><span id="line-1443"> HFileBlock blk = nextBlock();</span> |
| <span class="source-line-no">1444</span><span id="line-1444"> if (blk.getBlockType() != blockType) {</span> |
| <span class="source-line-no">1445</span><span id="line-1445"> throw new IOException(</span> |
| <span class="source-line-no">1446</span><span id="line-1446"> "Expected block of type " + blockType + " but found " + blk.getBlockType());</span> |
| <span class="source-line-no">1447</span><span id="line-1447"> }</span> |
| <span class="source-line-no">1448</span><span id="line-1448"> return blk;</span> |
| <span class="source-line-no">1449</span><span id="line-1449"> }</span> |
| <span class="source-line-no">1450</span><span id="line-1450"></span> |
| <span class="source-line-no">1451</span><span id="line-1451"> @Override</span> |
| <span class="source-line-no">1452</span><span id="line-1452"> public void freeBlocks() {</span> |
| <span class="source-line-no">1453</span><span id="line-1453"> if (freed) {</span> |
| <span class="source-line-no">1454</span><span id="line-1454"> return;</span> |
| <span class="source-line-no">1455</span><span id="line-1455"> }</span> |
| <span class="source-line-no">1456</span><span id="line-1456"> blockTracker.forEach(HFileBlock::release);</span> |
| <span class="source-line-no">1457</span><span id="line-1457"> blockTracker = null;</span> |
| <span class="source-line-no">1458</span><span id="line-1458"> freed = true;</span> |
| <span class="source-line-no">1459</span><span id="line-1459"> }</span> |
| <span class="source-line-no">1460</span><span id="line-1460"> };</span> |
| <span class="source-line-no">1461</span><span id="line-1461"> }</span> |
| <span class="source-line-no">1462</span><span id="line-1462"></span> |
| <span class="source-line-no">1463</span><span id="line-1463"> /**</span> |
| <span class="source-line-no">1464</span><span id="line-1464"> * Does a positional read or a seek and read into the given byte buffer. We need take care that</span> |
| <span class="source-line-no">1465</span><span id="line-1465"> * we will call the {@link ByteBuff#release()} for every exit to deallocate the ByteBuffers,</span> |
| <span class="source-line-no">1466</span><span id="line-1466"> * otherwise the memory leak may happen.</span> |
| <span class="source-line-no">1467</span><span id="line-1467"> * @param dest destination buffer</span> |
| <span class="source-line-no">1468</span><span id="line-1468"> * @param size size of read</span> |
| <span class="source-line-no">1469</span><span id="line-1469"> * @param peekIntoNextBlock whether to read the next block's on-disk size</span> |
| <span class="source-line-no">1470</span><span id="line-1470"> * @param fileOffset position in the stream to read at</span> |
| <span class="source-line-no">1471</span><span id="line-1471"> * @param pread whether we should do a positional read</span> |
| <span class="source-line-no">1472</span><span id="line-1472"> * @param istream The input source of data</span> |
| <span class="source-line-no">1473</span><span id="line-1473"> * @return true to indicate the destination buffer include the next block header, otherwise only</span> |
| <span class="source-line-no">1474</span><span id="line-1474"> * include the current block data without the next block header.</span> |
| <span class="source-line-no">1475</span><span id="line-1475"> * @throws IOException if any IO error happen.</span> |
| <span class="source-line-no">1476</span><span id="line-1476"> */</span> |
| <span class="source-line-no">1477</span><span id="line-1477"> protected boolean readAtOffset(FSDataInputStream istream, ByteBuff dest, int size,</span> |
| <span class="source-line-no">1478</span><span id="line-1478"> boolean peekIntoNextBlock, long fileOffset, boolean pread) throws IOException {</span> |
| <span class="source-line-no">1479</span><span id="line-1479"> if (!pread) {</span> |
| <span class="source-line-no">1480</span><span id="line-1480"> // Seek + read. Better for scanning.</span> |
| <span class="source-line-no">1481</span><span id="line-1481"> istream.seek(fileOffset);</span> |
| <span class="source-line-no">1482</span><span id="line-1482"> long realOffset = istream.getPos();</span> |
| <span class="source-line-no">1483</span><span id="line-1483"> if (realOffset != fileOffset) {</span> |
| <span class="source-line-no">1484</span><span id="line-1484"> throw new IOException("Tried to seek to " + fileOffset + " to read " + size</span> |
| <span class="source-line-no">1485</span><span id="line-1485"> + " bytes, but pos=" + realOffset + " after seek");</span> |
| <span class="source-line-no">1486</span><span id="line-1486"> }</span> |
| <span class="source-line-no">1487</span><span id="line-1487"> if (!peekIntoNextBlock) {</span> |
| <span class="source-line-no">1488</span><span id="line-1488"> BlockIOUtils.readFully(dest, istream, size);</span> |
| <span class="source-line-no">1489</span><span id="line-1489"> return false;</span> |
| <span class="source-line-no">1490</span><span id="line-1490"> }</span> |
| <span class="source-line-no">1491</span><span id="line-1491"></span> |
| <span class="source-line-no">1492</span><span id="line-1492"> // Try to read the next block header</span> |
| <span class="source-line-no">1493</span><span id="line-1493"> if (!BlockIOUtils.readWithExtra(dest, istream, size, hdrSize)) {</span> |
| <span class="source-line-no">1494</span><span id="line-1494"> // did not read the next block header.</span> |
| <span class="source-line-no">1495</span><span id="line-1495"> return false;</span> |
| <span class="source-line-no">1496</span><span id="line-1496"> }</span> |
| <span class="source-line-no">1497</span><span id="line-1497"> } else {</span> |
| <span class="source-line-no">1498</span><span id="line-1498"> // Positional read. Better for random reads; or when the streamLock is already locked.</span> |
| <span class="source-line-no">1499</span><span id="line-1499"> int extraSize = peekIntoNextBlock ? hdrSize : 0;</span> |
| <span class="source-line-no">1500</span><span id="line-1500"> if (</span> |
| <span class="source-line-no">1501</span><span id="line-1501"> !BlockIOUtils.preadWithExtra(dest, istream, fileOffset, size, extraSize, isPreadAllBytes)</span> |
| <span class="source-line-no">1502</span><span id="line-1502"> ) {</span> |
| <span class="source-line-no">1503</span><span id="line-1503"> // did not read the next block header.</span> |
| <span class="source-line-no">1504</span><span id="line-1504"> return false;</span> |
| <span class="source-line-no">1505</span><span id="line-1505"> }</span> |
| <span class="source-line-no">1506</span><span id="line-1506"> }</span> |
| <span class="source-line-no">1507</span><span id="line-1507"> assert peekIntoNextBlock;</span> |
| <span class="source-line-no">1508</span><span id="line-1508"> return true;</span> |
| <span class="source-line-no">1509</span><span id="line-1509"> }</span> |
| <span class="source-line-no">1510</span><span id="line-1510"></span> |
| <span class="source-line-no">1511</span><span id="line-1511"> /**</span> |
| <span class="source-line-no">1512</span><span id="line-1512"> * Reads a version 2 block (version 1 blocks not supported and not expected). Tries to do as</span> |
| <span class="source-line-no">1513</span><span id="line-1513"> * little memory allocation as possible, using the provided on-disk size.</span> |
| <span class="source-line-no">1514</span><span id="line-1514"> * @param offset the offset in the stream to read at</span> |
| <span class="source-line-no">1515</span><span id="line-1515"> * @param onDiskSizeWithHeaderL the on-disk size of the block, including the header, or -1 if</span> |
| <span class="source-line-no">1516</span><span id="line-1516"> * unknown; i.e. when iterating over blocks reading in the file</span> |
| <span class="source-line-no">1517</span><span id="line-1517"> * metadata info.</span> |
| <span class="source-line-no">1518</span><span id="line-1518"> * @param pread whether to use a positional read</span> |
| <span class="source-line-no">1519</span><span id="line-1519"> * @param updateMetrics whether to update the metrics</span> |
| <span class="source-line-no">1520</span><span id="line-1520"> * @param intoHeap allocate ByteBuff of block from heap or off-heap.</span> |
| <span class="source-line-no">1521</span><span id="line-1521"> * @see FSReader#readBlockData(long, long, boolean, boolean, boolean) for more details about the</span> |
| <span class="source-line-no">1522</span><span id="line-1522"> * useHeap.</span> |
| <span class="source-line-no">1523</span><span id="line-1523"> */</span> |
| <span class="source-line-no">1524</span><span id="line-1524"> @Override</span> |
| <span class="source-line-no">1525</span><span id="line-1525"> public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL, boolean pread,</span> |
| <span class="source-line-no">1526</span><span id="line-1526"> boolean updateMetrics, boolean intoHeap) throws IOException {</span> |
| <span class="source-line-no">1527</span><span id="line-1527"> // Get a copy of the current state of whether to validate</span> |
| <span class="source-line-no">1528</span><span id="line-1528"> // hbase checksums or not for this read call. This is not</span> |
| <span class="source-line-no">1529</span><span id="line-1529"> // thread-safe but the one constraint is that if we decide</span> |
| <span class="source-line-no">1530</span><span id="line-1530"> // to skip hbase checksum verification then we are</span> |
| <span class="source-line-no">1531</span><span id="line-1531"> // guaranteed to use hdfs checksum verification.</span> |
| <span class="source-line-no">1532</span><span id="line-1532"> boolean doVerificationThruHBaseChecksum = streamWrapper.shouldUseHBaseChecksum();</span> |
| <span class="source-line-no">1533</span><span id="line-1533"> FSDataInputStream is = streamWrapper.getStream(doVerificationThruHBaseChecksum);</span> |
| <span class="source-line-no">1534</span><span id="line-1534"> final Context context = Context.current().with(CONTEXT_KEY,</span> |
| <span class="source-line-no">1535</span><span id="line-1535"> new HFileContextAttributesBuilderConsumer(fileContext)</span> |
| <span class="source-line-no">1536</span><span id="line-1536"> .setSkipChecksum(doVerificationThruHBaseChecksum)</span> |
| <span class="source-line-no">1537</span><span id="line-1537"> .setReadType(pread ? ReadType.POSITIONAL_READ : ReadType.SEEK_PLUS_READ));</span> |
| <span class="source-line-no">1538</span><span id="line-1538"> try (Scope ignored = context.makeCurrent()) {</span> |
| <span class="source-line-no">1539</span><span id="line-1539"> HFileBlock blk = readBlockDataInternal(is, offset, onDiskSizeWithHeaderL, pread,</span> |
| <span class="source-line-no">1540</span><span id="line-1540"> doVerificationThruHBaseChecksum, updateMetrics, intoHeap);</span> |
| <span class="source-line-no">1541</span><span id="line-1541"> if (blk == null) {</span> |
| <span class="source-line-no">1542</span><span id="line-1542"> HFile.LOG.warn("HBase checksum verification failed for file {} at offset {} filesize {}."</span> |
| <span class="source-line-no">1543</span><span id="line-1543"> + " Retrying read with HDFS checksums turned on...", pathName, offset, fileSize);</span> |
| <span class="source-line-no">1544</span><span id="line-1544"></span> |
| <span class="source-line-no">1545</span><span id="line-1545"> if (!doVerificationThruHBaseChecksum) {</span> |
| <span class="source-line-no">1546</span><span id="line-1546"> String msg = "HBase checksum verification failed for file " + pathName + " at offset "</span> |
| <span class="source-line-no">1547</span><span id="line-1547"> + offset + " filesize " + fileSize + " but this cannot happen because doVerify is "</span> |
| <span class="source-line-no">1548</span><span id="line-1548"> + doVerificationThruHBaseChecksum;</span> |
| <span class="source-line-no">1549</span><span id="line-1549"> HFile.LOG.warn(msg);</span> |
| <span class="source-line-no">1550</span><span id="line-1550"> throw new IOException(msg); // cannot happen case here</span> |
| <span class="source-line-no">1551</span><span id="line-1551"> }</span> |
| <span class="source-line-no">1552</span><span id="line-1552"> HFile.CHECKSUM_FAILURES.increment(); // update metrics</span> |
| <span class="source-line-no">1553</span><span id="line-1553"></span> |
| <span class="source-line-no">1554</span><span id="line-1554"> // If we have a checksum failure, we fall back into a mode where</span> |
| <span class="source-line-no">1555</span><span id="line-1555"> // the next few reads use HDFS level checksums. We aim to make the</span> |
| <span class="source-line-no">1556</span><span id="line-1556"> // next CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD reads avoid</span> |
| <span class="source-line-no">1557</span><span id="line-1557"> // hbase checksum verification, but since this value is set without</span> |
| <span class="source-line-no">1558</span><span id="line-1558"> // holding any locks, it can so happen that we might actually do</span> |
| <span class="source-line-no">1559</span><span id="line-1559"> // a few more than precisely this number.</span> |
| <span class="source-line-no">1560</span><span id="line-1560"> is = this.streamWrapper.fallbackToFsChecksum(CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD);</span> |
| <span class="source-line-no">1561</span><span id="line-1561"> doVerificationThruHBaseChecksum = false;</span> |
| <span class="source-line-no">1562</span><span id="line-1562"> blk = readBlockDataInternal(is, offset, onDiskSizeWithHeaderL, pread,</span> |
| <span class="source-line-no">1563</span><span id="line-1563"> doVerificationThruHBaseChecksum, updateMetrics, intoHeap);</span> |
| <span class="source-line-no">1564</span><span id="line-1564"> if (blk != null) {</span> |
| <span class="source-line-no">1565</span><span id="line-1565"> HFile.LOG.warn(</span> |
| <span class="source-line-no">1566</span><span id="line-1566"> "HDFS checksum verification succeeded for file {} at offset {} filesize" + " {}",</span> |
| <span class="source-line-no">1567</span><span id="line-1567"> pathName, offset, fileSize);</span> |
| <span class="source-line-no">1568</span><span id="line-1568"> }</span> |
| <span class="source-line-no">1569</span><span id="line-1569"> }</span> |
| <span class="source-line-no">1570</span><span id="line-1570"> if (blk == null && !doVerificationThruHBaseChecksum) {</span> |
| <span class="source-line-no">1571</span><span id="line-1571"> String msg =</span> |
| <span class="source-line-no">1572</span><span id="line-1572"> "readBlockData failed, possibly due to " + "checksum verification failed for file "</span> |
| <span class="source-line-no">1573</span><span id="line-1573"> + pathName + " at offset " + offset + " filesize " + fileSize;</span> |
| <span class="source-line-no">1574</span><span id="line-1574"> HFile.LOG.warn(msg);</span> |
| <span class="source-line-no">1575</span><span id="line-1575"> throw new IOException(msg);</span> |
| <span class="source-line-no">1576</span><span id="line-1576"> }</span> |
| <span class="source-line-no">1577</span><span id="line-1577"></span> |
| <span class="source-line-no">1578</span><span id="line-1578"> // If there is a checksum mismatch earlier, then retry with</span> |
| <span class="source-line-no">1579</span><span id="line-1579"> // HBase checksums switched off and use HDFS checksum verification.</span> |
| <span class="source-line-no">1580</span><span id="line-1580"> // This triggers HDFS to detect and fix corrupt replicas. The</span> |
| <span class="source-line-no">1581</span><span id="line-1581"> // next checksumOffCount read requests will use HDFS checksums.</span> |
| <span class="source-line-no">1582</span><span id="line-1582"> // The decrementing of this.checksumOffCount is not thread-safe,</span> |
| <span class="source-line-no">1583</span><span id="line-1583"> // but it is harmless because eventually checksumOffCount will be</span> |
| <span class="source-line-no">1584</span><span id="line-1584"> // a negative number.</span> |
| <span class="source-line-no">1585</span><span id="line-1585"> streamWrapper.checksumOk();</span> |
| <span class="source-line-no">1586</span><span id="line-1586"> return blk;</span> |
| <span class="source-line-no">1587</span><span id="line-1587"> }</span> |
| <span class="source-line-no">1588</span><span id="line-1588"> }</span> |
| <span class="source-line-no">1589</span><span id="line-1589"></span> |
| <span class="source-line-no">1590</span><span id="line-1590"> /**</span> |
| <span class="source-line-no">1591</span><span id="line-1591"> * Check that {@code value} read from a block header seems reasonable, within a large margin of</span> |
| <span class="source-line-no">1592</span><span id="line-1592"> * error.</span> |
| <span class="source-line-no">1593</span><span id="line-1593"> * @return {@code true} if the value is safe to proceed, {@code false} otherwise.</span> |
| <span class="source-line-no">1594</span><span id="line-1594"> */</span> |
| <span class="source-line-no">1595</span><span id="line-1595"> private boolean checkOnDiskSizeWithHeader(int value) {</span> |
| <span class="source-line-no">1596</span><span id="line-1596"> if (value < 0) {</span> |
| <span class="source-line-no">1597</span><span id="line-1597"> if (LOG.isTraceEnabled()) {</span> |
| <span class="source-line-no">1598</span><span id="line-1598"> LOG.trace(</span> |
| <span class="source-line-no">1599</span><span id="line-1599"> "onDiskSizeWithHeader={}; value represents a size, so it should never be negative.",</span> |
| <span class="source-line-no">1600</span><span id="line-1600"> value);</span> |
| <span class="source-line-no">1601</span><span id="line-1601"> }</span> |
| <span class="source-line-no">1602</span><span id="line-1602"> return false;</span> |
| <span class="source-line-no">1603</span><span id="line-1603"> }</span> |
| <span class="source-line-no">1604</span><span id="line-1604"> if (value - hdrSize < 0) {</span> |
| <span class="source-line-no">1605</span><span id="line-1605"> if (LOG.isTraceEnabled()) {</span> |
| <span class="source-line-no">1606</span><span id="line-1606"> LOG.trace("onDiskSizeWithHeader={}, hdrSize={}; don't accept a value that is negative"</span> |
| <span class="source-line-no">1607</span><span id="line-1607"> + " after the header size is excluded.", value, hdrSize);</span> |
| <span class="source-line-no">1608</span><span id="line-1608"> }</span> |
| <span class="source-line-no">1609</span><span id="line-1609"> return false;</span> |
| <span class="source-line-no">1610</span><span id="line-1610"> }</span> |
| <span class="source-line-no">1611</span><span id="line-1611"> return true;</span> |
| <span class="source-line-no">1612</span><span id="line-1612"> }</span> |
| <span class="source-line-no">1613</span><span id="line-1613"></span> |
| <span class="source-line-no">1614</span><span id="line-1614"> /**</span> |
| <span class="source-line-no">1615</span><span id="line-1615"> * Check that {@code value} provided by the calling context seems reasonable, within a large</span> |
| <span class="source-line-no">1616</span><span id="line-1616"> * margin of error.</span> |
| <span class="source-line-no">1617</span><span id="line-1617"> * @return {@code true} if the value is safe to proceed, {@code false} otherwise.</span> |
| <span class="source-line-no">1618</span><span id="line-1618"> */</span> |
| <span class="source-line-no">1619</span><span id="line-1619"> private boolean checkCallerProvidedOnDiskSizeWithHeader(long value) {</span> |
| <span class="source-line-no">1620</span><span id="line-1620"> // same validation logic as is used by Math.toIntExact(long)</span> |
| <span class="source-line-no">1621</span><span id="line-1621"> int intValue = (int) value;</span> |
| <span class="source-line-no">1622</span><span id="line-1622"> if (intValue != value) {</span> |
| <span class="source-line-no">1623</span><span id="line-1623"> if (LOG.isTraceEnabled()) {</span> |
| <span class="source-line-no">1624</span><span id="line-1624"> LOG.trace("onDiskSizeWithHeaderL={}; value exceeds int size limits.", value);</span> |
| <span class="source-line-no">1625</span><span id="line-1625"> }</span> |
| <span class="source-line-no">1626</span><span id="line-1626"> return false;</span> |
| <span class="source-line-no">1627</span><span id="line-1627"> }</span> |
| <span class="source-line-no">1628</span><span id="line-1628"> if (intValue == -1) {</span> |
| <span class="source-line-no">1629</span><span id="line-1629"> // a magic value we expect to see.</span> |
| <span class="source-line-no">1630</span><span id="line-1630"> return true;</span> |
| <span class="source-line-no">1631</span><span id="line-1631"> }</span> |
| <span class="source-line-no">1632</span><span id="line-1632"> return checkOnDiskSizeWithHeader(intValue);</span> |
| <span class="source-line-no">1633</span><span id="line-1633"> }</span> |
| <span class="source-line-no">1634</span><span id="line-1634"></span> |
| <span class="source-line-no">1635</span><span id="line-1635"> /**</span> |
| <span class="source-line-no">1636</span><span id="line-1636"> * Check atomic reference cache for this block's header. Cache only good if next read coming</span> |
| <span class="source-line-no">1637</span><span id="line-1637"> * through is next in sequence in the block. We read next block's header on the tail of reading</span> |
| <span class="source-line-no">1638</span><span id="line-1638"> * the previous block to save a seek. Otherwise, we have to do a seek to read the header before</span> |
| <span class="source-line-no">1639</span><span id="line-1639"> * we can pull in the block OR we have to backup the stream because we over-read (the next</span> |
| <span class="source-line-no">1640</span><span id="line-1640"> * block's header).</span> |
| <span class="source-line-no">1641</span><span id="line-1641"> * @see PrefetchedHeader</span> |
| <span class="source-line-no">1642</span><span id="line-1642"> * @return The cached block header or null if not found.</span> |
| <span class="source-line-no">1643</span><span id="line-1643"> * @see #cacheNextBlockHeader(long, ByteBuff, int, int)</span> |
| <span class="source-line-no">1644</span><span id="line-1644"> */</span> |
| <span class="source-line-no">1645</span><span id="line-1645"> private ByteBuff getCachedHeader(final long offset) {</span> |
| <span class="source-line-no">1646</span><span id="line-1646"> PrefetchedHeader ph = this.prefetchedHeader.get();</span> |
| <span class="source-line-no">1647</span><span id="line-1647"> return ph != null && ph.offset == offset ? ph.buf : null;</span> |
| <span class="source-line-no">1648</span><span id="line-1648"> }</span> |
| <span class="source-line-no">1649</span><span id="line-1649"></span> |
| <span class="source-line-no">1650</span><span id="line-1650"> /**</span> |
| <span class="source-line-no">1651</span><span id="line-1651"> * Save away the next blocks header in atomic reference.</span> |
| <span class="source-line-no">1652</span><span id="line-1652"> * @see #getCachedHeader(long)</span> |
| <span class="source-line-no">1653</span><span id="line-1653"> * @see PrefetchedHeader</span> |
| <span class="source-line-no">1654</span><span id="line-1654"> */</span> |
| <span class="source-line-no">1655</span><span id="line-1655"> private void cacheNextBlockHeader(final long offset, ByteBuff onDiskBlock,</span> |
| <span class="source-line-no">1656</span><span id="line-1656"> int onDiskSizeWithHeader, int headerLength) {</span> |
| <span class="source-line-no">1657</span><span id="line-1657"> PrefetchedHeader ph = new PrefetchedHeader();</span> |
| <span class="source-line-no">1658</span><span id="line-1658"> ph.offset = offset;</span> |
| <span class="source-line-no">1659</span><span id="line-1659"> onDiskBlock.get(onDiskSizeWithHeader, ph.header, 0, headerLength);</span> |
| <span class="source-line-no">1660</span><span id="line-1660"> this.prefetchedHeader.set(ph);</span> |
| <span class="source-line-no">1661</span><span id="line-1661"> }</span> |
| <span class="source-line-no">1662</span><span id="line-1662"></span> |
| <span class="source-line-no">1663</span><span id="line-1663"> /**</span> |
| <span class="source-line-no">1664</span><span id="line-1664"> * Clear the cached value when its integrity is suspect.</span> |
| <span class="source-line-no">1665</span><span id="line-1665"> */</span> |
| <span class="source-line-no">1666</span><span id="line-1666"> private void invalidateNextBlockHeader() {</span> |
| <span class="source-line-no">1667</span><span id="line-1667"> prefetchedHeader.set(null);</span> |
| <span class="source-line-no">1668</span><span id="line-1668"> }</span> |
| <span class="source-line-no">1669</span><span id="line-1669"></span> |
| <span class="source-line-no">1670</span><span id="line-1670"> private int getNextBlockOnDiskSize(ByteBuff onDiskBlock, int onDiskSizeWithHeader) {</span> |
| <span class="source-line-no">1671</span><span id="line-1671"> return onDiskBlock.getIntAfterPosition(onDiskSizeWithHeader + BlockType.MAGIC_LENGTH)</span> |
| <span class="source-line-no">1672</span><span id="line-1672"> + hdrSize;</span> |
| <span class="source-line-no">1673</span><span id="line-1673"> }</span> |
| <span class="source-line-no">1674</span><span id="line-1674"></span> |
| <span class="source-line-no">1675</span><span id="line-1675"> private ByteBuff allocate(int size, boolean intoHeap) {</span> |
| <span class="source-line-no">1676</span><span id="line-1676"> return intoHeap ? HEAP.allocate(size) : allocator.allocate(size);</span> |
| <span class="source-line-no">1677</span><span id="line-1677"> }</span> |
| <span class="source-line-no">1678</span><span id="line-1678"></span> |
| <span class="source-line-no">1679</span><span id="line-1679"> /**</span> |
| <span class="source-line-no">1680</span><span id="line-1680"> * Reads a version 2 block.</span> |
| <span class="source-line-no">1681</span><span id="line-1681"> * @param offset the offset in the stream to read at.</span> |
| <span class="source-line-no">1682</span><span id="line-1682"> * @param onDiskSizeWithHeaderL the on-disk size of the block, including the header and</span> |
| <span class="source-line-no">1683</span><span id="line-1683"> * checksums if present or -1 if unknown (as a long). Can be -1 if</span> |
| <span class="source-line-no">1684</span><span id="line-1684"> * we are doing raw iteration of blocks as when loading up file</span> |
| <span class="source-line-no">1685</span><span id="line-1685"> * metadata; i.e. the first read of a new file. Usually non-null</span> |
| <span class="source-line-no">1686</span><span id="line-1686"> * gotten from the file index.</span> |
| <span class="source-line-no">1687</span><span id="line-1687"> * @param pread whether to use a positional read</span> |
| <span class="source-line-no">1688</span><span id="line-1688"> * @param verifyChecksum Whether to use HBase checksums. If HBase checksum is switched</span> |
| <span class="source-line-no">1689</span><span id="line-1689"> * off, then use HDFS checksum. Can also flip on/off reading same</span> |
| <span class="source-line-no">1690</span><span id="line-1690"> * file if we hit a troublesome patch in an hfile.</span> |
| <span class="source-line-no">1691</span><span id="line-1691"> * @param updateMetrics whether need to update the metrics.</span> |
| <span class="source-line-no">1692</span><span id="line-1692"> * @param intoHeap allocate the ByteBuff of block from heap or off-heap.</span> |
| <span class="source-line-no">1693</span><span id="line-1693"> * @return the HFileBlock or null if there is a HBase checksum mismatch</span> |
| <span class="source-line-no">1694</span><span id="line-1694"> */</span> |
| <span class="source-line-no">1695</span><span id="line-1695"> protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset,</span> |
| <span class="source-line-no">1696</span><span id="line-1696"> long onDiskSizeWithHeaderL, boolean pread, boolean verifyChecksum, boolean updateMetrics,</span> |
| <span class="source-line-no">1697</span><span id="line-1697"> boolean intoHeap) throws IOException {</span> |
| <span class="source-line-no">1698</span><span id="line-1698"> final Span span = Span.current();</span> |
| <span class="source-line-no">1699</span><span id="line-1699"> final AttributesBuilder attributesBuilder = Attributes.builder();</span> |
| <span class="source-line-no">1700</span><span id="line-1700"> Optional.of(Context.current()).map(val -> val.get(CONTEXT_KEY))</span> |
| <span class="source-line-no">1701</span><span id="line-1701"> .ifPresent(c -> c.accept(attributesBuilder));</span> |
| <span class="source-line-no">1702</span><span id="line-1702"> if (offset < 0) {</span> |
| <span class="source-line-no">1703</span><span id="line-1703"> throw new IOException("Invalid offset=" + offset + " trying to read " + "block (onDiskSize="</span> |
| <span class="source-line-no">1704</span><span id="line-1704"> + onDiskSizeWithHeaderL + ")");</span> |
| <span class="source-line-no">1705</span><span id="line-1705"> }</span> |
| <span class="source-line-no">1706</span><span id="line-1706"> if (!checkCallerProvidedOnDiskSizeWithHeader(onDiskSizeWithHeaderL)) {</span> |
| <span class="source-line-no">1707</span><span id="line-1707"> LOG.trace("Caller provided invalid onDiskSizeWithHeaderL={}", onDiskSizeWithHeaderL);</span> |
| <span class="source-line-no">1708</span><span id="line-1708"> onDiskSizeWithHeaderL = -1;</span> |
| <span class="source-line-no">1709</span><span id="line-1709"> }</span> |
| <span class="source-line-no">1710</span><span id="line-1710"> int onDiskSizeWithHeader = (int) onDiskSizeWithHeaderL;</span> |
| <span class="source-line-no">1711</span><span id="line-1711"></span> |
| <span class="source-line-no">1712</span><span id="line-1712"> // Try to use the cached header. Will serve us in rare case where onDiskSizeWithHeaderL==-1</span> |
| <span class="source-line-no">1713</span><span id="line-1713"> // and will save us having to seek the stream backwards to reread the header we</span> |
| <span class="source-line-no">1714</span><span id="line-1714"> // read the last time through here.</span> |
| <span class="source-line-no">1715</span><span id="line-1715"> ByteBuff headerBuf = getCachedHeader(offset);</span> |
| <span class="source-line-no">1716</span><span id="line-1716"> LOG.trace(</span> |
| <span class="source-line-no">1717</span><span id="line-1717"> "Reading {} at offset={}, pread={}, verifyChecksum={}, cachedHeader={}, "</span> |
| <span class="source-line-no">1718</span><span id="line-1718"> + "onDiskSizeWithHeader={}",</span> |
| <span class="source-line-no">1719</span><span id="line-1719"> this.fileContext.getHFileName(), offset, pread, verifyChecksum, headerBuf,</span> |
| <span class="source-line-no">1720</span><span id="line-1720"> onDiskSizeWithHeader);</span> |
| <span class="source-line-no">1721</span><span id="line-1721"> // This is NOT same as verifyChecksum. This latter is whether to do hbase</span> |
| <span class="source-line-no">1722</span><span id="line-1722"> // checksums. Can change with circumstances. The below flag is whether the</span> |
| <span class="source-line-no">1723</span><span id="line-1723"> // file has support for checksums (version 2+).</span> |
| <span class="source-line-no">1724</span><span id="line-1724"> boolean checksumSupport = this.fileContext.isUseHBaseChecksum();</span> |
| <span class="source-line-no">1725</span><span id="line-1725"> long startTime = EnvironmentEdgeManager.currentTime();</span> |
| <span class="source-line-no">1726</span><span id="line-1726"> if (onDiskSizeWithHeader == -1) {</span> |
| <span class="source-line-no">1727</span><span id="line-1727"> // The caller does not know the block size. Need to get it from the header. If header was</span> |
| <span class="source-line-no">1728</span><span id="line-1728"> // not cached (see getCachedHeader above), need to seek to pull it in. This is costly</span> |
| <span class="source-line-no">1729</span><span id="line-1729"> // and should happen very rarely. Currently happens on open of a hfile reader where we</span> |
| <span class="source-line-no">1730</span><span id="line-1730"> // read the trailer blocks to pull in the indices. Otherwise, we are reading block sizes</span> |
| <span class="source-line-no">1731</span><span id="line-1731"> // out of the hfile index. To check, enable TRACE in this file and you'll get an exception</span> |
| <span class="source-line-no">1732</span><span id="line-1732"> // in a LOG every time we seek. See HBASE-17072 for more detail.</span> |
| <span class="source-line-no">1733</span><span id="line-1733"> if (headerBuf == null) {</span> |
| <span class="source-line-no">1734</span><span id="line-1734"> if (LOG.isTraceEnabled()) {</span> |
| <span class="source-line-no">1735</span><span id="line-1735"> LOG.trace("Extra seek to get block size!", new RuntimeException());</span> |
| <span class="source-line-no">1736</span><span id="line-1736"> }</span> |
| <span class="source-line-no">1737</span><span id="line-1737"> span.addEvent("Extra seek to get block size!", attributesBuilder.build());</span> |
| <span class="source-line-no">1738</span><span id="line-1738"> headerBuf = HEAP.allocate(hdrSize);</span> |
| <span class="source-line-no">1739</span><span id="line-1739"> readAtOffset(is, headerBuf, hdrSize, false, offset, pread);</span> |
| <span class="source-line-no">1740</span><span id="line-1740"> headerBuf.rewind();</span> |
| <span class="source-line-no">1741</span><span id="line-1741"> }</span> |
| <span class="source-line-no">1742</span><span id="line-1742"> onDiskSizeWithHeader = getOnDiskSizeWithHeader(headerBuf, checksumSupport);</span> |
| <span class="source-line-no">1743</span><span id="line-1743"> }</span> |
| <span class="source-line-no">1744</span><span id="line-1744"></span> |
| <span class="source-line-no">1745</span><span id="line-1745"> // The common case is that onDiskSizeWithHeader was produced by a read without checksum</span> |
| <span class="source-line-no">1746</span><span id="line-1746"> // validation, so give it a sanity check before trying to use it.</span> |
| <span class="source-line-no">1747</span><span id="line-1747"> if (!checkOnDiskSizeWithHeader(onDiskSizeWithHeader)) {</span> |
| <span class="source-line-no">1748</span><span id="line-1748"> if (verifyChecksum) {</span> |
| <span class="source-line-no">1749</span><span id="line-1749"> invalidateNextBlockHeader();</span> |
| <span class="source-line-no">1750</span><span id="line-1750"> span.addEvent("Falling back to HDFS checksumming.", attributesBuilder.build());</span> |
| <span class="source-line-no">1751</span><span id="line-1751"> return null;</span> |
| <span class="source-line-no">1752</span><span id="line-1752"> } else {</span> |
| <span class="source-line-no">1753</span><span id="line-1753"> throw new IOException("Invalid onDiskSizeWithHeader=" + onDiskSizeWithHeader);</span> |
| <span class="source-line-no">1754</span><span id="line-1754"> }</span> |
| <span class="source-line-no">1755</span><span id="line-1755"> }</span> |
| <span class="source-line-no">1756</span><span id="line-1756"></span> |
| <span class="source-line-no">1757</span><span id="line-1757"> int preReadHeaderSize = headerBuf == null ? 0 : hdrSize;</span> |
| <span class="source-line-no">1758</span><span id="line-1758"> // Allocate enough space to fit the next block's header too; saves a seek next time through.</span> |
| <span class="source-line-no">1759</span><span id="line-1759"> // onDiskBlock is whole block + header + checksums then extra hdrSize to read next header;</span> |
| <span class="source-line-no">1760</span><span id="line-1760"> // onDiskSizeWithHeader is header, body, and any checksums if present. preReadHeaderSize</span> |
| <span class="source-line-no">1761</span><span id="line-1761"> // says where to start reading. If we have the header cached, then we don't need to read</span> |
| <span class="source-line-no">1762</span><span id="line-1762"> // it again and we can likely read from last place we left off w/o need to backup and reread</span> |
| <span class="source-line-no">1763</span><span id="line-1763"> // the header we read last time through here.</span> |
| <span class="source-line-no">1764</span><span id="line-1764"> ByteBuff onDiskBlock = this.allocate(onDiskSizeWithHeader + hdrSize, intoHeap);</span> |
| <span class="source-line-no">1765</span><span id="line-1765"> boolean initHFileBlockSuccess = false;</span> |
| <span class="source-line-no">1766</span><span id="line-1766"> try {</span> |
| <span class="source-line-no">1767</span><span id="line-1767"> if (headerBuf != null) {</span> |
| <span class="source-line-no">1768</span><span id="line-1768"> onDiskBlock.put(0, headerBuf, 0, hdrSize).position(hdrSize);</span> |
| <span class="source-line-no">1769</span><span id="line-1769"> }</span> |
| <span class="source-line-no">1770</span><span id="line-1770"> boolean readNextHeader = readAtOffset(is, onDiskBlock,</span> |
| <span class="source-line-no">1771</span><span id="line-1771"> onDiskSizeWithHeader - preReadHeaderSize, true, offset + preReadHeaderSize, pread);</span> |
| <span class="source-line-no">1772</span><span id="line-1772"> onDiskBlock.rewind(); // in case of moving position when copying a cached header</span> |
| <span class="source-line-no">1773</span><span id="line-1773"></span> |
| <span class="source-line-no">1774</span><span id="line-1774"> // the call to validateChecksum for this block excludes the next block header over-read, so</span> |
| <span class="source-line-no">1775</span><span id="line-1775"> // no reason to delay extracting this value.</span> |
| <span class="source-line-no">1776</span><span id="line-1776"> int nextBlockOnDiskSize = -1;</span> |
| <span class="source-line-no">1777</span><span id="line-1777"> if (readNextHeader) {</span> |
| <span class="source-line-no">1778</span><span id="line-1778"> int parsedVal = getNextBlockOnDiskSize(onDiskBlock, onDiskSizeWithHeader);</span> |
| <span class="source-line-no">1779</span><span id="line-1779"> if (checkOnDiskSizeWithHeader(parsedVal)) {</span> |
| <span class="source-line-no">1780</span><span id="line-1780"> nextBlockOnDiskSize = parsedVal;</span> |
| <span class="source-line-no">1781</span><span id="line-1781"> }</span> |
| <span class="source-line-no">1782</span><span id="line-1782"> }</span> |
| <span class="source-line-no">1783</span><span id="line-1783"> if (headerBuf == null) {</span> |
| <span class="source-line-no">1784</span><span id="line-1784"> headerBuf = onDiskBlock.duplicate().position(0).limit(hdrSize);</span> |
| <span class="source-line-no">1785</span><span id="line-1785"> }</span> |
| <span class="source-line-no">1786</span><span id="line-1786"></span> |
| <span class="source-line-no">1787</span><span id="line-1787"> ByteBuff curBlock = onDiskBlock.duplicate().position(0).limit(onDiskSizeWithHeader);</span> |
| <span class="source-line-no">1788</span><span id="line-1788"> // Verify checksum of the data before using it for building HFileBlock.</span> |
| <span class="source-line-no">1789</span><span id="line-1789"> if (verifyChecksum && !validateChecksum(offset, curBlock, hdrSize)) {</span> |
| <span class="source-line-no">1790</span><span id="line-1790"> invalidateNextBlockHeader();</span> |
| <span class="source-line-no">1791</span><span id="line-1791"> span.addEvent("Falling back to HDFS checksumming.", attributesBuilder.build());</span> |
| <span class="source-line-no">1792</span><span id="line-1792"> return null;</span> |
| <span class="source-line-no">1793</span><span id="line-1793"> }</span> |
| <span class="source-line-no">1794</span><span id="line-1794"></span> |
| <span class="source-line-no">1795</span><span id="line-1795"> // TODO: is this check necessary or can we proceed with a provided value regardless of</span> |
| <span class="source-line-no">1796</span><span id="line-1796"> // what is in the header?</span> |
| <span class="source-line-no">1797</span><span id="line-1797"> int fromHeader = getOnDiskSizeWithHeader(headerBuf, checksumSupport);</span> |
| <span class="source-line-no">1798</span><span id="line-1798"> if (onDiskSizeWithHeader != fromHeader) {</span> |
| <span class="source-line-no">1799</span><span id="line-1799"> if (LOG.isTraceEnabled()) {</span> |
| <span class="source-line-no">1800</span><span id="line-1800"> LOG.trace("Passed in onDiskSizeWithHeader={} != {}, offset={}, fileContext={}",</span> |
| <span class="source-line-no">1801</span><span id="line-1801"> onDiskSizeWithHeader, fromHeader, offset, this.fileContext);</span> |
| <span class="source-line-no">1802</span><span id="line-1802"> }</span> |
| <span class="source-line-no">1803</span><span id="line-1803"> if (checksumSupport && verifyChecksum) {</span> |
| <span class="source-line-no">1804</span><span id="line-1804"> // This file supports HBase checksums and verification of those checksums was</span> |
| <span class="source-line-no">1805</span><span id="line-1805"> // requested. The block size provided by the caller (presumably from the block index)</span> |
| <span class="source-line-no">1806</span><span id="line-1806"> // does not match the block size written to the block header. treat this as</span> |
| <span class="source-line-no">1807</span><span id="line-1807"> // HBase-checksum failure.</span> |
| <span class="source-line-no">1808</span><span id="line-1808"> span.addEvent("Falling back to HDFS checksumming.", attributesBuilder.build());</span> |
| <span class="source-line-no">1809</span><span id="line-1809"> invalidateNextBlockHeader();</span> |
| <span class="source-line-no">1810</span><span id="line-1810"> return null;</span> |
| <span class="source-line-no">1811</span><span id="line-1811"> }</span> |
| <span class="source-line-no">1812</span><span id="line-1812"> throw new IOException("Passed in onDiskSizeWithHeader=" + onDiskSizeWithHeader + " != "</span> |
| <span class="source-line-no">1813</span><span id="line-1813"> + fromHeader + ", offset=" + offset + ", fileContext=" + this.fileContext);</span> |
| <span class="source-line-no">1814</span><span id="line-1814"> }</span> |
| <span class="source-line-no">1815</span><span id="line-1815"></span> |
| <span class="source-line-no">1816</span><span id="line-1816"> // remove checksum from buffer now that it's verified</span> |
| <span class="source-line-no">1817</span><span id="line-1817"> int sizeWithoutChecksum = curBlock.getInt(Header.ON_DISK_DATA_SIZE_WITH_HEADER_INDEX);</span> |
| <span class="source-line-no">1818</span><span id="line-1818"> curBlock.limit(sizeWithoutChecksum);</span> |
| <span class="source-line-no">1819</span><span id="line-1819"> long duration = EnvironmentEdgeManager.currentTime() - startTime;</span> |
| <span class="source-line-no">1820</span><span id="line-1820"> boolean tooSlow = this.readWarnTime >= 0 && duration > this.readWarnTime;</span> |
| <span class="source-line-no">1821</span><span id="line-1821"> if (updateMetrics) {</span> |
| <span class="source-line-no">1822</span><span id="line-1822"> HFile.updateReadLatency(duration, pread, tooSlow);</span> |
| <span class="source-line-no">1823</span><span id="line-1823"> }</span> |
| <span class="source-line-no">1824</span><span id="line-1824"> // The onDiskBlock will become the headerAndDataBuffer for this block.</span> |
| <span class="source-line-no">1825</span><span id="line-1825"> // If nextBlockOnDiskSizeWithHeader is not zero, the onDiskBlock already</span> |
| <span class="source-line-no">1826</span><span id="line-1826"> // contains the header of next block, so no need to set next block's header in it.</span> |
| <span class="source-line-no">1827</span><span id="line-1827"> HFileBlock hFileBlock = createFromBuff(curBlock, checksumSupport, offset,</span> |
| <span class="source-line-no">1828</span><span id="line-1828"> nextBlockOnDiskSize, fileContext, intoHeap ? HEAP : allocator);</span> |
| <span class="source-line-no">1829</span><span id="line-1829"> // Run check on uncompressed sizings.</span> |
| <span class="source-line-no">1830</span><span id="line-1830"> if (!fileContext.isCompressedOrEncrypted()) {</span> |
| <span class="source-line-no">1831</span><span id="line-1831"> hFileBlock.sanityCheckUncompressed();</span> |
| <span class="source-line-no">1832</span><span id="line-1832"> }</span> |
| <span class="source-line-no">1833</span><span id="line-1833"> LOG.trace("Read {} in {} ms", hFileBlock, duration);</span> |
| <span class="source-line-no">1834</span><span id="line-1834"> if (!LOG.isTraceEnabled() && tooSlow) {</span> |
| <span class="source-line-no">1835</span><span id="line-1835"> LOG.warn("Read Block Slow: read {} cost {} ms, threshold = {} ms", hFileBlock, duration,</span> |
| <span class="source-line-no">1836</span><span id="line-1836"> this.readWarnTime);</span> |
| <span class="source-line-no">1837</span><span id="line-1837"> }</span> |
| <span class="source-line-no">1838</span><span id="line-1838"> span.addEvent("Read block", attributesBuilder.build());</span> |
| <span class="source-line-no">1839</span><span id="line-1839"> // Cache next block header if we read it for the next time through here.</span> |
| <span class="source-line-no">1840</span><span id="line-1840"> if (nextBlockOnDiskSize != -1) {</span> |
| <span class="source-line-no">1841</span><span id="line-1841"> cacheNextBlockHeader(offset + hFileBlock.getOnDiskSizeWithHeader(), onDiskBlock,</span> |
| <span class="source-line-no">1842</span><span id="line-1842"> onDiskSizeWithHeader, hdrSize);</span> |
| <span class="source-line-no">1843</span><span id="line-1843"> }</span> |
| <span class="source-line-no">1844</span><span id="line-1844"> initHFileBlockSuccess = true;</span> |
| <span class="source-line-no">1845</span><span id="line-1845"> return hFileBlock;</span> |
| <span class="source-line-no">1846</span><span id="line-1846"> } finally {</span> |
| <span class="source-line-no">1847</span><span id="line-1847"> if (!initHFileBlockSuccess) {</span> |
| <span class="source-line-no">1848</span><span id="line-1848"> onDiskBlock.release();</span> |
| <span class="source-line-no">1849</span><span id="line-1849"> }</span> |
| <span class="source-line-no">1850</span><span id="line-1850"> }</span> |
| <span class="source-line-no">1851</span><span id="line-1851"> }</span> |
| <span class="source-line-no">1852</span><span id="line-1852"></span> |
| <span class="source-line-no">1853</span><span id="line-1853"> @Override</span> |
| <span class="source-line-no">1854</span><span id="line-1854"> public void setIncludesMemStoreTS(boolean includesMemstoreTS) {</span> |
| <span class="source-line-no">1855</span><span id="line-1855"> this.fileContext =</span> |
| <span class="source-line-no">1856</span><span id="line-1856"> new HFileContextBuilder(this.fileContext).withIncludesMvcc(includesMemstoreTS).build();</span> |
| <span class="source-line-no">1857</span><span id="line-1857"> }</span> |
| <span class="source-line-no">1858</span><span id="line-1858"></span> |
| <span class="source-line-no">1859</span><span id="line-1859"> @Override</span> |
| <span class="source-line-no">1860</span><span id="line-1860"> public void setDataBlockEncoder(HFileDataBlockEncoder encoder, Configuration conf) {</span> |
| <span class="source-line-no">1861</span><span id="line-1861"> encodedBlockDecodingCtx = encoder.newDataBlockDecodingContext(conf, fileContext);</span> |
| <span class="source-line-no">1862</span><span id="line-1862"> }</span> |
| <span class="source-line-no">1863</span><span id="line-1863"></span> |
| <span class="source-line-no">1864</span><span id="line-1864"> @Override</span> |
| <span class="source-line-no">1865</span><span id="line-1865"> public HFileBlockDecodingContext getBlockDecodingContext() {</span> |
| <span class="source-line-no">1866</span><span id="line-1866"> return this.encodedBlockDecodingCtx;</span> |
| <span class="source-line-no">1867</span><span id="line-1867"> }</span> |
| <span class="source-line-no">1868</span><span id="line-1868"></span> |
| <span class="source-line-no">1869</span><span id="line-1869"> @Override</span> |
| <span class="source-line-no">1870</span><span id="line-1870"> public HFileBlockDecodingContext getDefaultBlockDecodingContext() {</span> |
| <span class="source-line-no">1871</span><span id="line-1871"> return this.defaultDecodingCtx;</span> |
| <span class="source-line-no">1872</span><span id="line-1872"> }</span> |
| <span class="source-line-no">1873</span><span id="line-1873"></span> |
| <span class="source-line-no">1874</span><span id="line-1874"> /**</span> |
| <span class="source-line-no">1875</span><span id="line-1875"> * Generates the checksum for the header as well as the data and then validates it. If the block</span> |
| <span class="source-line-no">1876</span><span id="line-1876"> * doesn't uses checksum, returns false.</span> |
| <span class="source-line-no">1877</span><span id="line-1877"> * @return True if checksum matches, else false.</span> |
| <span class="source-line-no">1878</span><span id="line-1878"> */</span> |
| <span class="source-line-no">1879</span><span id="line-1879"> private boolean validateChecksum(long offset, ByteBuff data, int hdrSize) {</span> |
| <span class="source-line-no">1880</span><span id="line-1880"> // If this is an older version of the block that does not have checksums, then return false</span> |
| <span class="source-line-no">1881</span><span id="line-1881"> // indicating that checksum verification did not succeed. Actually, this method should never</span> |
| <span class="source-line-no">1882</span><span id="line-1882"> // be called when the minorVersion is 0, thus this is a defensive check for a cannot-happen</span> |
| <span class="source-line-no">1883</span><span id="line-1883"> // case. Since this is a cannot-happen case, it is better to return false to indicate a</span> |
| <span class="source-line-no">1884</span><span id="line-1884"> // checksum validation failure.</span> |
| <span class="source-line-no">1885</span><span id="line-1885"> if (!fileContext.isUseHBaseChecksum()) {</span> |
| <span class="source-line-no">1886</span><span id="line-1886"> return false;</span> |
| <span class="source-line-no">1887</span><span id="line-1887"> }</span> |
| <span class="source-line-no">1888</span><span id="line-1888"> return ChecksumUtil.validateChecksum(data, pathName, offset, hdrSize);</span> |
| <span class="source-line-no">1889</span><span id="line-1889"> }</span> |
| <span class="source-line-no">1890</span><span id="line-1890"></span> |
| <span class="source-line-no">1891</span><span id="line-1891"> @Override</span> |
| <span class="source-line-no">1892</span><span id="line-1892"> public void closeStreams() throws IOException {</span> |
| <span class="source-line-no">1893</span><span id="line-1893"> streamWrapper.close();</span> |
| <span class="source-line-no">1894</span><span id="line-1894"> }</span> |
| <span class="source-line-no">1895</span><span id="line-1895"></span> |
| <span class="source-line-no">1896</span><span id="line-1896"> @Override</span> |
| <span class="source-line-no">1897</span><span id="line-1897"> public void unbufferStream() {</span> |
| <span class="source-line-no">1898</span><span id="line-1898"> // To handle concurrent reads, ensure that no other client is accessing the streams while we</span> |
| <span class="source-line-no">1899</span><span id="line-1899"> // unbuffer it.</span> |
| <span class="source-line-no">1900</span><span id="line-1900"> if (streamLock.tryLock()) {</span> |
| <span class="source-line-no">1901</span><span id="line-1901"> try {</span> |
| <span class="source-line-no">1902</span><span id="line-1902"> this.streamWrapper.unbuffer();</span> |
| <span class="source-line-no">1903</span><span id="line-1903"> } finally {</span> |
| <span class="source-line-no">1904</span><span id="line-1904"> streamLock.unlock();</span> |
| <span class="source-line-no">1905</span><span id="line-1905"> }</span> |
| <span class="source-line-no">1906</span><span id="line-1906"> }</span> |
| <span class="source-line-no">1907</span><span id="line-1907"> }</span> |
| <span class="source-line-no">1908</span><span id="line-1908"></span> |
| <span class="source-line-no">1909</span><span id="line-1909"> @Override</span> |
| <span class="source-line-no">1910</span><span id="line-1910"> public String toString() {</span> |
| <span class="source-line-no">1911</span><span id="line-1911"> return "hfs=" + hfs + ", path=" + pathName + ", fileContext=" + fileContext;</span> |
| <span class="source-line-no">1912</span><span id="line-1912"> }</span> |
| <span class="source-line-no">1913</span><span id="line-1913"> }</span> |
| <span class="source-line-no">1914</span><span id="line-1914"></span> |
| <span class="source-line-no">1915</span><span id="line-1915"> /** An additional sanity-check in case no compression or encryption is being used. */</span> |
| <span class="source-line-no">1916</span><span id="line-1916"> void sanityCheckUncompressed() throws IOException {</span> |
| <span class="source-line-no">1917</span><span id="line-1917"> if (onDiskSizeWithoutHeader != uncompressedSizeWithoutHeader + totalChecksumBytes()) {</span> |
| <span class="source-line-no">1918</span><span id="line-1918"> throw new IOException("Using no compression but " + "onDiskSizeWithoutHeader="</span> |
| <span class="source-line-no">1919</span><span id="line-1919"> + onDiskSizeWithoutHeader + ", " + "uncompressedSizeWithoutHeader="</span> |
| <span class="source-line-no">1920</span><span id="line-1920"> + uncompressedSizeWithoutHeader + ", numChecksumbytes=" + totalChecksumBytes());</span> |
| <span class="source-line-no">1921</span><span id="line-1921"> }</span> |
| <span class="source-line-no">1922</span><span id="line-1922"> }</span> |
| <span class="source-line-no">1923</span><span id="line-1923"></span> |
| <span class="source-line-no">1924</span><span id="line-1924"> // Cacheable implementation</span> |
| <span class="source-line-no">1925</span><span id="line-1925"> @Override</span> |
| <span class="source-line-no">1926</span><span id="line-1926"> public int getSerializedLength() {</span> |
| <span class="source-line-no">1927</span><span id="line-1927"> if (bufWithoutChecksum != null) {</span> |
| <span class="source-line-no">1928</span><span id="line-1928"> // Include extra bytes for block metadata.</span> |
| <span class="source-line-no">1929</span><span id="line-1929"> return this.bufWithoutChecksum.limit() + BLOCK_METADATA_SPACE;</span> |
| <span class="source-line-no">1930</span><span id="line-1930"> }</span> |
| <span class="source-line-no">1931</span><span id="line-1931"> return 0;</span> |
| <span class="source-line-no">1932</span><span id="line-1932"> }</span> |
| <span class="source-line-no">1933</span><span id="line-1933"></span> |
| <span class="source-line-no">1934</span><span id="line-1934"> // Cacheable implementation</span> |
| <span class="source-line-no">1935</span><span id="line-1935"> @Override</span> |
| <span class="source-line-no">1936</span><span id="line-1936"> public void serialize(ByteBuffer destination, boolean includeNextBlockMetadata) {</span> |
| <span class="source-line-no">1937</span><span id="line-1937"> this.bufWithoutChecksum.get(destination, 0, getSerializedLength() - BLOCK_METADATA_SPACE);</span> |
| <span class="source-line-no">1938</span><span id="line-1938"> destination = addMetaData(destination, includeNextBlockMetadata);</span> |
| <span class="source-line-no">1939</span><span id="line-1939"></span> |
| <span class="source-line-no">1940</span><span id="line-1940"> // Make it ready for reading. flip sets position to zero and limit to current position which</span> |
| <span class="source-line-no">1941</span><span id="line-1941"> // is what we want if we do not want to serialize the block plus checksums if present plus</span> |
| <span class="source-line-no">1942</span><span id="line-1942"> // metadata.</span> |
| <span class="source-line-no">1943</span><span id="line-1943"> destination.flip();</span> |
| <span class="source-line-no">1944</span><span id="line-1944"> }</span> |
| <span class="source-line-no">1945</span><span id="line-1945"></span> |
| <span class="source-line-no">1946</span><span id="line-1946"> /**</span> |
| <span class="source-line-no">1947</span><span id="line-1947"> * For use by bucketcache. This exposes internals.</span> |
| <span class="source-line-no">1948</span><span id="line-1948"> */</span> |
| <span class="source-line-no">1949</span><span id="line-1949"> public ByteBuffer getMetaData(ByteBuffer bb) {</span> |
| <span class="source-line-no">1950</span><span id="line-1950"> bb = addMetaData(bb, true);</span> |
| <span class="source-line-no">1951</span><span id="line-1951"> bb.flip();</span> |
| <span class="source-line-no">1952</span><span id="line-1952"> return bb;</span> |
| <span class="source-line-no">1953</span><span id="line-1953"> }</span> |
| <span class="source-line-no">1954</span><span id="line-1954"></span> |
| <span class="source-line-no">1955</span><span id="line-1955"> /**</span> |
| <span class="source-line-no">1956</span><span id="line-1956"> * Adds metadata at current position (position is moved forward). Does not flip or reset.</span> |
| <span class="source-line-no">1957</span><span id="line-1957"> * @return The passed <code>destination</code> with metadata added.</span> |
| <span class="source-line-no">1958</span><span id="line-1958"> */</span> |
| <span class="source-line-no">1959</span><span id="line-1959"> private ByteBuffer addMetaData(final ByteBuffer destination, boolean includeNextBlockMetadata) {</span> |
| <span class="source-line-no">1960</span><span id="line-1960"> destination.put(this.fileContext.isUseHBaseChecksum() ? (byte) 1 : (byte) 0);</span> |
| <span class="source-line-no">1961</span><span id="line-1961"> destination.putLong(this.offset);</span> |
| <span class="source-line-no">1962</span><span id="line-1962"> if (includeNextBlockMetadata) {</span> |
| <span class="source-line-no">1963</span><span id="line-1963"> destination.putInt(this.nextBlockOnDiskSize);</span> |
| <span class="source-line-no">1964</span><span id="line-1964"> }</span> |
| <span class="source-line-no">1965</span><span id="line-1965"> return destination;</span> |
| <span class="source-line-no">1966</span><span id="line-1966"> }</span> |
| <span class="source-line-no">1967</span><span id="line-1967"></span> |
| <span class="source-line-no">1968</span><span id="line-1968"> // Cacheable implementation</span> |
| <span class="source-line-no">1969</span><span id="line-1969"> @Override</span> |
| <span class="source-line-no">1970</span><span id="line-1970"> public CacheableDeserializer<Cacheable> getDeserializer() {</span> |
| <span class="source-line-no">1971</span><span id="line-1971"> return HFileBlock.BLOCK_DESERIALIZER;</span> |
| <span class="source-line-no">1972</span><span id="line-1972"> }</span> |
| <span class="source-line-no">1973</span><span id="line-1973"></span> |
| <span class="source-line-no">1974</span><span id="line-1974"> @Override</span> |
| <span class="source-line-no">1975</span><span id="line-1975"> public int hashCode() {</span> |
| <span class="source-line-no">1976</span><span id="line-1976"> int result = 1;</span> |
| <span class="source-line-no">1977</span><span id="line-1977"> result = result * 31 + blockType.hashCode();</span> |
| <span class="source-line-no">1978</span><span id="line-1978"> result = result * 31 + nextBlockOnDiskSize;</span> |
| <span class="source-line-no">1979</span><span id="line-1979"> result = result * 31 + (int) (offset ^ (offset >>> 32));</span> |
| <span class="source-line-no">1980</span><span id="line-1980"> result = result * 31 + onDiskSizeWithoutHeader;</span> |
| <span class="source-line-no">1981</span><span id="line-1981"> result = result * 31 + (int) (prevBlockOffset ^ (prevBlockOffset >>> 32));</span> |
| <span class="source-line-no">1982</span><span id="line-1982"> result = result * 31 + uncompressedSizeWithoutHeader;</span> |
| <span class="source-line-no">1983</span><span id="line-1983"> result = result * 31 + bufWithoutChecksum.hashCode();</span> |
| <span class="source-line-no">1984</span><span id="line-1984"> return result;</span> |
| <span class="source-line-no">1985</span><span id="line-1985"> }</span> |
| <span class="source-line-no">1986</span><span id="line-1986"></span> |
| <span class="source-line-no">1987</span><span id="line-1987"> @Override</span> |
| <span class="source-line-no">1988</span><span id="line-1988"> public boolean equals(Object comparison) {</span> |
| <span class="source-line-no">1989</span><span id="line-1989"> if (this == comparison) {</span> |
| <span class="source-line-no">1990</span><span id="line-1990"> return true;</span> |
| <span class="source-line-no">1991</span><span id="line-1991"> }</span> |
| <span class="source-line-no">1992</span><span id="line-1992"> if (comparison == null) {</span> |
| <span class="source-line-no">1993</span><span id="line-1993"> return false;</span> |
| <span class="source-line-no">1994</span><span id="line-1994"> }</span> |
| <span class="source-line-no">1995</span><span id="line-1995"> if (!(comparison instanceof HFileBlock)) {</span> |
| <span class="source-line-no">1996</span><span id="line-1996"> return false;</span> |
| <span class="source-line-no">1997</span><span id="line-1997"> }</span> |
| <span class="source-line-no">1998</span><span id="line-1998"></span> |
| <span class="source-line-no">1999</span><span id="line-1999"> HFileBlock castedComparison = (HFileBlock) comparison;</span> |
| <span class="source-line-no">2000</span><span id="line-2000"></span> |
| <span class="source-line-no">2001</span><span id="line-2001"> if (castedComparison.blockType != this.blockType) {</span> |
| <span class="source-line-no">2002</span><span id="line-2002"> return false;</span> |
| <span class="source-line-no">2003</span><span id="line-2003"> }</span> |
| <span class="source-line-no">2004</span><span id="line-2004"> if (castedComparison.nextBlockOnDiskSize != this.nextBlockOnDiskSize) {</span> |
| <span class="source-line-no">2005</span><span id="line-2005"> return false;</span> |
| <span class="source-line-no">2006</span><span id="line-2006"> }</span> |
| <span class="source-line-no">2007</span><span id="line-2007"> // Offset is important. Needed when we have to remake cachekey when block is returned to cache.</span> |
| <span class="source-line-no">2008</span><span id="line-2008"> if (castedComparison.offset != this.offset) {</span> |
| <span class="source-line-no">2009</span><span id="line-2009"> return false;</span> |
| <span class="source-line-no">2010</span><span id="line-2010"> }</span> |
| <span class="source-line-no">2011</span><span id="line-2011"> if (castedComparison.onDiskSizeWithoutHeader != this.onDiskSizeWithoutHeader) {</span> |
| <span class="source-line-no">2012</span><span id="line-2012"> return false;</span> |
| <span class="source-line-no">2013</span><span id="line-2013"> }</span> |
| <span class="source-line-no">2014</span><span id="line-2014"> if (castedComparison.prevBlockOffset != this.prevBlockOffset) {</span> |
| <span class="source-line-no">2015</span><span id="line-2015"> return false;</span> |
| <span class="source-line-no">2016</span><span id="line-2016"> }</span> |
| <span class="source-line-no">2017</span><span id="line-2017"> if (castedComparison.uncompressedSizeWithoutHeader != this.uncompressedSizeWithoutHeader) {</span> |
| <span class="source-line-no">2018</span><span id="line-2018"> return false;</span> |
| <span class="source-line-no">2019</span><span id="line-2019"> }</span> |
| <span class="source-line-no">2020</span><span id="line-2020"> if (</span> |
| <span class="source-line-no">2021</span><span id="line-2021"> ByteBuff.compareTo(this.bufWithoutChecksum, 0, this.bufWithoutChecksum.limit(),</span> |
| <span class="source-line-no">2022</span><span id="line-2022"> castedComparison.bufWithoutChecksum, 0, castedComparison.bufWithoutChecksum.limit()) != 0</span> |
| <span class="source-line-no">2023</span><span id="line-2023"> ) {</span> |
| <span class="source-line-no">2024</span><span id="line-2024"> return false;</span> |
| <span class="source-line-no">2025</span><span id="line-2025"> }</span> |
| <span class="source-line-no">2026</span><span id="line-2026"> return true;</span> |
| <span class="source-line-no">2027</span><span id="line-2027"> }</span> |
| <span class="source-line-no">2028</span><span id="line-2028"></span> |
| <span class="source-line-no">2029</span><span id="line-2029"> DataBlockEncoding getDataBlockEncoding() {</span> |
| <span class="source-line-no">2030</span><span id="line-2030"> if (blockType == BlockType.ENCODED_DATA) {</span> |
| <span class="source-line-no">2031</span><span id="line-2031"> return DataBlockEncoding.getEncodingById(getDataBlockEncodingId());</span> |
| <span class="source-line-no">2032</span><span id="line-2032"> }</span> |
| <span class="source-line-no">2033</span><span id="line-2033"> return DataBlockEncoding.NONE;</span> |
| <span class="source-line-no">2034</span><span id="line-2034"> }</span> |
| <span class="source-line-no">2035</span><span id="line-2035"></span> |
| <span class="source-line-no">2036</span><span id="line-2036"> byte getChecksumType() {</span> |
| <span class="source-line-no">2037</span><span id="line-2037"> return this.fileContext.getChecksumType().getCode();</span> |
| <span class="source-line-no">2038</span><span id="line-2038"> }</span> |
| <span class="source-line-no">2039</span><span id="line-2039"></span> |
| <span class="source-line-no">2040</span><span id="line-2040"> int getBytesPerChecksum() {</span> |
| <span class="source-line-no">2041</span><span id="line-2041"> return this.fileContext.getBytesPerChecksum();</span> |
| <span class="source-line-no">2042</span><span id="line-2042"> }</span> |
| <span class="source-line-no">2043</span><span id="line-2043"></span> |
| <span class="source-line-no">2044</span><span id="line-2044"> /** Returns the size of data on disk + header. Excludes checksum. */</span> |
| <span class="source-line-no">2045</span><span id="line-2045"> int getOnDiskDataSizeWithHeader() {</span> |
| <span class="source-line-no">2046</span><span id="line-2046"> return this.onDiskDataSizeWithHeader;</span> |
| <span class="source-line-no">2047</span><span id="line-2047"> }</span> |
| <span class="source-line-no">2048</span><span id="line-2048"></span> |
| <span class="source-line-no">2049</span><span id="line-2049"> /**</span> |
| <span class="source-line-no">2050</span><span id="line-2050"> * Return the number of bytes required to store all the checksums for this block. Each checksum</span> |
| <span class="source-line-no">2051</span><span id="line-2051"> * value is a 4 byte integer. <br/></span> |
| <span class="source-line-no">2052</span><span id="line-2052"> * NOTE: ByteBuff returned by {@link HFileBlock#getBufferWithoutHeader()} and</span> |
| <span class="source-line-no">2053</span><span id="line-2053"> * {@link HFileBlock#getBufferReadOnly} or DataInputStream returned by</span> |
| <span class="source-line-no">2054</span><span id="line-2054"> * {@link HFileBlock#getByteStream()} does not include checksum.</span> |
| <span class="source-line-no">2055</span><span id="line-2055"> */</span> |
| <span class="source-line-no">2056</span><span id="line-2056"> int totalChecksumBytes() {</span> |
| <span class="source-line-no">2057</span><span id="line-2057"> return totalChecksumBytes;</span> |
| <span class="source-line-no">2058</span><span id="line-2058"> }</span> |
| <span class="source-line-no">2059</span><span id="line-2059"></span> |
| <span class="source-line-no">2060</span><span id="line-2060"> private int computeTotalChecksumBytes() {</span> |
| <span class="source-line-no">2061</span><span id="line-2061"> // If the hfile block has minorVersion 0, then there are no checksum</span> |
| <span class="source-line-no">2062</span><span id="line-2062"> // data to validate. Similarly, a zero value in this.bytesPerChecksum</span> |
| <span class="source-line-no">2063</span><span id="line-2063"> // indicates that cached blocks do not have checksum data because</span> |
| <span class="source-line-no">2064</span><span id="line-2064"> // checksums were already validated when the block was read from disk.</span> |
| <span class="source-line-no">2065</span><span id="line-2065"> if (!fileContext.isUseHBaseChecksum() || this.fileContext.getBytesPerChecksum() == 0) {</span> |
| <span class="source-line-no">2066</span><span id="line-2066"> return 0;</span> |
| <span class="source-line-no">2067</span><span id="line-2067"> }</span> |
| <span class="source-line-no">2068</span><span id="line-2068"> return (int) ChecksumUtil.numBytes(onDiskDataSizeWithHeader,</span> |
| <span class="source-line-no">2069</span><span id="line-2069"> this.fileContext.getBytesPerChecksum());</span> |
| <span class="source-line-no">2070</span><span id="line-2070"> }</span> |
| <span class="source-line-no">2071</span><span id="line-2071"></span> |
| <span class="source-line-no">2072</span><span id="line-2072"> /**</span> |
| <span class="source-line-no">2073</span><span id="line-2073"> * Returns the size of this block header.</span> |
| <span class="source-line-no">2074</span><span id="line-2074"> */</span> |
| <span class="source-line-no">2075</span><span id="line-2075"> public int headerSize() {</span> |
| <span class="source-line-no">2076</span><span id="line-2076"> return headerSize(this.fileContext.isUseHBaseChecksum());</span> |
| <span class="source-line-no">2077</span><span id="line-2077"> }</span> |
| <span class="source-line-no">2078</span><span id="line-2078"></span> |
| <span class="source-line-no">2079</span><span id="line-2079"> /**</span> |
| <span class="source-line-no">2080</span><span id="line-2080"> * Maps a minor version to the size of the header.</span> |
| <span class="source-line-no">2081</span><span id="line-2081"> */</span> |
| <span class="source-line-no">2082</span><span id="line-2082"> public static int headerSize(boolean usesHBaseChecksum) {</span> |
| <span class="source-line-no">2083</span><span id="line-2083"> return usesHBaseChecksum</span> |
| <span class="source-line-no">2084</span><span id="line-2084"> ? HConstants.HFILEBLOCK_HEADER_SIZE</span> |
| <span class="source-line-no">2085</span><span id="line-2085"> : HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM;</span> |
| <span class="source-line-no">2086</span><span id="line-2086"> }</span> |
| <span class="source-line-no">2087</span><span id="line-2087"></span> |
| <span class="source-line-no">2088</span><span id="line-2088"> /**</span> |
| <span class="source-line-no">2089</span><span id="line-2089"> * Return the appropriate DUMMY_HEADER for the minor version</span> |
| <span class="source-line-no">2090</span><span id="line-2090"> */</span> |
| <span class="source-line-no">2091</span><span id="line-2091"> // TODO: Why is this in here?</span> |
| <span class="source-line-no">2092</span><span id="line-2092"> byte[] getDummyHeaderForVersion() {</span> |
| <span class="source-line-no">2093</span><span id="line-2093"> return getDummyHeaderForVersion(this.fileContext.isUseHBaseChecksum());</span> |
| <span class="source-line-no">2094</span><span id="line-2094"> }</span> |
| <span class="source-line-no">2095</span><span id="line-2095"></span> |
| <span class="source-line-no">2096</span><span id="line-2096"> /**</span> |
| <span class="source-line-no">2097</span><span id="line-2097"> * Return the appropriate DUMMY_HEADER for the minor version</span> |
| <span class="source-line-no">2098</span><span id="line-2098"> */</span> |
| <span class="source-line-no">2099</span><span id="line-2099"> static private byte[] getDummyHeaderForVersion(boolean usesHBaseChecksum) {</span> |
| <span class="source-line-no">2100</span><span id="line-2100"> return usesHBaseChecksum ? HConstants.HFILEBLOCK_DUMMY_HEADER : DUMMY_HEADER_NO_CHECKSUM;</span> |
| <span class="source-line-no">2101</span><span id="line-2101"> }</span> |
| <span class="source-line-no">2102</span><span id="line-2102"></span> |
| <span class="source-line-no">2103</span><span id="line-2103"> /**</span> |
| <span class="source-line-no">2104</span><span id="line-2104"> * @return This HFileBlocks fileContext which will a derivative of the fileContext for the file</span> |
| <span class="source-line-no">2105</span><span id="line-2105"> * from which this block's data was originally read.</span> |
| <span class="source-line-no">2106</span><span id="line-2106"> */</span> |
| <span class="source-line-no">2107</span><span id="line-2107"> public HFileContext getHFileContext() {</span> |
| <span class="source-line-no">2108</span><span id="line-2108"> return this.fileContext;</span> |
| <span class="source-line-no">2109</span><span id="line-2109"> }</span> |
| <span class="source-line-no">2110</span><span id="line-2110"></span> |
| <span class="source-line-no">2111</span><span id="line-2111"> /**</span> |
| <span class="source-line-no">2112</span><span id="line-2112"> * Convert the contents of the block header into a human readable string. This is mostly helpful</span> |
| <span class="source-line-no">2113</span><span id="line-2113"> * for debugging. This assumes that the block has minor version > 0.</span> |
| <span class="source-line-no">2114</span><span id="line-2114"> */</span> |
| <span class="source-line-no">2115</span><span id="line-2115"> static String toStringHeader(ByteBuff buf) throws IOException {</span> |
| <span class="source-line-no">2116</span><span id="line-2116"> byte[] magicBuf = new byte[Math.min(buf.limit() - buf.position(), BlockType.MAGIC_LENGTH)];</span> |
| <span class="source-line-no">2117</span><span id="line-2117"> buf.get(magicBuf);</span> |
| <span class="source-line-no">2118</span><span id="line-2118"> BlockType bt = BlockType.parse(magicBuf, 0, BlockType.MAGIC_LENGTH);</span> |
| <span class="source-line-no">2119</span><span id="line-2119"> int compressedBlockSizeNoHeader = buf.getInt();</span> |
| <span class="source-line-no">2120</span><span id="line-2120"> int uncompressedBlockSizeNoHeader = buf.getInt();</span> |
| <span class="source-line-no">2121</span><span id="line-2121"> long prevBlockOffset = buf.getLong();</span> |
| <span class="source-line-no">2122</span><span id="line-2122"> byte cksumtype = buf.get();</span> |
| <span class="source-line-no">2123</span><span id="line-2123"> long bytesPerChecksum = buf.getInt();</span> |
| <span class="source-line-no">2124</span><span id="line-2124"> long onDiskDataSizeWithHeader = buf.getInt();</span> |
| <span class="source-line-no">2125</span><span id="line-2125"> return " Header dump: magic: " + Bytes.toString(magicBuf) + " blockType " + bt</span> |
| <span class="source-line-no">2126</span><span id="line-2126"> + " compressedBlockSizeNoHeader " + compressedBlockSizeNoHeader</span> |
| <span class="source-line-no">2127</span><span id="line-2127"> + " uncompressedBlockSizeNoHeader " + uncompressedBlockSizeNoHeader + " prevBlockOffset "</span> |
| <span class="source-line-no">2128</span><span id="line-2128"> + prevBlockOffset + " checksumType " + ChecksumType.codeToType(cksumtype)</span> |
| <span class="source-line-no">2129</span><span id="line-2129"> + " bytesPerChecksum " + bytesPerChecksum + " onDiskDataSizeWithHeader "</span> |
| <span class="source-line-no">2130</span><span id="line-2130"> + onDiskDataSizeWithHeader;</span> |
| <span class="source-line-no">2131</span><span id="line-2131"> }</span> |
| <span class="source-line-no">2132</span><span id="line-2132"></span> |
| <span class="source-line-no">2133</span><span id="line-2133"> /**</span> |
| <span class="source-line-no">2134</span><span id="line-2134"> * Creates a new HFileBlockBuilder from the existing block and a new ByteBuff. The builder will be</span> |
| <span class="source-line-no">2135</span><span id="line-2135"> * loaded with all of the original fields from blk, except now using the newBuff and setting</span> |
| <span class="source-line-no">2136</span><span id="line-2136"> * isSharedMem based on the source of the passed in newBuff. An existing HFileBlock may have been</span> |
| <span class="source-line-no">2137</span><span id="line-2137"> * an {@link ExclusiveMemHFileBlock}, but the new buffer might call for a</span> |
| <span class="source-line-no">2138</span><span id="line-2138"> * {@link SharedMemHFileBlock}. Or vice versa.</span> |
| <span class="source-line-no">2139</span><span id="line-2139"> * @param blk the block to clone from</span> |
| <span class="source-line-no">2140</span><span id="line-2140"> * @param newBuff the new buffer to use</span> |
| <span class="source-line-no">2141</span><span id="line-2141"> */</span> |
| <span class="source-line-no">2142</span><span id="line-2142"> private static HFileBlockBuilder createBuilder(HFileBlock blk, ByteBuff newBuff) {</span> |
| <span class="source-line-no">2143</span><span id="line-2143"> return new HFileBlockBuilder().withBlockType(blk.blockType)</span> |
| <span class="source-line-no">2144</span><span id="line-2144"> .withOnDiskSizeWithoutHeader(blk.onDiskSizeWithoutHeader)</span> |
| <span class="source-line-no">2145</span><span id="line-2145"> .withUncompressedSizeWithoutHeader(blk.uncompressedSizeWithoutHeader)</span> |
| <span class="source-line-no">2146</span><span id="line-2146"> .withPrevBlockOffset(blk.prevBlockOffset).withByteBuff(newBuff).withOffset(blk.offset)</span> |
| <span class="source-line-no">2147</span><span id="line-2147"> .withOnDiskDataSizeWithHeader(blk.onDiskDataSizeWithHeader)</span> |
| <span class="source-line-no">2148</span><span id="line-2148"> .withNextBlockOnDiskSize(blk.nextBlockOnDiskSize).withHFileContext(blk.fileContext)</span> |
| <span class="source-line-no">2149</span><span id="line-2149"> .withByteBuffAllocator(blk.allocator).withShared(!newBuff.hasArray());</span> |
| <span class="source-line-no">2150</span><span id="line-2150"> }</span> |
| <span class="source-line-no">2151</span><span id="line-2151"></span> |
| <span class="source-line-no">2152</span><span id="line-2152"> private static HFileBlock shallowClone(HFileBlock blk, ByteBuff newBuf) {</span> |
| <span class="source-line-no">2153</span><span id="line-2153"> return createBuilder(blk, newBuf).build();</span> |
| <span class="source-line-no">2154</span><span id="line-2154"> }</span> |
| <span class="source-line-no">2155</span><span id="line-2155"></span> |
| <span class="source-line-no">2156</span><span id="line-2156"> static HFileBlock deepCloneOnHeap(HFileBlock blk) {</span> |
| <span class="source-line-no">2157</span><span id="line-2157"> ByteBuff deepCloned = ByteBuff</span> |
| <span class="source-line-no">2158</span><span id="line-2158"> .wrap(ByteBuffer.wrap(blk.bufWithoutChecksum.toBytes(0, blk.bufWithoutChecksum.limit())));</span> |
| <span class="source-line-no">2159</span><span id="line-2159"> return createBuilder(blk, deepCloned).build();</span> |
| <span class="source-line-no">2160</span><span id="line-2160"> }</span> |
| <span class="source-line-no">2161</span><span id="line-2161">}</span> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </pre> |
| </div> |
| </main> |
| </body> |
| </html> |
