HBASE-23705 Add CellComparator to HFileContext (#1062)
Codecs don't have access to what CellComparator to use. Backfill.
M hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java
Adds a new compareRows with default implementation that takes a ByteBuffer.
Needed by the index in a block encoder implementation.
M hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java
Adds implementation for meta of new compareRows method. Adds utility
method for figuring comparator based off tablename.
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/AbstractDataBlockEncoder.java
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexCodecV1.java
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java
Comparator is in context. Remove redundant handling.
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
Comparator is in context. Remove redundant handling. Clean javadoc.
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java
Clean javadoc.
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexEncoderV1.java
Cache context so can use it to get comparator to use later.
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java
Cache cellcomparator to use. Javdoc on diff between HFileContext and
HFileInfo.
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java
Add CellComparator
M hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java
M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
Remove comparator caching. Get from context instead.
M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
Skip a reflection if we can.
M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
Javadoc. Removed unused filed.
Signed-off-by: Anoop Sam John <anoopsamjohn@apacher.org>
Signed-off-by: Ramkrishna <ramkrishna.s.vasudevan@intel.com>
Signed-off-by: Jan Hentschel <janh@apache.org>
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java
index 83a868d..474f772 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java
@@ -17,8 +17,9 @@
*/
package org.apache.hadoop.hbase;
+import java.nio.ByteBuffer;
import java.util.Comparator;
-
+import org.apache.hadoop.hbase.util.ByteBufferUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
@@ -31,11 +32,12 @@
public interface CellComparator extends Comparator<Cell> {
/**
* A comparator for ordering cells in user-space tables. Useful when writing cells in sorted
- * order as necessary for bulk import (i.e. via MapReduce)
+ * order as necessary for bulk import (i.e. via MapReduce).
* <p>
* CAUTION: This comparator may provide inaccurate ordering for cells from system tables,
* and should not be relied upon in that case.
*/
+ // For internal use, see CellComparatorImpl utility methods.
static CellComparator getInstance() {
return CellComparatorImpl.COMPARATOR;
}
@@ -81,6 +83,24 @@
int compareRows(Cell cell, byte[] bytes, int offset, int length);
/**
+ * @param row ByteBuffer that wraps a row; will read from current position and will reading all
+ * remaining; will not disturb the ByteBuffer internal state.
+ * @return greater than 0 if leftCell is bigger, less than 0 if rightCell is bigger, 0 if both
+ * cells are equal
+ */
+ default int compareRows(ByteBuffer row, Cell cell) {
+ if (cell instanceof ByteBufferExtendedCell) {
+ return ByteBufferUtils.compareTo(row, row.position(), row.remaining(),
+ ((ByteBufferExtendedCell) cell).getRowByteBuffer(),
+ ((ByteBufferExtendedCell) cell).getRowPosition(),
+ cell.getRowLength());
+ }
+ return ByteBufferUtils.compareTo(row, row.position(), row.remaining(),
+ cell.getRowArray(), cell.getRowOffset(),
+ cell.getRowLength());
+ }
+
+ /**
* Lexographically compares the two cells excluding the row part. It compares family, qualifier,
* timestamp and the type
* @param leftCell the left hand side cell
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java
index c647318..e6c8e3d 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.hbase;
+import java.nio.ByteBuffer;
import java.util.Comparator;
-
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
import org.apache.hadoop.hbase.util.Bytes;
@@ -26,7 +26,6 @@
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hbase.thirdparty.com.google.common.primitives.Longs;
/**
@@ -378,6 +377,26 @@
}
@Override
+ public int compareRows(ByteBuffer row, Cell cell) {
+ byte [] array;
+ int offset;
+ int len = row.remaining();
+ if (row.hasArray()) {
+ array = row.array();
+ offset = row.position() + row.arrayOffset();
+ } else {
+ // We copy the row array if offheap just so we can do a compare. We do this elsewhere too
+ // in BBUtils when Cell is backed by an offheap ByteBuffer. Needs fixing so no copy. TODO.
+ array = new byte[len];
+ offset = 0;
+ ByteBufferUtils.copyFromBufferToArray(array, row, row.position(),
+ 0, len);
+ }
+ // Reverse result since we swap the order of the params we pass below.
+ return -compareRows(cell, array, offset, len);
+ }
+
+ @Override
public Comparator getSimpleComparator() {
return this;
}
@@ -387,4 +406,24 @@
public Comparator getSimpleComparator() {
return new BBKVComparator(this);
}
+
+ /**
+ * Utility method that makes a guess at comparator to use based off passed tableName.
+ * Use in extreme when no comparator specified.
+ * @return CellComparator to use going off the {@code tableName} passed.
+ */
+ public static CellComparator getCellComparator(TableName tableName) {
+ return getCellComparator(tableName.toBytes());
+ }
+
+ /**
+ * Utility method that makes a guess at comparator to use based off passed tableName.
+ * Use in extreme when no comparator specified.
+ * @return CellComparator to use going off the {@code tableName} passed.
+ */
+ public static CellComparator getCellComparator(byte [] tableName) {
+ // FYI, TableName.toBytes does not create an array; just returns existing array pointer.
+ return Bytes.equals(tableName, TableName.META_TABLE_NAME.toBytes())?
+ CellComparatorImpl.META_COMPARATOR: CellComparatorImpl.COMPARATOR;
+ }
}
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java
index 51c8757..f096ef9 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java
@@ -292,6 +292,9 @@
return qualifierAsString;
}
+ /**
+ * @return A pointer to TableName as String bytes.
+ */
public byte[] toBytes() {
return name;
}
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/AbstractDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/AbstractDataBlockEncoder.java
index ab95717..e96b800 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/AbstractDataBlockEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/AbstractDataBlockEncoder.java
@@ -18,10 +18,8 @@
import java.io.IOException;
import java.nio.ByteBuffer;
-
import org.apache.hadoop.hbase.ByteBufferKeyOnlyKeyValue;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.hfile.BlockType;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
@@ -59,14 +57,13 @@
}
}
- protected abstract static class AbstractEncodedSeeker implements
- EncodedSeeker {
+ /**
+ * Decorates EncodedSeeker with a {@link HFileBlockDecodingContext}
+ */
+ protected abstract static class AbstractEncodedSeeker implements EncodedSeeker {
protected HFileBlockDecodingContext decodingCtx;
- protected final CellComparator comparator;
- public AbstractEncodedSeeker(CellComparator comparator,
- HFileBlockDecodingContext decodingCtx) {
- this.comparator = comparator;
+ public AbstractEncodedSeeker(HFileBlockDecodingContext decodingCtx) {
this.decodingCtx = decodingCtx;
}
@@ -77,7 +74,5 @@
protected boolean includesTags() {
return this.decodingCtx.getHFileContext().isIncludesTags();
}
-
}
-
}
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
index bb22ad8..755647d 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
@@ -732,9 +732,8 @@
protected final ObjectIntPair<ByteBuffer> tmpPair = new ObjectIntPair<>();
protected STATE current, previous;
- public BufferedEncodedSeeker(CellComparator comparator,
- HFileBlockDecodingContext decodingCtx) {
- super(comparator, decodingCtx);
+ public BufferedEncodedSeeker(HFileBlockDecodingContext decodingCtx) {
+ super(decodingCtx);
if (decodingCtx.getHFileContext().isCompressTags()) {
try {
tagCompressionContext = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE);
@@ -1008,11 +1007,7 @@
}
/**
- * @param cell
- * @param out
- * @param encodingCtx
* @return unencoded size added
- * @throws IOException
*/
protected final int afterEncodingKeyValue(Cell cell, DataOutputStream out,
HFileBlockDefaultEncodingContext encodingCtx) throws IOException {
@@ -1102,7 +1097,7 @@
public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out)
throws IOException {
if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
- throw new IOException (this.getClass().getName() + " only accepts "
+ throw new IOException(this.getClass().getName() + " only accepts "
+ HFileBlockDefaultEncodingContext.class.getName() + " as the " +
"encoding context.");
}
@@ -1154,8 +1149,8 @@
.getEncodingState();
// Write the unencodedDataSizeWritten (with header size)
Bytes.putInt(uncompressedBytesWithHeader,
- HConstants.HFILEBLOCK_HEADER_SIZE + DataBlockEncoding.ID_SIZE, state.unencodedDataSizeWritten
- );
+ HConstants.HFILEBLOCK_HEADER_SIZE + DataBlockEncoding.ID_SIZE,
+ state.unencodedDataSizeWritten);
postEncoding(encodingCtx);
}
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
index d7ab009..02bb843 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
@@ -20,9 +20,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
import org.apache.hadoop.hbase.util.Bytes;
@@ -48,7 +46,8 @@
+ "encoding context.");
}
- HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx;
+ HFileBlockDefaultEncodingContext encodingCtx =
+ (HFileBlockDefaultEncodingContext) blkEncodingCtx;
encodingCtx.prepareEncoding(out);
NoneEncoder encoder = new NoneEncoder(out, encodingCtx);
@@ -81,9 +80,8 @@
}
@Override
- public EncodedSeeker createSeeker(CellComparator comparator,
- final HFileBlockDecodingContext decodingCtx) {
- return new BufferedEncodedSeeker<SeekerState>(comparator, decodingCtx) {
+ public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) {
+ return new BufferedEncodedSeeker<SeekerState>(decodingCtx) {
@Override
protected void decodeNext() {
current.keyLength = currentBuffer.getInt();
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
index e6f339d..d3c41fb 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
@@ -20,7 +20,6 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
@@ -45,20 +44,13 @@
* Starts encoding for a block of KeyValues. Call
* {@link #endBlockEncoding(HFileBlockEncodingContext, DataOutputStream, byte[])} to finish
* encoding of a block.
- * @param encodingCtx
- * @param out
- * @throws IOException
*/
void startBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out)
throws IOException;
/**
* Encodes a KeyValue.
- * @param cell
- * @param encodingCtx
- * @param out
* @return unencoded kv size written
- * @throws IOException
*/
int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out)
throws IOException;
@@ -66,10 +58,6 @@
/**
* Ends encoding for a block of KeyValues. Gives a chance for the encoder to do the finishing
* stuff for the encoded block. It must be called at the end of block encoding.
- * @param encodingCtx
- * @param out
- * @param uncompressedBytesWithHeader
- * @throws IOException
*/
void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out,
byte[] uncompressedBytesWithHeader) throws IOException;
@@ -77,7 +65,6 @@
/**
* Decode.
* @param source Compressed stream of KeyValues.
- * @param decodingCtx
* @return Uncompressed block of KeyValues.
* @throws IOException If there is an error in source.
*/
@@ -96,11 +83,9 @@
/**
* Create a HFileBlock seeker which find KeyValues within a block.
- * @param comparator what kind of comparison should be used
- * @param decodingCtx
* @return A newly created seeker.
*/
- EncodedSeeker createSeeker(CellComparator comparator, HFileBlockDecodingContext decodingCtx);
+ EncodedSeeker createSeeker(HFileBlockDecodingContext decodingCtx);
/**
* Creates a encoder specific encoding context
@@ -188,8 +173,6 @@
/**
* Compare the given key against the current key
- * @param comparator
- * @param key
* @return -1 is the passed key is smaller than the current key, 0 if equal and 1 if greater
*/
public int compareKey(CellComparator comparator, Cell key);
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
index ab93d19..afb3e6d 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
@@ -20,9 +20,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.PrivateCellUtil;
@@ -382,9 +380,8 @@
}
@Override
- public EncodedSeeker createSeeker(CellComparator comparator,
- HFileBlockDecodingContext decodingCtx) {
- return new BufferedEncodedSeeker<DiffSeekerState>(comparator, decodingCtx) {
+ public EncodedSeeker createSeeker(HFileBlockDecodingContext decodingCtx) {
+ return new BufferedEncodedSeeker<DiffSeekerState>(decodingCtx) {
private byte[] familyNameWithSize;
private static final int TIMESTAMP_WITH_TYPE_LENGTH =
Bytes.SIZEOF_LONG + Bytes.SIZEOF_BYTE;
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
index aa9a436..a5774e5 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
@@ -21,9 +21,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.PrivateCellUtil;
@@ -397,9 +395,8 @@
}
@Override
- public EncodedSeeker createSeeker(CellComparator comparator,
- final HFileBlockDecodingContext decodingCtx) {
- return new BufferedEncodedSeeker<FastDiffSeekerState>(comparator, decodingCtx) {
+ public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) {
+ return new BufferedEncodedSeeker<FastDiffSeekerState>(decodingCtx) {
private void decode(boolean isFirst) {
byte flag = currentBuffer.get();
if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java
index 7f29302..2cbffa9 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java
@@ -17,20 +17,18 @@
package org.apache.hadoop.hbase.io.encoding;
import java.io.IOException;
-
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.yetus.audience.InterfaceAudience;
/**
* A decoding context that is created by a reader's encoder, and is shared
- * across the reader's all read operations.
+ * across all of the reader's read operations.
*
* @see HFileBlockEncodingContext for encoding
*/
@InterfaceAudience.Private
public interface HFileBlockDecodingContext {
-
/**
* Perform all actions that need to be done before the encoder's real decoding
* process. Decompression needs to be done if
@@ -46,7 +44,6 @@
* ByteBuffer pointed after the header but before the data
* @param onDiskBlock
* on disk data to be decoded
- * @throws IOException
*/
void prepareDecoding(
int onDiskSizeWithoutHeader,
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultDecodingContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultDecodingContext.java
index 97d0e6b..e321a25 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultDecodingContext.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultDecodingContext.java
@@ -19,7 +19,6 @@
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
-
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.io.ByteBuffInputStream;
import org.apache.hadoop.hbase.io.TagCompressionContext;
@@ -41,11 +40,10 @@
*
*/
@InterfaceAudience.Private
-public class HFileBlockDefaultDecodingContext implements
- HFileBlockDecodingContext {
+public class HFileBlockDefaultDecodingContext implements HFileBlockDecodingContext {
private final HFileContext fileContext;
private TagCompressionContext tagCompressionContext;
-
+
public HFileBlockDefaultDecodingContext(HFileContext fileContext) {
this.fileContext = fileContext;
}
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
index d029e1c..169f979 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
@@ -17,13 +17,11 @@
package org.apache.hadoop.hbase.io.encoding;
import static org.apache.hadoop.hbase.io.compress.Compression.Algorithm.NONE;
-
import java.io.ByteArrayInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.SecureRandom;
-
import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
import org.apache.hadoop.hbase.io.TagCompressionContext;
import org.apache.hadoop.hbase.io.compress.Compression;
@@ -36,7 +34,6 @@
import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.io.compress.Compressor;
import org.apache.yetus.audience.InterfaceAudience;
-
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
/**
@@ -47,8 +44,7 @@
*
*/
@InterfaceAudience.Private
-public class HFileBlockDefaultEncodingContext implements
- HFileBlockEncodingContext {
+public class HFileBlockDefaultEncodingContext implements HFileBlockEncodingContext {
private BlockType blockType;
private final DataBlockEncoding encodingAlgo;
@@ -113,7 +109,6 @@
/**
* prepare to start a new encoding.
- * @throws IOException
*/
public void prepareEncoding(DataOutputStream out) throws IOException {
if (encodingAlgo != null && encodingAlgo != DataBlockEncoding.NONE) {
@@ -133,7 +128,8 @@
}
private Bytes compressAfterEncoding(byte[] uncompressedBytesWithHeaderBuffer,
- int uncompressedBytesWithHeaderOffset, int uncompressedBytesWithHeaderLength, byte[] headerBytes)
+ int uncompressedBytesWithHeaderOffset, int uncompressedBytesWithHeaderLength,
+ byte[] headerBytes)
throws IOException {
Encryption.Context cryptoContext = fileContext.getEncryptionContext();
if (cryptoContext != Encryption.Context.NONE) {
@@ -158,7 +154,8 @@
compressedByteStream.reset();
compressionStream.resetState();
compressionStream.write(uncompressedBytesWithHeaderBuffer,
- headerBytes.length + uncompressedBytesWithHeaderOffset, uncompressedBytesWithHeaderLength - headerBytes.length);
+ headerBytes.length + uncompressedBytesWithHeaderOffset,
+ uncompressedBytesWithHeaderLength - headerBytes.length);
compressionStream.flush();
compressionStream.finish();
byte[] plaintext = compressedByteStream.toByteArray();
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockEncodingContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockEncodingContext.java
index 9602229..487dd45 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockEncodingContext.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockEncodingContext.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.hbase.io.encoding;
import java.io.IOException;
-
import org.apache.hadoop.hbase.io.hfile.BlockType;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.util.Bytes;
@@ -46,9 +45,6 @@
/**
* Do any action that needs to be performed after the encoding.
* Compression is also included if a non-null compression algorithm is used
- *
- * @param blockType
- * @throws IOException
*/
void postEncoding(BlockType blockType) throws IOException;
@@ -64,7 +60,6 @@
/**
* Sets the encoding state.
- * @param state
*/
void setEncodingState(EncodingState state);
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
index 176bea3..7af0c8c 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
@@ -21,9 +21,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.PrivateCellUtil;
@@ -195,9 +193,8 @@
}
@Override
- public EncodedSeeker createSeeker(CellComparator comparator,
- final HFileBlockDecodingContext decodingCtx) {
- return new BufferedEncodedSeeker<SeekerState>(comparator, decodingCtx) {
+ public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) {
+ return new BufferedEncodedSeeker<SeekerState>(decodingCtx) {
@Override
protected void decodeNext() {
current.keyLength = ByteBuff.readCompressedInt(currentBuffer);
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexCodecV1.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexCodecV1.java
index 12cd1ea..5de20dd 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexCodecV1.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexCodecV1.java
@@ -22,10 +22,7 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
-
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparator;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
@@ -71,7 +68,8 @@
+ "encoding context.");
}
- HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx;
+ HFileBlockDefaultEncodingContext encodingCtx =
+ (HFileBlockDefaultEncodingContext) blkEncodingCtx;
encodingCtx.prepareEncoding(out);
RowIndexEncoderV1 encoder = new RowIndexEncoderV1(out, encodingCtx);
@@ -115,8 +113,7 @@
dup.limit(sourceAsBuffer.position() + onDiskSize);
return dup.slice();
} else {
- RowIndexSeekerV1 seeker = new RowIndexSeekerV1(CellComparatorImpl.COMPARATOR,
- decodingCtx);
+ RowIndexSeekerV1 seeker = new RowIndexSeekerV1(decodingCtx);
seeker.setCurrentBuffer(new SingleByteBuff(sourceAsBuffer));
List<Cell> kvs = new ArrayList<>();
kvs.add(seeker.getCell());
@@ -151,9 +148,7 @@
}
@Override
- public EncodedSeeker createSeeker(CellComparator comparator,
- HFileBlockDecodingContext decodingCtx) {
- return new RowIndexSeekerV1(comparator, decodingCtx);
+ public EncodedSeeker createSeeker(HFileBlockDecodingContext decodingCtx) {
+ return new RowIndexSeekerV1(decodingCtx);
}
-
}
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexEncoderV1.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexEncoderV1.java
index 2388714..711b9db 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexEncoderV1.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexEncoderV1.java
@@ -12,9 +12,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
-
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
import org.apache.yetus.audience.InterfaceAudience;
@@ -32,10 +30,12 @@
private NoneEncoder encoder;
private int startOffset = -1;
private ByteArrayOutputStream rowsOffsetBAOS = new ByteArrayOutputStream(64 * 4);
+ private final HFileBlockEncodingContext context;
public RowIndexEncoderV1(DataOutputStream out, HFileBlockDefaultEncodingContext encodingCtx) {
this.out = out;
this.encoder = new NoneEncoder(out, encodingCtx);
+ this.context = encodingCtx;
}
public int write(Cell cell) throws IOException {
@@ -56,7 +56,7 @@
throw new IOException("Key cannot be null or empty");
}
if (lastCell != null) {
- int keyComp = CellComparatorImpl.COMPARATOR.compareRows(lastCell, cell);
+ int keyComp = this.context.getHFileContext().getCellComparator().compareRows(lastCell, cell);
if (keyComp > 0) {
throw new IOException("Added a key not lexically larger than"
+ " previous. Current cell = " + cell + ", lastCell = " + lastCell);
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java
index 9c0532e..7ff7555 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java
@@ -17,8 +17,6 @@
package org.apache.hadoop.hbase.io.encoding;
import java.nio.ByteBuffer;
-
-import org.apache.hadoop.hbase.ByteBufferExtendedCell;
import org.apache.hadoop.hbase.ByteBufferKeyOnlyKeyValue;
import org.apache.hadoop.hbase.ByteBufferKeyValue;
import org.apache.hadoop.hbase.Cell;
@@ -50,10 +48,11 @@
private int rowNumber;
private ByteBuff rowOffsets = null;
+ private final CellComparator cellComparator;
- public RowIndexSeekerV1(CellComparator comparator,
- HFileBlockDecodingContext decodingCtx) {
- super(comparator, decodingCtx);
+ public RowIndexSeekerV1(HFileBlockDecodingContext decodingCtx) {
+ super(decodingCtx);
+ this.cellComparator = decodingCtx.getHFileContext().getCellComparator();
}
@Override
@@ -131,8 +130,7 @@
int comp = 0;
while (low <= high) {
mid = low + ((high - low) >> 1);
- ByteBuffer row = getRow(mid);
- comp = compareRows(row, seekCell);
+ comp = this.cellComparator.compareRows(getRow(mid), seekCell);
if (comp < 0) {
low = mid + 1;
} else if (comp > 0) {
@@ -154,19 +152,6 @@
}
}
- private int compareRows(ByteBuffer row, Cell seekCell) {
- if (seekCell instanceof ByteBufferExtendedCell) {
- return ByteBufferUtils.compareTo(row, row.position(), row.remaining(),
- ((ByteBufferExtendedCell) seekCell).getRowByteBuffer(),
- ((ByteBufferExtendedCell) seekCell).getRowPosition(),
- seekCell.getRowLength());
- } else {
- return ByteBufferUtils.compareTo(row, row.position(), row.remaining(),
- seekCell.getRowArray(), seekCell.getRowOffset(),
- seekCell.getRowLength());
- }
- }
-
private ByteBuffer getRow(int index) {
int offset = rowOffsets.getIntAfterPosition(index * Bytes.SIZEOF_INT);
ByteBuff block = currentBuffer.duplicate();
@@ -191,8 +176,8 @@
}
}
do {
- int comp;
- comp = PrivateCellUtil.compareKeyIgnoresMvcc(comparator, seekCell, current.currentKey);
+ int comp =
+ PrivateCellUtil.compareKeyIgnoresMvcc(this.cellComparator, seekCell, current.currentKey);
if (comp == 0) { // exact match
if (seekBefore) {
if (!previous.isValid()) {
@@ -400,5 +385,4 @@
return ret;
}
}
-
}
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java
index d606497..ea4782d 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.io.hfile;
+import org.apache.hadoop.hbase.CellComparator;
+import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.compress.Compression;
@@ -28,9 +30,10 @@
import org.apache.yetus.audience.InterfaceAudience;
/**
- * This carries the information on some of the meta data about the HFile. This
- * meta data is used across the HFileWriter/Readers and the HFileBlocks.
- * This helps to add new information to the HFile.
+ * Read-only HFile Context Information. Meta data that is used by HFileWriter/Readers and by
+ * HFileBlocks. Create one using the {@link HFileContextBuilder} (See HFileInfo and the HFile
+ * Trailer class).
+ * @see HFileContextBuilder
*/
@InterfaceAudience.Private
public class HFileContext implements HeapSize, Cloneable {
@@ -42,7 +45,7 @@
//byte[] headers for column family and table name
2 * ClassSize.ARRAY + 2 * ClassSize.REFERENCE);
- public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
+ private static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
/** Whether checksum is enabled or not**/
private boolean usesHBaseChecksum = true;
@@ -67,6 +70,7 @@
private String hfileName;
private byte[] columnFamily;
private byte[] tableName;
+ private CellComparator cellComparator;
//Empty constructor. Go with setters
public HFileContext() {
@@ -74,7 +78,6 @@
/**
* Copy constructor
- * @param context
*/
public HFileContext(HFileContext context) {
this.usesHBaseChecksum = context.usesHBaseChecksum;
@@ -91,13 +94,14 @@
this.hfileName = context.hfileName;
this.columnFamily = context.columnFamily;
this.tableName = context.tableName;
+ this.cellComparator = context.cellComparator;
}
HFileContext(boolean useHBaseChecksum, boolean includesMvcc, boolean includesTags,
Compression.Algorithm compressAlgo, boolean compressTags, ChecksumType checksumType,
int bytesPerChecksum, int blockSize, DataBlockEncoding encoding,
Encryption.Context cryptoContext, long fileCreateTime, String hfileName,
- byte[] columnFamily, byte[] tableName) {
+ byte[] columnFamily, byte[] tableName, CellComparator cellComparator) {
this.usesHBaseChecksum = useHBaseChecksum;
this.includesMvcc = includesMvcc;
this.includesTags = includesTags;
@@ -114,11 +118,14 @@
this.hfileName = hfileName;
this.columnFamily = columnFamily;
this.tableName = tableName;
+ // If no cellComparator specified, make a guess based off tablename. If hbase:meta, then should
+ // be the meta table comparator. Comparators are per table.
+ this.cellComparator = cellComparator != null ? cellComparator : this.tableName != null ?
+ CellComparatorImpl.getCellComparator(this.tableName) : CellComparator.getInstance();
}
/**
- * @return true when on-disk blocks from this file are compressed, and/or encrypted;
- * false otherwise.
+ * @return true when on-disk blocks are compressed, and/or encrypted; false otherwise.
*/
public boolean isCompressedOrEncrypted() {
Compression.Algorithm compressAlgo = getCompression();
@@ -208,6 +215,11 @@
public byte[] getTableName() {
return this.tableName;
}
+
+ public CellComparator getCellComparator() {
+ return this.cellComparator;
+ }
+
/**
* HeapSize implementation. NOTE : The heap size should be altered when new state variable are
* added.
@@ -263,8 +275,9 @@
sb.append(", columnFamily=");
sb.append(Bytes.toStringBinary(columnFamily));
}
+ sb.append(", cellComparator=");
+ sb.append(this.cellComparator);
sb.append("]");
return sb.toString();
}
-
}
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java
index 5fa5626..a44f273 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.io.hfile;
+import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.crypto.Encryption;
@@ -56,6 +57,7 @@
private String hfileName = null;
private byte[] columnFamily = null;
private byte[] tableName = null;
+ private CellComparator cellComparator;
public HFileContextBuilder() {}
@@ -77,6 +79,7 @@
this.hfileName = hfc.getHFileName();
this.columnFamily = hfc.getColumnFamily();
this.tableName = hfc.getTableName();
+ this.cellComparator = hfc.getCellComparator();
}
public HFileContextBuilder withHBaseCheckSum(boolean useHBaseCheckSum) {
@@ -149,9 +152,14 @@
return this;
}
+ public HFileContextBuilder withCellComparator(CellComparator cellComparator) {
+ this.cellComparator = cellComparator;
+ return this;
+ }
+
public HFileContext build() {
return new HFileContext(usesHBaseChecksum, includesMvcc, includesTags, compression,
compressTags, checksumType, bytesPerChecksum, blocksize, encoding, cryptoContext,
- fileCreateTime, hfileName, columnFamily, tableName);
+ fileCreateTime, hfileName, columnFamily, tableName, cellComparator);
}
}
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
index 35a9052..ab1ede6 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -21,7 +21,6 @@
import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TIME_KEY;
import static org.apache.hadoop.hbase.regionserver.HStoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY;
import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY;
-
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.InetSocketAddress;
@@ -38,13 +37,11 @@
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
@@ -91,7 +88,6 @@
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
@@ -179,18 +175,17 @@
return combineTableNameSuffix(tableName, family);
}
- static <V extends Cell> RecordWriter<ImmutableBytesWritable, V>
- createRecordWriter(final TaskAttemptContext context, final OutputCommitter committer)
- throws IOException {
+ static <V extends Cell> RecordWriter<ImmutableBytesWritable, V> createRecordWriter(
+ final TaskAttemptContext context, final OutputCommitter committer) throws IOException {
// Get the path of the temporary output file
final Path outputDir = ((FileOutputCommitter)committer).getWorkPath();
final Configuration conf = context.getConfiguration();
- final boolean writeMultipleTables = conf.getBoolean(MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY, false) ;
+ final boolean writeMultipleTables =
+ conf.getBoolean(MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY, false);
final String writeTableNames = conf.get(OUTPUT_TABLE_NAME_CONF_KEY);
- if (writeTableNames==null || writeTableNames.isEmpty()) {
- throw new IllegalArgumentException("Configuration parameter " + OUTPUT_TABLE_NAME_CONF_KEY
- + " cannot be empty");
+ if (writeTableNames == null || writeTableNames.isEmpty()) {
+ throw new IllegalArgumentException("" + OUTPUT_TABLE_NAME_CONF_KEY + " cannot be empty");
}
final FileSystem fs = outputDir.getFileSystem(conf);
// These configs. are from hbase-*.xml
@@ -199,19 +194,12 @@
// Invented config. Add to hbase-*.xml if other than default compression.
final String defaultCompressionStr = conf.get("hfile.compression",
Compression.Algorithm.NONE.getName());
- final Algorithm defaultCompression = HFileWriterImpl
- .compressionByName(defaultCompressionStr);
+ final Algorithm defaultCompression = HFileWriterImpl.compressionByName(defaultCompressionStr);
String compressionStr = conf.get(COMPRESSION_OVERRIDE_CONF_KEY);
- final Algorithm overriddenCompression;
- if (compressionStr != null) {
- overriddenCompression = Compression.getCompressionAlgorithmByName(compressionStr);
- } else {
- overriddenCompression = null;
- }
-
+ final Algorithm overriddenCompression = compressionStr != null ?
+ Compression.getCompressionAlgorithmByName(compressionStr): null;
final boolean compactionExclude = conf.getBoolean(
"hbase.mapreduce.hfileoutputformat.compaction.exclude", false);
-
final Set<String> allTableNames = Arrays.stream(writeTableNames.split(
Bytes.toString(tableSeparator))).collect(Collectors.toSet());
@@ -224,24 +212,17 @@
String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_OVERRIDE_CONF_KEY);
final Map<byte[], DataBlockEncoding> datablockEncodingMap
= createFamilyDataBlockEncodingMap(conf);
- final DataBlockEncoding overriddenEncoding;
- if (dataBlockEncodingStr != null) {
- overriddenEncoding = DataBlockEncoding.valueOf(dataBlockEncodingStr);
- } else {
- overriddenEncoding = null;
- }
+ final DataBlockEncoding overriddenEncoding = dataBlockEncodingStr != null ?
+ DataBlockEncoding.valueOf(dataBlockEncodingStr) : null;
return new RecordWriter<ImmutableBytesWritable, V>() {
// Map of families to writers and how much has been output on the writer.
- private final Map<byte[], WriterLength> writers =
- new TreeMap<>(Bytes.BYTES_COMPARATOR);
- private final Map<byte[], byte[]> previousRows =
- new TreeMap<>(Bytes.BYTES_COMPARATOR);
+ private final Map<byte[], WriterLength> writers = new TreeMap<>(Bytes.BYTES_COMPARATOR);
+ private final Map<byte[], byte[]> previousRows = new TreeMap<>(Bytes.BYTES_COMPARATOR);
private final long now = EnvironmentEdgeManager.currentTime();
@Override
- public void write(ImmutableBytesWritable row, V cell)
- throws IOException {
+ public void write(ImmutableBytesWritable row, V cell) throws IOException {
Cell kv = cell;
// null input == user explicitly wants to flush
if (row == null && kv == null) {
@@ -255,12 +236,11 @@
byte[] tableNameBytes = null;
if (writeMultipleTables) {
tableNameBytes = MultiTableHFileOutputFormat.getTableName(row.get());
- tableNameBytes =
- TableName.valueOf(tableNameBytes).getNameWithNamespaceInclAsString()
+ tableNameBytes = TableName.valueOf(tableNameBytes).getNameWithNamespaceInclAsString()
.getBytes(Charset.defaultCharset());
if (!allTableNames.contains(Bytes.toString(tableNameBytes))) {
- throw new IllegalArgumentException("TableName '" + Bytes.toString(tableNameBytes) +
- "' not" + " expected");
+ throw new IllegalArgumentException("TableName " + Bytes.toString(tableNameBytes) +
+ " not expected");
}
} else {
tableNameBytes = Bytes.toBytes(writeTableNames);
@@ -275,9 +255,7 @@
if (wl == null) {
Path writerPath = null;
if (writeMultipleTables) {
- writerPath = new Path(outputDir,new Path(tableRelPath, Bytes
- .toString(family)));
-
+ writerPath = new Path(outputDir,new Path(tableRelPath, Bytes.toString(family)));
}
else {
writerPath = new Path(outputDir, Bytes.toString(family));
@@ -303,33 +281,23 @@
connection.getRegionLocator(TableName.valueOf(tableName))) {
loc = locator.getRegionLocation(rowKey);
} catch (Throwable e) {
- LOG.warn("There's something wrong when locating rowkey: " +
- Bytes.toString(rowKey) + " for tablename: " + tableName, e);
+ LOG.warn("Something wrong locating rowkey {} in {}",
+ Bytes.toString(rowKey), tableName, e);
loc = null;
} }
if (null == loc) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("failed to get region location, so use default writer for rowkey: " +
- Bytes.toString(rowKey));
- }
+ LOG.trace("Failed get of location, use default writer {}", Bytes.toString(rowKey));
wl = getNewWriter(tableNameBytes, family, conf, null);
} else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("first rowkey: [" + Bytes.toString(rowKey) + "]");
- }
+ LOG.debug("First rowkey: [{}]", Bytes.toString(rowKey));
InetSocketAddress initialIsa =
new InetSocketAddress(loc.getHostname(), loc.getPort());
if (initialIsa.isUnresolved()) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("failed to resolve bind address: " + loc.getHostname() + ":"
- + loc.getPort() + ", so use default writer");
- }
+ LOG.trace("Failed resolve address {}, use default writer", loc.getHostnamePort());
wl = getNewWriter(tableNameBytes, family, conf, null);
} else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("use favored nodes writer: " + initialIsa.getHostString());
- }
+ LOG.debug("Use favored nodes writer: {}", initialIsa.getHostString());
wl = getNewWriter(tableNameBytes, family, conf, new InetSocketAddress[] { initialIsa
});
}
@@ -369,29 +337,27 @@
private void closeWriter(WriterLength wl) throws IOException {
if (wl.writer != null) {
- LOG.info(
- "Writer=" + wl.writer.getPath() + ((wl.written == 0)? "": ", wrote=" + wl.written));
+ LOG.info("Writer=" + wl.writer.getPath() +
+ ((wl.written == 0)? "": ", wrote=" + wl.written));
close(wl.writer);
+ wl.writer = null;
}
- wl.writer = null;
wl.written = 0;
}
/*
* Create a new StoreFile.Writer.
- * @param family
* @return A WriterLength, containing a new StoreFile.Writer.
- * @throws IOException
*/
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="BX_UNBOXING_IMMEDIATELY_REBOXED",
justification="Not important")
- private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration
- conf, InetSocketAddress[] favoredNodes) throws IOException {
+ private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration conf,
+ InetSocketAddress[] favoredNodes) throws IOException {
byte[] tableAndFamily = getTableNameSuffixedWithFamily(tableName, family);
Path familydir = new Path(outputDir, Bytes.toString(family));
if (writeMultipleTables) {
familydir = new Path(outputDir,
- new Path(getTableRelativePath(tableName), Bytes.toString(family)));
+ new Path(getTableRelativePath(tableName), Bytes.toString(family)));
}
WriterLength wl = new WriterLength();
Algorithm compression = overriddenCompression;
@@ -409,12 +375,9 @@
encoding = encoding == null ? datablockEncodingMap.get(tableAndFamily) : encoding;
encoding = encoding == null ? DataBlockEncoding.NONE : encoding;
HFileContextBuilder contextBuilder = new HFileContextBuilder()
- .withCompression(compression)
- .withChecksumType(HStore.getChecksumType(conf))
- .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf))
- .withBlockSize(blockSize)
- .withColumnFamily(family)
- .withTableName(tableName);
+ .withCompression(compression).withChecksumType(HStore.getChecksumType(conf))
+ .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(blockSize)
+ .withColumnFamily(family).withTableName(tableName);
if (HFile.getFormatVersion(conf) >= HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
contextBuilder.withIncludesTags(true);
@@ -423,16 +386,13 @@
contextBuilder.withDataBlockEncoding(encoding);
HFileContext hFileContext = contextBuilder.build();
if (null == favoredNodes) {
- wl.writer =
- new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, fs)
- .withOutputDir(familydir).withBloomType(bloomType)
- .withComparator(CellComparator.getInstance()).withFileContext(hFileContext).build();
+ wl.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, fs)
+ .withOutputDir(familydir).withBloomType(bloomType)
+ .withFileContext(hFileContext).build();
} else {
- wl.writer =
- new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, new HFileSystem(fs))
- .withOutputDir(familydir).withBloomType(bloomType)
- .withComparator(CellComparator.getInstance()).withFileContext(hFileContext)
- .withFavoredNodes(favoredNodes).build();
+ wl.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, new HFileSystem(fs))
+ .withOutputDir(familydir).withBloomType(bloomType)
+ .withFileContext(hFileContext).withFavoredNodes(favoredNodes).build();
}
this.writers.put(tableAndFamily, wl);
@@ -441,22 +401,17 @@
private void close(final StoreFileWriter w) throws IOException {
if (w != null) {
- w.appendFileInfo(BULKLOAD_TIME_KEY,
- Bytes.toBytes(System.currentTimeMillis()));
- w.appendFileInfo(BULKLOAD_TASK_KEY,
- Bytes.toBytes(context.getTaskAttemptID().toString()));
- w.appendFileInfo(MAJOR_COMPACTION_KEY,
- Bytes.toBytes(true));
- w.appendFileInfo(EXCLUDE_FROM_MINOR_COMPACTION_KEY,
- Bytes.toBytes(compactionExclude));
+ w.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis()));
+ w.appendFileInfo(BULKLOAD_TASK_KEY, Bytes.toBytes(context.getTaskAttemptID().toString()));
+ w.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(true));
+ w.appendFileInfo(EXCLUDE_FROM_MINOR_COMPACTION_KEY, Bytes.toBytes(compactionExclude));
w.appendTrackedTimestampsToMetadata();
w.close();
}
}
@Override
- public void close(TaskAttemptContext c)
- throws IOException, InterruptedException {
+ public void close(TaskAttemptContext c) throws IOException, InterruptedException {
for (WriterLength wl: this.writers.values()) {
close(wl.writer);
}
@@ -496,21 +451,18 @@
throws IOException {
ArrayList<ImmutableBytesWritable> ret = new ArrayList<>();
- for(RegionLocator regionLocator : regionLocators)
- {
+ for(RegionLocator regionLocator : regionLocators) {
TableName tableName = regionLocator.getName();
LOG.info("Looking up current regions for table " + tableName);
byte[][] byteKeys = regionLocator.getStartKeys();
for (byte[] byteKey : byteKeys) {
byte[] fullKey = byteKey; //HFileOutputFormat2 use case
- if (writeMultipleTables)
- {
+ if (writeMultipleTables) {
//MultiTableHFileOutputFormat use case
fullKey = combineTableNameSuffix(tableName.getName(), byteKey);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("SplitPoint startkey for table [" + tableName + "]: [" + Bytes.toStringBinary
- (fullKey) + "]");
+ LOG.debug("SplitPoint startkey for " + tableName + ": " + Bytes.toStringBinary(fullKey));
}
ret.add(new ImmutableBytesWritable(fullKey));
}
@@ -537,8 +489,8 @@
TreeSet<ImmutableBytesWritable> sorted = new TreeSet<>(startKeys);
ImmutableBytesWritable first = sorted.first();
if (writeMultipleTables) {
- first = new ImmutableBytesWritable(MultiTableHFileOutputFormat.getSuffix(sorted.first
- ().get()));
+ first =
+ new ImmutableBytesWritable(MultiTableHFileOutputFormat.getSuffix(sorted.first().get()));
}
if (!first.equals(HConstants.EMPTY_BYTE_ARRAY)) {
throw new IllegalArgumentException(
@@ -641,11 +593,10 @@
/* Now get the region start keys for every table required */
List<String> allTableNames = new ArrayList<>(multiTableInfo.size());
- List<RegionLocator> regionLocators = new ArrayList<>( multiTableInfo.size());
- List<TableDescriptor> tableDescriptors = new ArrayList<>( multiTableInfo.size());
+ List<RegionLocator> regionLocators = new ArrayList<>(multiTableInfo.size());
+ List<TableDescriptor> tableDescriptors = new ArrayList<>(multiTableInfo.size());
- for( TableInfo tableInfo : multiTableInfo )
- {
+ for(TableInfo tableInfo : multiTableInfo) {
regionLocators.add(tableInfo.getRegionLocator());
String tn = writeMultipleTables?
tableInfo.getRegionLocator().getName().getNameWithNamespaceInclAsString():
@@ -653,10 +604,12 @@
allTableNames.add(tn);
tableDescriptors.add(tableInfo.getTableDescriptor());
}
- // Record tablenames for creating writer by favored nodes, and decoding compression, block size and other attributes of columnfamily per table
+ // Record tablenames for creating writer by favored nodes, and decoding compression,
+ // block size and other attributes of columnfamily per table
conf.set(OUTPUT_TABLE_NAME_CONF_KEY, StringUtils.join(allTableNames, Bytes
.toString(tableSeparator)));
- List<ImmutableBytesWritable> startKeys = getRegionStartKeys(regionLocators, writeMultipleTables);
+ List<ImmutableBytesWritable> startKeys =
+ getRegionStartKeys(regionLocators, writeMultipleTables);
// Use table's region boundaries for TOP split points.
LOG.info("Configuring " + startKeys.size() + " reduce partitions " +
"to match current region count for all tables");
@@ -852,9 +805,11 @@
TotalOrderPartitioner.setPartitionFile(conf, partitionsPath);
}
- @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value =
+ "RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
@VisibleForTesting
- static String serializeColumnFamilyAttribute(Function<ColumnFamilyDescriptor, String> fn, List<TableDescriptor> allTables)
+ static String serializeColumnFamilyAttribute(Function<ColumnFamilyDescriptor, String> fn,
+ List<TableDescriptor> allTables)
throws UnsupportedEncodingException {
StringBuilder attributeValue = new StringBuilder();
int i = 0;
@@ -869,8 +824,8 @@
attributeValue.append('&');
}
attributeValue.append(URLEncoder.encode(
- Bytes.toString(combineTableNameSuffix(tableDescriptor.getTableName().getName(), familyDescriptor.getName())),
- "UTF-8"));
+ Bytes.toString(combineTableNameSuffix(tableDescriptor.getTableName().getName(),
+ familyDescriptor.getName())), "UTF-8"));
attributeValue.append('=');
attributeValue.append(URLEncoder.encode(fn.apply(familyDescriptor), "UTF-8"));
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
index 3c74d11..35fb281 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
@@ -26,27 +26,25 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.CellComparatorImpl.MetaCellComparator;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.io.compress.Compression;
-import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos;
import org.apache.hadoop.hbase.util.Bytes;
-
+import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos;
/**
* The {@link HFile} has a fixed trailer which contains offsets to other
* variable parts of the file. Also includes basic metadata on this file. The
* trailer size is fixed within a given {@link HFile} format version only, but
* we always store the version number as the last four-byte integer of the file.
- * The version number itself is split into two portions, a major
+ * The version number itself is split into two portions, a major
* version and a minor version. The last three bytes of a file are the major
* version and a single preceding byte is the minor number. The major version
* determines which readers/writers to use to read/write a hfile while a minor
@@ -77,16 +75,24 @@
*/
private long loadOnOpenDataOffset;
- /** The number of entries in the root data index. */
+ /**
+ * The number of entries in the root data index.
+ */
private int dataIndexCount;
- /** Total uncompressed size of all blocks of the data index */
+ /**
+ * Total uncompressed size of all blocks of the data index
+ */
private long uncompressedDataIndexSize;
- /** The number of entries in the meta index */
+ /**
+ * The number of entries in the meta index
+ */
private int metaIndexCount;
- /** The total uncompressed size of keys/values stored in the file. */
+ /**
+ * The total uncompressed size of keys/values stored in the file.
+ */
private long totalUncompressedBytes;
/**
@@ -95,7 +101,9 @@
*/
private long entryCount;
- /** The compression codec used for all blocks. */
+ /**
+ * The compression codec used for all blocks.
+ */
private Compression.Algorithm compressionCodec = Compression.Algorithm.NONE;
/**
@@ -104,7 +112,9 @@
*/
private int numDataIndexLevels;
- /** The offset of the first data block. */
+ /**
+ * The offset of the first data block.
+ */
private long firstDataBlockOffset;
/**
@@ -113,17 +123,25 @@
*/
private long lastDataBlockOffset;
- /** Raw key comparator class name in version 3 */
+ /**
+ * Raw key comparator class name in version 3
+ */
// We could write the actual class name from 2.0 onwards and handle BC
private String comparatorClassName = CellComparator.getInstance().getClass().getName();
- /** The encryption key */
+ /**
+ * The encryption key
+ */
private byte[] encryptionKey;
- /** The {@link HFile} format major version. */
+ /**
+ * The {@link HFile} format major version.
+ */
private final int majorVersion;
- /** The {@link HFile} format minor version. */
+ /**
+ * The {@link HFile} format minor version.
+ */
private final int minorVersion;
FixedFileTrailer(int majorVersion, int minorVersion) {
@@ -133,7 +151,7 @@
}
private static int[] computeTrailerSizeByVersion() {
- int versionToSize[] = new int[HFile.MAX_FORMAT_VERSION + 1];
+ int[] versionToSize = new int[HFile.MAX_FORMAT_VERSION + 1];
// We support only 2 major versions now. ie. V2, V3
versionToSize[2] = 212;
for (int version = 3; version <= HFile.MAX_FORMAT_VERSION; version++) {
@@ -147,14 +165,13 @@
private static int getMaxTrailerSize() {
int maxSize = 0;
- for (int version = HFile.MIN_FORMAT_VERSION;
- version <= HFile.MAX_FORMAT_VERSION;
- ++version)
+ for (int version = HFile.MIN_FORMAT_VERSION; version <= HFile.MAX_FORMAT_VERSION; ++version) {
maxSize = Math.max(getTrailerSize(version), maxSize);
+ }
return maxSize;
}
- private static final int TRAILER_SIZE[] = computeTrailerSizeByVersion();
+ private static final int[] TRAILER_SIZE = computeTrailerSizeByVersion();
private static final int MAX_TRAILER_SIZE = getMaxTrailerSize();
private static final int NOT_PB_SIZE = BlockType.MAGIC_LENGTH + Bytes.SIZEOF_INT;
@@ -171,9 +188,6 @@
* Write the trailer to a data stream. We support writing version 1 for
* testing and for determining version 1 trailer size. It is also easy to see
* what fields changed in version 2.
- *
- * @param outputStream
- * @throws IOException
*/
void serialize(DataOutputStream outputStream) throws IOException {
HFile.checkFormatVersion(majorVersion);
@@ -193,18 +207,18 @@
@org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting
HFileProtos.FileTrailerProto toProtobuf() {
HFileProtos.FileTrailerProto.Builder builder = HFileProtos.FileTrailerProto.newBuilder()
- .setFileInfoOffset(fileInfoOffset)
- .setLoadOnOpenDataOffset(loadOnOpenDataOffset)
- .setUncompressedDataIndexSize(uncompressedDataIndexSize)
- .setTotalUncompressedBytes(totalUncompressedBytes)
- .setDataIndexCount(dataIndexCount)
- .setMetaIndexCount(metaIndexCount)
- .setEntryCount(entryCount)
- .setNumDataIndexLevels(numDataIndexLevels)
- .setFirstDataBlockOffset(firstDataBlockOffset)
- .setLastDataBlockOffset(lastDataBlockOffset)
- .setComparatorClassName(getHBase1CompatibleName(comparatorClassName))
- .setCompressionCodec(compressionCodec.ordinal());
+ .setFileInfoOffset(fileInfoOffset)
+ .setLoadOnOpenDataOffset(loadOnOpenDataOffset)
+ .setUncompressedDataIndexSize(uncompressedDataIndexSize)
+ .setTotalUncompressedBytes(totalUncompressedBytes)
+ .setDataIndexCount(dataIndexCount)
+ .setMetaIndexCount(metaIndexCount)
+ .setEntryCount(entryCount)
+ .setNumDataIndexLevels(numDataIndexLevels)
+ .setFirstDataBlockOffset(firstDataBlockOffset)
+ .setLastDataBlockOffset(lastDataBlockOffset)
+ .setComparatorClassName(getHBase1CompatibleName(comparatorClassName))
+ .setCompressionCodec(compressionCodec.ordinal());
if (encryptionKey != null) {
builder.setEncryptionKey(UnsafeByteOperations.unsafeWrap(encryptionKey));
}
@@ -239,9 +253,6 @@
* Deserialize the fixed file trailer from the given stream. The version needs
* to already be specified. Make sure this is consistent with
* {@link #serialize(DataOutputStream)}.
- *
- * @param inputStream
- * @throws IOException
*/
void deserialize(DataInputStream inputStream) throws IOException {
HFile.checkFormatVersion(majorVersion);
@@ -249,7 +260,7 @@
BlockType.TRAILER.readAndCheck(inputStream);
if (majorVersion > 2
- || (majorVersion == 2 && minorVersion >= HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION)) {
+ || (majorVersion == 2 && minorVersion >= HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION)) {
deserializeFromPB(inputStream);
} else {
deserializeFromWritable(inputStream);
@@ -263,14 +274,12 @@
/**
* Deserialize the file trailer as protobuf
- * @param inputStream
- * @throws IOException
*/
void deserializeFromPB(DataInputStream inputStream) throws IOException {
// read PB and skip padding
int start = inputStream.available();
HFileProtos.FileTrailerProto trailerProto =
- HFileProtos.FileTrailerProto.PARSER.parseDelimitedFrom(inputStream);
+ HFileProtos.FileTrailerProto.PARSER.parseDelimitedFrom(inputStream);
int size = start - inputStream.available();
inputStream.skip(getTrailerSize() - NOT_PB_SIZE - size);
@@ -320,8 +329,6 @@
/**
* Deserialize the file trailer as writable data
- * @param input
- * @throws IOException
*/
void deserializeFromWritable(DataInput input) throws IOException {
fileInfoOffset = input.readLong();
@@ -339,12 +346,13 @@
// TODO this is a classname encoded into an HFile's trailer. We are going to need to have
// some compat code here.
setComparatorClass(getComparatorClass(Bytes.readStringFixedSize(input,
- MAX_COMPARATOR_NAME_LENGTH)));
+ MAX_COMPARATOR_NAME_LENGTH)));
}
-
+
private void append(StringBuilder sb, String s) {
- if (sb.length() > 0)
+ if (sb.length() > 0) {
sb.append(", ");
+ }
sb.append(s);
}
@@ -375,18 +383,18 @@
/**
* Reads a file trailer from the given file.
*
- * @param istream the input stream with the ability to seek. Does not have to
- * be buffered, as only one read operation is made.
+ * @param istream the input stream with the ability to seek. Does not have to
+ * be buffered, as only one read operation is made.
* @param fileSize the file size. Can be obtained using
- * {@link org.apache.hadoop.fs.FileSystem#getFileStatus(
- * org.apache.hadoop.fs.Path)}.
+ * {@link org.apache.hadoop.fs.FileSystem#getFileStatus(
+ *org.apache.hadoop.fs.Path)}.
* @return the fixed file trailer read
* @throws IOException if failed to read from the underlying stream, or the
- * trailer is corrupted, or the version of the trailer is
- * unsupported
+ * trailer is corrupted, or the version of the trailer is
+ * unsupported
*/
public static FixedFileTrailer readFromStream(FSDataInputStream istream,
- long fileSize) throws IOException {
+ long fileSize) throws IOException {
int bufferSize = MAX_TRAILER_SIZE;
long seekPoint = fileSize - bufferSize;
if (seekPoint < 0) {
@@ -399,7 +407,7 @@
ByteBuffer buf = ByteBuffer.allocate(bufferSize);
istream.readFully(buf.array(), buf.arrayOffset(),
- buf.arrayOffset() + buf.limit());
+ buf.arrayOffset() + buf.limit());
// Read the version from the last int of the file.
buf.position(buf.limit() - Bytes.SIZEOF_INT);
@@ -415,30 +423,30 @@
FixedFileTrailer fft = new FixedFileTrailer(majorVersion, minorVersion);
fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(),
- buf.arrayOffset() + bufferSize - trailerSize, trailerSize)));
+ buf.arrayOffset() + bufferSize - trailerSize, trailerSize)));
return fft;
}
public void expectMajorVersion(int expected) {
if (majorVersion != expected) {
throw new IllegalArgumentException("Invalid HFile major version: "
- + majorVersion
- + " (expected: " + expected + ")");
+ + majorVersion
+ + " (expected: " + expected + ")");
}
}
public void expectMinorVersion(int expected) {
if (minorVersion != expected) {
throw new IllegalArgumentException("Invalid HFile minor version: "
- + minorVersion + " (expected: " + expected + ")");
+ + minorVersion + " (expected: " + expected + ")");
}
}
public void expectAtLeastMajorVersion(int lowerBound) {
if (majorVersion < lowerBound) {
throw new IllegalArgumentException("Invalid HFile major version: "
- + majorVersion
- + " (expected: " + lowerBound + " or higher).");
+ + majorVersion
+ + " (expected: " + lowerBound + " or higher).");
}
}
@@ -566,7 +574,7 @@
* than the new name; writing the new name will make it so newly-written hfiles are not parseable
* by hbase-1.x, a facility we'd like to preserve across rolling upgrade and hbase-1.x clusters
* reading hbase-2.x produce.
- *
+ * <p>
* The Comparators in hbase-2.x work the same as they did in hbase-1.x; they compare
* KeyValues. In hbase-2.x they were renamed making use of the more generic 'Cell'
* nomenclature to indicate that we intend to move away from KeyValues post hbase-2. A naming
@@ -579,9 +587,9 @@
*
* @param comparator String class name of the Comparator used in this hfile.
* @return What to store in the trailer as our comparator name.
+ * @see #getComparatorClass(String)
* @since hbase-2.0.0.
* @deprecated Since hbase-2.0.0. Will be removed in hbase-3.0.0.
- * @see #getComparatorClass(String)
*/
@Deprecated
private String getHBase1CompatibleName(final String comparator) {
@@ -596,20 +604,20 @@
@SuppressWarnings("unchecked")
private static Class<? extends CellComparator> getComparatorClass(String comparatorClassName)
- throws IOException {
+ throws IOException {
Class<? extends CellComparator> comparatorKlass;
// for BC
if (comparatorClassName.equals(KeyValue.COMPARATOR.getLegacyKeyComparatorName())
- || comparatorClassName.equals(KeyValue.COMPARATOR.getClass().getName())
- || (comparatorClassName.equals("org.apache.hadoop.hbase.CellComparator"))) {
+ || comparatorClassName.equals(KeyValue.COMPARATOR.getClass().getName())
+ || (comparatorClassName.equals("org.apache.hadoop.hbase.CellComparator"))) {
comparatorKlass = CellComparatorImpl.class;
} else if (comparatorClassName.equals(KeyValue.META_COMPARATOR.getLegacyKeyComparatorName())
- || comparatorClassName.equals(KeyValue.META_COMPARATOR.getClass().getName())
- || (comparatorClassName
- .equals("org.apache.hadoop.hbase.CellComparator$MetaCellComparator"))) {
+ || comparatorClassName.equals(KeyValue.META_COMPARATOR.getClass().getName())
+ || (comparatorClassName
+ .equals("org.apache.hadoop.hbase.CellComparator$MetaCellComparator"))) {
comparatorKlass = MetaCellComparator.class;
} else if (comparatorClassName.equals("org.apache.hadoop.hbase.KeyValue$RawBytesComparator")
- || comparatorClassName.equals("org.apache.hadoop.hbase.util.Bytes$ByteArrayComparator")) {
+ || comparatorClassName.equals("org.apache.hadoop.hbase.util.Bytes$ByteArrayComparator")) {
// When the comparator to be used is Bytes.BYTES_RAWCOMPARATOR, we just return null from here
// Bytes.BYTES_RAWCOMPARATOR is not a CellComparator
comparatorKlass = null;
@@ -624,19 +632,22 @@
return comparatorKlass;
}
- public static CellComparator createComparator(
- String comparatorClassName) throws IOException {
+ static CellComparator createComparator(String comparatorClassName) throws IOException {
+ if (comparatorClassName.equals(CellComparatorImpl.COMPARATOR.getClass().getName())) {
+ return CellComparatorImpl.COMPARATOR;
+ } else if (comparatorClassName.equals(
+ CellComparatorImpl.META_COMPARATOR.getClass().getName())) {
+ return CellComparatorImpl.META_COMPARATOR;
+ }
try {
-
Class<? extends CellComparator> comparatorClass = getComparatorClass(comparatorClassName);
- if(comparatorClass != null){
+ if (comparatorClass != null) {
return comparatorClass.getDeclaredConstructor().newInstance();
}
LOG.warn("No Comparator class for " + comparatorClassName + ". Returning Null.");
return null;
} catch (Exception e) {
- throw new IOException("Comparator class " + comparatorClassName +
- " is not instantiable", e);
+ throw new IOException("Comparator class " + comparatorClassName + " is not instantiable", e);
}
}
@@ -650,7 +661,7 @@
}
public void setUncompressedDataIndexSize(
- long uncompressedDataIndexSize) {
+ long uncompressedDataIndexSize) {
expectAtLeastMajorVersion(2);
this.uncompressedDataIndexSize = uncompressedDataIndexSize;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index cb6a352..46cec4a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -26,7 +26,6 @@
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.LongAdder;
-
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -52,7 +51,6 @@
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
@@ -121,7 +119,7 @@
* an HFile instance?
*/
@InterfaceAudience.Private
-public class HFile {
+public final class HFile {
// LOG is being used in HFileBlock and CheckSumUtil
static final Logger LOG = LoggerFactory.getLogger(HFile.class);
@@ -178,6 +176,11 @@
static final MetricsIO metrics = new MetricsIO(new MetricsIOWrapperImpl());
/**
+ * Shutdown constructor.
+ */
+ private HFile() {}
+
+ /**
* Number of checksum verification failures. It also
* clears the counter.
*/
@@ -222,10 +225,11 @@
*/
void addInlineBlockWriter(InlineBlockWriter bloomWriter);
- // The below three methods take Writables. We'd like to undo Writables but undoing the below would be pretty
- // painful. Could take a byte [] or a Message but we want to be backward compatible around hfiles so would need
- // to map between Message and Writable or byte [] and current Writable serialization. This would be a bit of work
- // to little gain. Thats my thinking at moment. St.Ack 20121129
+ // The below three methods take Writables. We'd like to undo Writables but undoing the below
+ // would be pretty painful. Could take a byte [] or a Message but we want to be backward
+ // compatible around hfiles so would need to map between Message and Writable or byte [] and
+ // current Writable serialization. This would be a bit of work to little gain. Thats my
+ // thinking at moment. St.Ack 20121129
void appendMetaBlock(String bloomFilterMetaKey, Writable metaWriter);
@@ -258,7 +262,6 @@
protected FileSystem fs;
protected Path path;
protected FSDataOutputStream ostream;
- protected CellComparator comparator = CellComparator.getInstance();
protected InetSocketAddress[] favoredNodes;
private HFileContext fileContext;
protected boolean shouldDropBehind = false;
@@ -282,12 +285,6 @@
return this;
}
- public WriterFactory withComparator(CellComparator comparator) {
- Preconditions.checkNotNull(comparator);
- this.comparator = comparator;
- return this;
- }
-
public WriterFactory withFavoredNodes(InetSocketAddress[] favoredNodes) {
// Deliberately not checking for null here.
this.favoredNodes = favoredNodes;
@@ -319,7 +316,7 @@
LOG.debug("Unable to set drop behind on {}", path.getName());
}
}
- return new HFileWriterImpl(conf, cacheConf, path, ostream, comparator, fileContext);
+ return new HFileWriterImpl(conf, cacheConf, path, ostream, fileContext);
}
}
@@ -349,16 +346,16 @@
CacheConfig cacheConf) {
int version = getFormatVersion(conf);
switch (version) {
- case 2:
- throw new IllegalArgumentException("This should never happen. " +
- "Did you change hfile.format.version to read v2? This version of the software writes v3" +
- " hfiles only (but it can read v2 files without having to update hfile.format.version " +
- "in hbase-site.xml)");
- case 3:
- return new HFile.WriterFactory(conf, cacheConf);
- default:
- throw new IllegalArgumentException("Cannot create writer for HFile " +
- "format version " + version);
+ case 2:
+ throw new IllegalArgumentException("This should never happen. " +
+ "Did you change hfile.format.version to read v2? This version of the software writes v3" +
+ " hfiles only (but it can read v2 files without having to update hfile.format.version " +
+ "in hbase-site.xml)");
+ case 3:
+ return new HFile.WriterFactory(conf, cacheConf);
+ default:
+ throw new IllegalArgumentException("Cannot create writer for HFile " +
+ "format version " + version);
}
}
@@ -372,18 +369,15 @@
* Read in a file block.
* @param offset offset to read.
* @param onDiskBlockSize size of the block
- * @param cacheBlock
- * @param pread
* @param isCompaction is this block being read as part of a compaction
* @param expectedBlockType the block type we are expecting to read with this read operation,
- * or null to read whatever block type is available and avoid checking (that might reduce
- * caching efficiency of encoded data blocks)
+ * or null to read whatever block type is available and avoid checking (that might reduce
+ * caching efficiency of encoded data blocks)
* @param expectedDataBlockEncoding the data block encoding the caller is expecting data blocks
- * to be in, or null to not perform this check and return the block irrespective of the
- * encoding. This check only applies to data blocks and can be set to null when the caller is
- * expecting to read a non-data block and has set expectedBlockType accordingly.
+ * to be in, or null to not perform this check and return the block irrespective of the
+ * encoding. This check only applies to data blocks and can be set to null when the caller is
+ * expecting to read a non-data block and has set expectedBlockType accordingly.
* @return Block wrapped in a ByteBuffer.
- * @throws IOException
*/
HFileBlock readBlock(long offset, long onDiskBlockSize,
boolean cacheBlock, final boolean pread, final boolean isCompaction,
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index 928b7a3..ec317e6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.io.hfile;
import static org.apache.hadoop.hbase.io.ByteBuffAllocator.HEAP;
-
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
@@ -29,18 +28,13 @@
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.io.ByteBuffAllocator;
-import org.apache.hadoop.hbase.io.util.BlockIOUtils;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
+import org.apache.hadoop.hbase.io.ByteBuffAllocator;
import org.apache.hadoop.hbase.io.ByteBuffInputStream;
import org.apache.hadoop.hbase.io.ByteBufferWriterDataOutputStream;
import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
@@ -50,6 +44,7 @@
import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext;
import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
+import org.apache.hadoop.hbase.io.util.BlockIOUtils;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.nio.MultiByteBuff;
import org.apache.hadoop.hbase.nio.SingleByteBuff;
@@ -57,7 +52,9 @@
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.ClassSize;
-
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
@@ -360,8 +357,8 @@
final long prevBlockOffset = buf.getLong(Header.PREV_BLOCK_OFFSET_INDEX);
// This constructor is called when we deserialize a block from cache and when we read a block in
// from the fs. fileCache is null when deserialized from cache so need to make up one.
- HFileContextBuilder fileContextBuilder =
- fileContext != null ? new HFileContextBuilder(fileContext) : new HFileContextBuilder();
+ HFileContextBuilder fileContextBuilder = fileContext != null ?
+ new HFileContextBuilder(fileContext) : new HFileContextBuilder();
fileContextBuilder.withHBaseCheckSum(usesHBaseChecksum);
int onDiskDataSizeWithHeader;
if (usesHBaseChecksum) {
@@ -407,8 +404,8 @@
/**
* @return the on-disk size of the next block (including the header size and any checksums if
- * present) read by peeking into the next block's header; use as a hint when doing
- * a read of the next block when scanning or running over a file.
+ * present) read by peeking into the next block's header; use as a hint when doing
+ * a read of the next block when scanning or running over a file.
*/
int getNextBlockOnDiskSize() {
return nextBlockOnDiskSize;
@@ -465,7 +462,7 @@
/**
* @return the uncompressed size of data part (header and checksum excluded).
*/
- int getUncompressedSizeWithoutHeader() {
+ int getUncompressedSizeWithoutHeader() {
return uncompressedSizeWithoutHeader;
}
@@ -705,8 +702,8 @@
}
/**
- * Cannot be {@link #UNSET}. Must be a legitimate value. Used re-making the {@link BlockCacheKey} when
- * block is returned to the cache.
+ * Cannot be {@link #UNSET}. Must be a legitimate value. Used re-making the {@link BlockCacheKey}
+ * when block is returned to the cache.
* @return the offset of this block in the file it was read from
*/
long getOffset() {
@@ -892,7 +889,6 @@
* Starts writing into the block. The previous block's data is discarded.
*
* @return the stream the user can write their data into
- * @throws IOException
*/
DataOutputStream startWriting(BlockType newBlockType)
throws IOException {
@@ -922,8 +918,6 @@
/**
* Writes the Cell to this block
- * @param cell
- * @throws IOException
*/
void write(Cell cell) throws IOException{
expectState(State.WRITING);
@@ -1041,9 +1035,6 @@
* Similar to {@link #writeHeaderAndData(FSDataOutputStream)}, but records
* the offset of this block so that it can be referenced in the next block
* of the same type.
- *
- * @param out
- * @throws IOException
*/
void writeHeaderAndData(FSDataOutputStream out) throws IOException {
long offset = out.getPos();
@@ -1061,9 +1052,7 @@
* data when not using compression) into the given stream. Can be called in
* the "writing" state or in the "block ready" state. If called in the
* "writing" state, transitions the writer to the "block ready" state.
- *
* @param out the output stream to write the
- * @throws IOException
*/
protected void finishBlockAndWriteHeaderAndData(DataOutputStream out)
throws IOException {
@@ -1082,7 +1071,6 @@
* the header + data + checksums stored on disk.
*
* @return header and data as they would be stored on disk in a byte array
- * @throws IOException
*/
byte[] getHeaderAndDataForTest() throws IOException {
ensureBlockReady();
@@ -1167,9 +1155,7 @@
* @return the number of bytes written
*/
public int encodedBlockSizeWritten() {
- if (state != State.WRITING)
- return 0;
- return this.encodedDataSizeWritten;
+ return state != State.WRITING ? 0 : this.encodedDataSizeWritten;
}
/**
@@ -1180,8 +1166,7 @@
* @return the number of bytes written
*/
int blockSizeWritten() {
- if (state != State.WRITING) return 0;
- return this.unencodedDataSizeWritten;
+ return state != State.WRITING ? 0 : this.unencodedDataSizeWritten;
}
/**
@@ -1234,7 +1219,6 @@
*
* @param bw the block-writable object to write as a block
* @param out the file system output stream
- * @throws IOException
*/
void writeBlock(BlockWritable bw, FSDataOutputStream out)
throws IOException {
@@ -1431,7 +1415,8 @@
* next blocks header seems unnecessary given we usually get the block size
* from the hfile index. Review!
*/
- private AtomicReference<PrefetchedHeader> prefetchedHeader = new AtomicReference<>(new PrefetchedHeader());
+ private AtomicReference<PrefetchedHeader> prefetchedHeader =
+ new AtomicReference<>(new PrefetchedHeader());
/** The size of the file we are reading from, or -1 if unknown. */
private long fileSize;
@@ -1643,10 +1628,9 @@
/**
* @return Check <code>onDiskSizeWithHeaderL</code> size is healthy and then return it as an int
- * @throws IOException
*/
private static int checkAndGetSizeAsInt(final long onDiskSizeWithHeaderL, final int hdrSize)
- throws IOException {
+ throws IOException {
if ((onDiskSizeWithHeaderL < hdrSize && onDiskSizeWithHeaderL != -1)
|| onDiskSizeWithHeaderL >= Integer.MAX_VALUE) {
throw new IOException("Invalid onDisksize=" + onDiskSizeWithHeaderL
@@ -1659,11 +1643,10 @@
/**
* Verify the passed in onDiskSizeWithHeader aligns with what is in the header else something
* is not right.
- * @throws IOException
*/
private void verifyOnDiskSizeMatchesHeader(final int passedIn, final ByteBuff headerBuf,
- final long offset, boolean verifyChecksum)
- throws IOException {
+ final long offset, boolean verifyChecksum)
+ throws IOException {
// Assert size provided aligns with what is in the header
int fromHeader = getOnDiskSizeWithHeader(headerBuf, verifyChecksum);
if (passedIn != fromHeader) {
@@ -1827,7 +1810,8 @@
@Override
public void setIncludesMemStoreTS(boolean includesMemstoreTS) {
- this.fileContext.setIncludesMvcc(includesMemstoreTS);
+ this.fileContext = new HFileContextBuilder(this.fileContext)
+ .withIncludesMvcc(includesMemstoreTS).build();
}
@Override
@@ -2073,7 +2057,7 @@
/**
* @return This HFileBlocks fileContext which will a derivative of the
- * fileContext for the file from which this block's data was originally read.
+ * fileContext for the file from which this block's data was originally read.
*/
HFileContext getHFileContext() {
return this.fileContext;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
index 89b3d34..1dd2573 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -33,7 +33,6 @@
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
-
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -52,7 +51,14 @@
import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos;
/**
- * Metadata for HFile. Conjured by the writer. Read in by the reader.
+ * Metadata Map of attributes for HFile written out as HFile Trailer. Created by the Writer and
+ * added to the tail of the file just before close. Metadata includes core attributes such as last
+ * key seen, comparator used writing the file, etc. Clients can add their own attributes via
+ * {@link #append(byte[], byte[], boolean)} and they'll be persisted and available at read time.
+ * Reader creates the HFileInfo on open by reading the tail of the HFile. The parse of the HFile
+ * trailer also creates a {@link HFileContext}, a read-only data structure that includes bulk of
+ * the HFileInfo and extras that is safe to pass around when working on HFiles.
+ * @see HFileContext
*/
@InterfaceAudience.Private
public class HFileInfo implements SortedMap<byte[], byte[]> {
@@ -62,7 +68,6 @@
static final byte [] AVG_KEY_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN");
static final byte [] AVG_VALUE_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN");
static final byte [] CREATE_TIME_TS = Bytes.toBytes(RESERVED_PREFIX + "CREATE_TIME_TS");
- static final byte [] COMPARATOR = Bytes.toBytes(RESERVED_PREFIX + "COMPARATOR");
static final byte [] TAGS_COMPRESSED = Bytes.toBytes(RESERVED_PREFIX + "TAGS_COMPRESSED");
public static final byte [] MAX_TAGS_LEN = Bytes.toBytes(RESERVED_PREFIX + "MAX_TAGS_LEN");
private final SortedMap<byte [], byte []> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);
@@ -380,7 +385,8 @@
HFileContextBuilder builder = new HFileContextBuilder()
.withHBaseCheckSum(true)
.withHFileName(path.getName())
- .withCompression(trailer.getCompressionCodec());
+ .withCompression(trailer.getCompressionCodec())
+ .withCellComparator(trailer.createComparator(trailer.getComparatorClassName()));
// Check for any key material available
byte[] keyBytes = trailer.getEncryptionKey();
if (keyBytes != null) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
index 2416074..5c51fe0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
@@ -22,7 +22,6 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Optional;
-
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -49,12 +48,10 @@
import org.apache.hadoop.hbase.util.IdLock;
import org.apache.hadoop.hbase.util.ObjectIntPair;
import org.apache.hadoop.io.WritableUtils;
-
import org.apache.htrace.core.TraceScope;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
@@ -85,9 +82,6 @@
*/
protected HFileDataBlockEncoder dataBlockEncoder = NoOpDataBlockEncoder.INSTANCE;
- /** Key comparator */
- protected CellComparator comparator = CellComparator.getInstance();
-
/** Block cache configuration. */
protected final CacheConfig cacheConf;
@@ -145,8 +139,6 @@
this.primaryReplicaReader = context.isPrimaryReplicaReader();
this.fileInfo = fileInfo;
this.trailer = fileInfo.getTrailer();
- // Comparator class name is stored in the trailer in version 2.
- this.comparator = trailer.createComparator();
this.hfileContext = fileInfo.getHFileContext();
this.fsBlockReader = new HFileBlock.FSReaderImpl(context, hfileContext,
cacheConf.getByteBuffAllocator());
@@ -238,7 +230,7 @@
/** @return comparator */
@Override
public CellComparator getComparator() {
- return comparator;
+ return this.hfileContext.getCellComparator();
}
@VisibleForTesting
@@ -398,8 +390,9 @@
}
protected void assertSeeked() {
- if (!isSeeked())
+ if (!isSeeked()) {
throw new NotSeekedException(reader.getPath());
+ }
}
@Override
@@ -476,7 +469,6 @@
/**
* Read mvcc. Does checks to see if we even need to read the mvcc at all.
- * @param offsetFromPos
*/
protected void readMvccVersion(final int offsetFromPos) {
// See if we even need to decode mvcc.
@@ -493,7 +485,6 @@
/**
* Actually do the mvcc read. Does no checks.
- * @param offsetFromPos
*/
private void _readMvccVersion(int offsetFromPos) {
// This is Bytes#bytesToVint inlined so can save a few instructions in this hot method; i.e.
@@ -681,7 +672,6 @@
* 0 if we are at the given key, 1 if we are past the given key
* -2 if the key is earlier than the first key of the file while
* using a faked index key
- * @throws IOException
*/
public int seekTo(Cell key, boolean rewind) throws IOException {
HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
@@ -747,7 +737,6 @@
* data block is found.
*
* @return the next block, or null if there are no more data blocks
- * @throws IOException
*/
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH",
justification="Yeah, unnecessary null check; could do w/ clean up")
@@ -764,7 +753,7 @@
}
if (block.getOffset() < 0) {
releaseIfNotCurBlock(block);
- throw new IOException("Invalid block file offset: " + block + ", path=" + reader.getPath());
+ throw new IOException("Invalid block offset: " + block + ", path=" + reader.getPath());
}
// We are reading the next block without block type validation, because
// it might turn out to be a non-data block.
@@ -786,8 +775,9 @@
@Override
public Cell getCell() {
- if (!isSeeked())
+ if (!isSeeked()) {
return null;
+ }
Cell ret;
int cellBufSize = getKVBufSize();
@@ -886,7 +876,6 @@
/**
* Set our selves up for the next 'next' invocation, set up next block.
* @return True is more to read else false if at the end.
- * @throws IOException
*/
private boolean positionForNextBlock() throws IOException {
// Methods are small so they get inlined because they are 'hot'.
@@ -942,7 +931,6 @@
*
* @return false if empty file; i.e. a call to next would return false and
* the current key and value are undefined.
- * @throws IOException
*/
@Override
public boolean seekTo() throws IOException {
@@ -975,7 +963,8 @@
isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
if (newBlock.getOffset() < 0) {
releaseIfNotCurBlock(newBlock);
- throw new IOException("Invalid block offset: " + newBlock.getOffset() + ", path=" + reader.getPath());
+ throw new IOException("Invalid offset=" + newBlock.getOffset() +
+ ", path=" + reader.getPath());
}
updateCurrentBlock(newBlock);
}
@@ -993,7 +982,6 @@
}
/**
- * @param v
* @return True if v <= 0 or v > current block buffer limit.
*/
protected final boolean checkKeyLen(final int v) {
@@ -1001,7 +989,6 @@
}
/**
- * @param v
* @return True if v < 0 or v > current block buffer limit.
*/
protected final boolean checkLen(final int v) {
@@ -1172,9 +1159,9 @@
// so blocks with the old encoding still linger in cache for some
// period of time. This event should be rare as it only happens on
// schema definition change.
- LOG.info("Evicting cached block with key " + cacheKey
- + " because of a data block encoding mismatch" + "; expected: "
- + expectedDataBlockEncoding + ", actual: " + actualDataBlockEncoding + ", path=" + path);
+ LOG.info("Evicting cached block with key {} because data block encoding mismatch; " +
+ "expected {}, actual {}, path={}", cacheKey, actualDataBlockEncoding,
+ expectedDataBlockEncoding, path);
// This is an error scenario. so here we need to release the block.
returnAndEvictBlock(cache, cacheKey, cachedBlock);
}
@@ -1192,10 +1179,8 @@
}
/**
- * @param metaBlockName
* @param cacheBlock Add block to cache, if found
* @return block wrapped in a ByteBuffer, with header skipped
- * @throws IOException
*/
@Override
public HFileBlock getMetaBlock(String metaBlockName, boolean cacheBlock)
@@ -1210,8 +1195,9 @@
byte[] mbname = Bytes.toBytes(metaBlockName);
int block = metaBlockIndexReader.rootBlockContainingKey(mbname,
0, mbname.length);
- if (block == -1)
+ if (block == -1) {
return null;
+ }
long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
// Per meta key from any given file, synchronize reads for said block. This
@@ -1418,7 +1404,6 @@
/**
* @return Midkey for this file. We work with block boundaries only so
* returned midkey is an approximation only.
- * @throws IOException
*/
@Override
public Optional<Cell> midKey() throws IOException {
@@ -1455,8 +1440,7 @@
DataBlockEncoding encoding = reader.getDataBlockEncoding();
dataBlockEncoder = encoding.getEncoder();
decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
- seeker = dataBlockEncoder.createSeeker(
- reader.getComparator(), decodingCtx);
+ seeker = dataBlockEncoder.createSeeker(decodingCtx);
}
@Override
@@ -1475,7 +1459,6 @@
* @param newBlock the block to make current, and read by {@link HFileReaderImpl#readBlock},
* it's a totally new block with new allocated {@link ByteBuff}, so if no further
* reference to this block, we should release it carefully.
- * @throws CorruptHFileException
*/
@Override
protected void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
@@ -1487,9 +1470,9 @@
short dataBlockEncoderId = newBlock.getDataBlockEncodingId();
if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
String encoderCls = dataBlockEncoder.getClass().getName();
- throw new CorruptHFileException(
- "Encoder " + encoderCls + " doesn't support data block encoding "
- + DataBlockEncoding.getNameFromId(dataBlockEncoderId) + ",path=" + reader.getPath());
+ throw new CorruptHFileException("Encoder " + encoderCls +
+ " doesn't support data block encoding " +
+ DataBlockEncoding.getNameFromId(dataBlockEncoderId) + ",path=" + reader.getPath());
}
updateCurrBlockRef(newBlock);
ByteBuff encodedBuffer = getEncodedBuffer(newBlock);
@@ -1606,7 +1589,7 @@
}
private DataInput getBloomFilterMetadata(BlockType blockType)
- throws IOException {
+ throws IOException {
if (blockType != BlockType.GENERAL_BLOOM_META &&
blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
throw new RuntimeException("Block Type: " + blockType.toString() +
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java
index 66a6c00..604ac1f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java
@@ -25,7 +25,6 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -65,7 +64,8 @@
private static final long UNSET = -1;
/** if this feature is enabled, preCalculate encoded data size before real encoding happens*/
- public static final String UNIFIED_ENCODED_BLOCKSIZE_RATIO = "hbase.writer.unified.encoded.blocksize.ratio";
+ public static final String UNIFIED_ENCODED_BLOCKSIZE_RATIO =
+ "hbase.writer.unified.encoded.blocksize.ratio";
/** Block size limit after encoding, used to unify encoded block Cache entry size*/
private final int encodedBlockSizeLimit;
@@ -94,9 +94,6 @@
/** Total uncompressed bytes, maybe calculate a compression ratio later. */
protected long totalUncompressedBytes = 0;
- /** Key comparator. Used to ensure we write in order. */
- protected final CellComparator comparator;
-
/** Meta block names. */
protected List<byte[]> metaNames = new ArrayList<>();
@@ -165,8 +162,7 @@
protected long maxMemstoreTS = 0;
public HFileWriterImpl(final Configuration conf, CacheConfig cacheConf, Path path,
- FSDataOutputStream outputStream,
- CellComparator comparator, HFileContext fileContext) {
+ FSDataOutputStream outputStream, HFileContext fileContext) {
this.outputStream = outputStream;
this.path = path;
this.name = path != null ? path.getName() : outputStream.toString();
@@ -177,8 +173,6 @@
} else {
this.blockEncoder = NoOpDataBlockEncoder.INSTANCE;
}
- this.comparator = comparator != null ? comparator : CellComparator.getInstance();
-
closeOutputStream = path != null;
this.cacheConf = cacheConf;
float encodeBlockSizeRatio = conf.getFloat(UNIFIED_ENCODED_BLOCKSIZE_RATIO, 1f);
@@ -187,7 +181,6 @@
if (LOG.isTraceEnabled()) {
LOG.trace("Writer" + (path != null ? " for " + path : "") +
" initialized with cacheConf: " + cacheConf +
- " comparator: " + comparator.getClass().getSimpleName() +
" fileContext: " + fileContext);
}
}
@@ -214,10 +207,9 @@
*
* @param trailer fixed file trailer
* @param out the data output to write the file info to
- * @throws IOException
*/
protected final void writeFileInfo(FixedFileTrailer trailer, DataOutputStream out)
- throws IOException {
+ throws IOException {
trailer.setFileInfoOffset(outputStream.getPos());
finishFileInfo();
long startTime = System.currentTimeMillis();
@@ -239,7 +231,8 @@
throw new IOException("Key cannot be null or empty");
}
if (lastCell != null) {
- int keyComp = PrivateCellUtil.compareKeyIgnoresMvcc(comparator, lastCell, cell);
+ int keyComp = PrivateCellUtil.compareKeyIgnoresMvcc(this.hFileContext.getCellComparator(),
+ lastCell, cell);
if (keyComp > 0) {
String message = getLexicalErrorMessage(cell);
throw new IOException(message);
@@ -285,8 +278,9 @@
}
public static Compression.Algorithm compressionByName(String algoName) {
- if (algoName == null)
+ if (algoName == null) {
return HFile.DEFAULT_COMPRESSION_ALGORITHM;
+ }
return Compression.getCompressionAlgorithmByName(algoName);
}
@@ -318,17 +312,15 @@
// Meta data block index writer
metaBlockIndexWriter = new HFileBlockIndex.BlockIndexWriter();
- if (LOG.isTraceEnabled()) LOG.trace("Initialized with " + cacheConf);
+ LOG.trace("Initialized with {}", cacheConf);
}
/**
* At a block boundary, write all the inline blocks and opens new block.
- *
- * @throws IOException
*/
protected void checkBlockBoundary() throws IOException {
- //for encoder like prefixTree, encoded size is not available, so we have to compare both encoded size
- //and unencoded size to blocksize limit.
+ // For encoder like prefixTree, encoded size is not available, so we have to compare both
+ // encoded size and unencoded size to blocksize limit.
if (blockWriter.encodedBlockSizeWritten() >= encodedBlockSizeLimit
|| blockWriter.blockSizeWritten() >= hFileContext.getBlocksize()) {
finishBlock();
@@ -339,7 +331,9 @@
/** Clean up the data block that is currently being written.*/
private void finishBlock() throws IOException {
- if (!blockWriter.isWriting() || blockWriter.blockSizeWritten() == 0) return;
+ if (!blockWriter.isWriting() || blockWriter.blockSizeWritten() == 0) {
+ return;
+ }
// Update the first data block offset if UNSET; used scanning.
if (firstDataBlockOffset == UNSET) {
@@ -350,7 +344,7 @@
blockWriter.writeHeaderAndData(outputStream);
int onDiskSize = blockWriter.getOnDiskSizeWithHeader();
Cell indexEntry =
- getMidpoint(this.comparator, lastCellOfPreviousBlock, firstCellInBlock);
+ getMidpoint(this.hFileContext.getCellComparator(), lastCellOfPreviousBlock, firstCellInBlock);
dataBlockIndexWriter.addEntry(PrivateCellUtil.getCellKeySerializedAsKeyValueKey(indexEntry),
lastDataBlockOffset, onDiskSize);
totalUncompressedBytes += blockWriter.getUncompressedSizeWithHeader();
@@ -364,11 +358,6 @@
* <code>right</code> but that is shorter; i.e. takes up less space. This
* trick is used building HFile block index. Its an optimization. It does not
* always work. In this case we'll just return the <code>right</code> cell.
- *
- * @param comparator
- * Comparator to use.
- * @param left
- * @param right
* @return A cell that sorts between <code>left</code> and <code>right</code>.
*/
public static Cell getMidpoint(final CellComparator comparator, final Cell left,
@@ -407,7 +396,9 @@
left.getRowLength(), right.getRowArray(), right.getRowOffset(), right.getRowLength());
}
// If midRow is null, just return 'right'. Can't do optimization.
- if (midRow == null) return right;
+ if (midRow == null) {
+ return right;
+ }
return PrivateCellUtil.createFirstOnRow(midRow);
}
// Rows are same. Compare on families.
@@ -428,7 +419,9 @@
right.getFamilyLength());
}
// If midRow is null, just return 'right'. Can't do optimization.
- if (midRow == null) return right;
+ if (midRow == null) {
+ return right;
+ }
// Return new Cell where we use right row and then a mid sort family.
return PrivateCellUtil.createFirstOnRowFamily(right, midRow, 0, midRow.length);
}
@@ -450,7 +443,9 @@
right.getQualifierLength());
}
// If midRow is null, just return 'right'. Can't do optimization.
- if (midRow == null) return right;
+ if (midRow == null) {
+ return right;
+ }
// Return new Cell where we use right row and family and then a mid sort qualifier.
return PrivateCellUtil.createFirstOnRowCol(right, midRow, 0, midRow.length);
}
@@ -459,12 +454,6 @@
}
/**
- * @param leftArray
- * @param leftOffset
- * @param leftLength
- * @param rightArray
- * @param rightOffset
- * @param rightLength
* @return Return a new array that is between left and right and minimally
* sized else just return null as indicator that we could not create a
* mid point.
@@ -566,8 +555,6 @@
/**
* Ready a new block for writing.
- *
- * @throws IOException
*/
protected void newBlock() throws IOException {
// This is where the next block begins.
@@ -678,7 +665,7 @@
dataBlockIndexWriter.getTotalUncompressedSize());
trailer.setFirstDataBlockOffset(firstDataBlockOffset);
trailer.setLastDataBlockOffset(lastDataBlockOffset);
- trailer.setComparatorClass(comparator.getClass());
+ trailer.setComparatorClass(this.hFileContext.getCellComparator().getClass());
trailer.setDataIndexCount(dataBlockIndexWriter.getNumRootEntries());
@@ -704,8 +691,9 @@
private void addBloomFilter(final BloomFilterWriter bfw,
final BlockType blockType) {
- if (bfw.getKeyCount() <= 0)
+ if (bfw.getKeyCount() <= 0) {
return;
+ }
if (blockType != BlockType.GENERAL_BLOOM_META &&
blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
@@ -722,8 +710,9 @@
public void writeToBlock(DataOutput out) throws IOException {
bfw.getMetaWriter().write(out);
Writable dataWriter = bfw.getDataWriter();
- if (dataWriter != null)
+ if (dataWriter != null) {
dataWriter.write(out);
+ }
}
});
}
@@ -739,7 +728,6 @@
*
* @param cell
* Cell to add. Cannot be empty nor null.
- * @throws IOException
*/
@Override
public void append(final Cell cell) throws IOException {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index 1d36446..1c00e25 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -35,13 +35,11 @@
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.TableName;
@@ -124,7 +122,6 @@
* Parses the string to a date.
* @param dateString The string format of a date, it's yyyymmdd.
* @return A date.
- * @throws ParseException
*/
public static Date parseDate(String dateString) throws ParseException {
return LOCAL_FORMAT.get().parse(dateString);
@@ -288,7 +285,6 @@
* @param columnDescriptor The descriptor of the current column family.
* @param cacheConfig The cacheConfig that disables the block cache.
* @param current The current time.
- * @throws IOException
*/
public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName,
ColumnFamilyDescriptor columnDescriptor, CacheConfig cacheConfig, long current)
@@ -384,7 +380,6 @@
* Gets the qualified root dir of the mob files.
* @param conf The current configuration.
* @return The qualified root dir.
- * @throws IOException
*/
public static Path getQualifiedMobRootDir(Configuration conf) throws IOException {
Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
@@ -505,7 +500,6 @@
* @param tableDir The table directory.
* @param family The name of the column family.
* @param storeFiles The files to be deleted.
- * @throws IOException
*/
public static void removeMobFiles(Configuration conf, FileSystem fs, TableName tableName,
Path tableDir, byte[] family, Collection<HStoreFile> storeFiles) throws IOException {
@@ -556,7 +550,6 @@
* @param cryptoContext The encryption context.
* @param isCompaction If the writer is used in compaction.
* @return The writer for the mob file.
- * @throws IOException
*/
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount,
@@ -580,7 +573,6 @@
* @param cryptoContext The encryption context.
* @param isCompaction If the writer is used in compaction.
* @return The writer for the mob file.
- * @throws IOException
*/
public static StoreFileWriter createRefFileWriter(Configuration conf, FileSystem fs,
ColumnFamilyDescriptor family, Path basePath, long maxKeyCount, CacheConfig cacheConfig,
@@ -607,7 +599,6 @@
* @param cryptoContext The encryption context.
* @param isCompaction If the writer is used in compaction.
* @return The writer for the mob file.
- * @throws IOException
*/
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount,
@@ -633,7 +624,6 @@
* @param cacheConfig The current cache config.
* @param cryptoContext The encryption context.
* @return The writer for the del file.
- * @throws IOException
*/
public static StoreFileWriter createDelFileWriter(Configuration conf, FileSystem fs,
ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount,
@@ -660,10 +650,9 @@
* @param cryptoContext The encryption context.
* @param isCompaction If the writer is used in compaction.
* @return The writer for the mob file.
- * @throws IOException
*/
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
- ColumnFamilyDescriptor family, MobFileName mobFileName, Path basePath, long maxKeyCount,
+ ColumnFamilyDescriptor family, MobFileName mobFileName, Path basePath, long maxKeyCount,
Compression.Algorithm compression, CacheConfig cacheConfig, Encryption.Context cryptoContext,
boolean isCompaction)
throws IOException {
@@ -689,7 +678,6 @@
* @param bloomType The bloom filter type.
* @param isCompaction If the writer is used in compaction.
* @return The writer for the mob file.
- * @throws IOException
*/
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
ColumnFamilyDescriptor family, Path path, long maxKeyCount,
@@ -718,8 +706,7 @@
.withCreateTime(EnvironmentEdgeManager.currentTime()).build();
StoreFileWriter w = new StoreFileWriter.Builder(conf, writerCacheConf, fs)
- .withFilePath(path)
- .withComparator(CellComparator.getInstance()).withBloomType(bloomType)
+ .withFilePath(path).withBloomType(bloomType)
.withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build();
return w;
}
@@ -732,7 +719,6 @@
* @param targetPath The directory path where the source file is renamed to.
* @param cacheConfig The current cache config.
* @return The target file path the source file is renamed to.
- * @throws IOException
*/
public static Path commitFile(Configuration conf, FileSystem fs, final Path sourceFile,
Path targetPath, CacheConfig cacheConfig) throws IOException {
@@ -837,7 +823,8 @@
* @param allFiles Whether add all mob files into the compaction.
*/
public static void doMobCompaction(Configuration conf, FileSystem fs, TableName tableName,
- ColumnFamilyDescriptor hcd, ExecutorService pool, boolean allFiles, LockManager.MasterLock lock)
+ ColumnFamilyDescriptor hcd, ExecutorService pool, boolean allFiles,
+ LockManager.MasterLock lock)
throws IOException {
String className = conf.get(MobConstants.MOB_COMPACTOR_CLASS_KEY,
PartitionedMobCompactor.class.getName());
@@ -943,7 +930,8 @@
* @param fileDate The date string parsed from the mob file name.
* @return True if the mob file is expired.
*/
- public static boolean isMobFileExpired(ColumnFamilyDescriptor column, long current, String fileDate) {
+ public static boolean isMobFileExpired(ColumnFamilyDescriptor column, long current,
+ String fileDate) {
if (column.getMinVersions() > 0) {
return false;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index dbc5876..fcdcb2c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -52,7 +52,6 @@
import java.util.function.ToLongFunction;
import java.util.stream.Collectors;
import java.util.stream.LongStream;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -132,7 +131,8 @@
* not be called directly but by an HRegion manager.
*/
@InterfaceAudience.Private
-public class HStore implements Store, HeapSize, StoreConfigInformation, PropagatingConfigurationObserver {
+public class HStore implements Store, HeapSize, StoreConfigInformation,
+ PropagatingConfigurationObserver {
public static final String MEMSTORE_CLASS_NAME = "hbase.regionserver.memstore.class";
public static final String COMPACTCHECKER_INTERVAL_MULTIPLIER_KEY =
"hbase.server.compactchecker.interval.multiplier";
@@ -237,11 +237,8 @@
/**
* Constructor
- * @param region
* @param family HColumnDescriptor for this column
- * @param confParam configuration object
- * failed. Can be null.
- * @throws IOException
+ * @param confParam configuration object failed. Can be null.
*/
protected HStore(final HRegion region, final ColumnFamilyDescriptor family,
final Configuration confParam, boolean warmup) throws IOException {
@@ -399,7 +396,6 @@
}
/**
- * @param family
* @return TTL in seconds of the specified family
*/
public static long determineTTLFromFamily(final ColumnFamilyDescriptor family) {
@@ -521,6 +517,7 @@
* @param hri {@link RegionInfo} for the region.
* @param family {@link ColumnFamilyDescriptor} describing the column family
* @return Path to family/Store home directory.
+ * @deprecated Since 05/05/2013, HBase-7808, hbase-1.0.0
*/
@Deprecated
public static Path getStoreHomedir(final Path tabledir,
@@ -533,6 +530,7 @@
* @param encodedName Encoded region name.
* @param family {@link ColumnFamilyDescriptor} describing the column family
* @return Path to family/Store home directory.
+ * @deprecated Since 05/05/2013, HBase-7808, hbase-1.0.0
*/
@Deprecated
public static Path getStoreHomedir(final Path tabledir,
@@ -558,7 +556,6 @@
/**
* Creates an unsorted list of StoreFile loaded in parallel
* from the given directory.
- * @throws IOException
*/
private List<HStoreFile> loadStoreFiles(boolean warmup) throws IOException {
Collection<StoreFileInfo> files = fs.getStoreFiles(getColumnFamilyName());
@@ -574,7 +571,8 @@
ThreadPoolExecutor storeFileOpenerThreadPool =
this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-"
+ this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName());
- CompletionService<HStoreFile> completionService = new ExecutorCompletionService<>(storeFileOpenerThreadPool);
+ CompletionService<HStoreFile> completionService =
+ new ExecutorCompletionService<>(storeFileOpenerThreadPool);
int totalValidStoreFile = 0;
for (StoreFileInfo storeFileInfo : files) {
@@ -596,9 +594,13 @@
compactedStoreFiles.addAll(storeFile.getCompactedStoreFiles());
}
} catch (InterruptedException e) {
- if (ioe == null) ioe = new InterruptedIOException(e.getMessage());
+ if (ioe == null) {
+ ioe = new InterruptedIOException(e.getMessage());
+ }
} catch (ExecutionException e) {
- if (ioe == null) ioe = new IOException(e.getCause());
+ if (ioe == null) {
+ ioe = new IOException(e.getCause());
+ }
}
}
} finally {
@@ -650,7 +652,6 @@
/**
* Replaces the store files that the store has with the given files. Mainly used by secondary
* region replicas to keep up to date with the primary region files.
- * @throws IOException
*/
public void refreshStoreFiles(Collection<String> newFiles) throws IOException {
List<StoreFileInfo> storeFiles = new ArrayList<>(newFiles.size());
@@ -665,15 +666,20 @@
* been opened, and removes the store file readers for store files no longer
* available. Mainly used by secondary region replicas to keep up to date with
* the primary region files.
- * @throws IOException
*/
private void refreshStoreFilesInternal(Collection<StoreFileInfo> newFiles) throws IOException {
StoreFileManager sfm = storeEngine.getStoreFileManager();
Collection<HStoreFile> currentFiles = sfm.getStorefiles();
Collection<HStoreFile> compactedFiles = sfm.getCompactedfiles();
- if (currentFiles == null) currentFiles = Collections.emptySet();
- if (newFiles == null) newFiles = Collections.emptySet();
- if (compactedFiles == null) compactedFiles = Collections.emptySet();
+ if (currentFiles == null) {
+ currentFiles = Collections.emptySet();
+ }
+ if (newFiles == null) {
+ newFiles = Collections.emptySet();
+ }
+ if (compactedFiles == null) {
+ compactedFiles = Collections.emptySet();
+ }
HashMap<StoreFileInfo, HStoreFile> currentFilesSet = new HashMap<>(currentFiles.size());
for (HStoreFile sf : currentFiles) {
@@ -868,12 +874,13 @@
}
prevCell = cell;
} while (scanner.next());
- LOG.info("Full verification complete for bulk load hfile: " + srcPath.toString()
- + " took " + (EnvironmentEdgeManager.currentTime() - verificationStartTime)
- + " ms");
+ LOG.info("Full verification complete for bulk load hfile: " + srcPath.toString() +
+ " took " + (EnvironmentEdgeManager.currentTime() - verificationStartTime) + " ms");
}
} finally {
- if (reader != null) reader.close();
+ if (reader != null) {
+ reader.close();
+ }
}
}
@@ -881,7 +888,6 @@
* This method should only be called from Region. It is assumed that the ranges of values in the
* HFile fit within the stores assigned region. (assertBulkLoadHFileOk checks this)
*
- * @param srcPathStr
* @param seqNum sequence Id associated with the HFile
*/
public Pair<Path, Path> preBulkLoadHFile(String srcPathStr, long seqNum) throws IOException {
@@ -993,13 +999,17 @@
ioe.initCause(e);
}
} catch (ExecutionException e) {
- if (ioe == null) ioe = new IOException(e.getCause());
+ if (ioe == null) {
+ ioe = new IOException(e.getCause());
+ }
}
}
} finally {
storeFileCloserThreadPool.shutdownNow();
}
- if (ioe != null) throw ioe;
+ if (ioe != null) {
+ throw ioe;
+ }
}
LOG.trace("Closed {}", this);
return result;
@@ -1027,9 +1037,6 @@
/**
* Write out current snapshot. Presumes {@link #snapshot()} has been called previously.
* @param logCacheFlushId flush sequence number
- * @param snapshot
- * @param status
- * @param throughputController
* @return The path name of the tmp file to which the store was flushed
* @throws IOException if exception occurs during process
*/
@@ -1081,10 +1088,7 @@
/**
* @param path The pathname of the tmp file into which the store was flushed
- * @param logCacheFlushId
- * @param status
* @return store file created.
- * @throws IOException
*/
private HStoreFile commitFile(Path path, long logCacheFlushId, MonitoredTask status)
throws IOException {
@@ -1107,7 +1111,6 @@
}
/**
- * @param maxKeyCount
* @param compression Compression algorithm to use
* @param isCompaction whether we are creating a new file in a compaction
* @param includeMVCCReadpoint - whether to include MVCC or not
@@ -1162,7 +1165,6 @@
StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf,
this.getFileSystem())
.withOutputDir(familyTempDir)
- .withComparator(comparator)
.withBloomType(family.getBloomFilterType())
.withMaxKeyCount(maxKeyCount)
.withFavoredNodes(favoredNodes)
@@ -1192,6 +1194,7 @@
.withColumnFamily(family.getName())
.withTableName(region.getTableDescriptor()
.getTableName().getName())
+ .withCellComparator(this.comparator)
.build();
return hFileContext;
}
@@ -1204,8 +1207,6 @@
/**
* Change storeFiles adding into place the Reader produced by this new flush.
* @param sfs Store files
- * @param snapshotId
- * @throws IOException
* @return Whether compaction is required.
*/
private boolean updateStorefiles(List<HStoreFile> sfs, long snapshotId) throws IOException {
@@ -1237,7 +1238,6 @@
/**
* Notify all observers that set of Readers has changed.
- * @throws IOException
*/
private void notifyChangedReadersObservers(List<HStoreFile> sfs) throws IOException {
for (ChangedReadersObserver o : this.changedReaderObservers) {
@@ -1454,7 +1454,6 @@
* See HBASE-2231 for details.
*
* @param compaction compaction details obtained from requestCompaction()
- * @throws IOException
* @return Storefile we compacted into or null if we failed or opted out early.
*/
public List<HStoreFile> compact(CompactionContext compaction,
@@ -1521,8 +1520,8 @@
return sfs;
}
- private List<HStoreFile> moveCompactedFilesIntoPlace(CompactionRequestImpl cr, List<Path> newFiles,
- User user) throws IOException {
+ private List<HStoreFile> moveCompactedFilesIntoPlace(CompactionRequestImpl cr,
+ List<Path> newFiles, User user) throws IOException {
List<HStoreFile> sfs = new ArrayList<>(newFiles.size());
for (Path newFile : newFiles) {
assert newFile != null;
@@ -1560,7 +1559,8 @@
newFiles.stream().map(HStoreFile::getPath).collect(Collectors.toList());
RegionInfo info = this.region.getRegionInfo();
CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,
- family.getName(), inputPaths, outputPaths, fs.getStoreDir(getColumnFamilyDescriptor().getNameAsString()));
+ family.getName(), inputPaths, outputPaths,
+ fs.getStoreDir(getColumnFamilyDescriptor().getNameAsString()));
// Fix reaching into Region to get the maxWaitForSeqId.
// Does this method belong in Region altogether given it is making so many references up there?
// Could be Region#writeCompactionMarker(compactionDescriptor);
@@ -1664,7 +1664,6 @@
* Call to complete a compaction. Its for the case where we find in the WAL a compaction
* that was not finished. We could find one recovering a WAL after a regionserver crash.
* See HBASE-2231.
- * @param compaction
*/
public void replayCompactionMarker(CompactionDescriptor compaction, boolean pickCompactionFiles,
boolean removeFiles) throws IOException {
@@ -1917,7 +1916,9 @@
}
private void removeUnneededFiles() throws IOException {
- if (!conf.getBoolean("hbase.store.delete.expired.storefile", true)) return;
+ if (!conf.getBoolean("hbase.store.delete.expired.storefile", true)) {
+ return;
+ }
if (getColumnFamilyDescriptor().getMinVersions() > 0) {
LOG.debug("Skipping expired store file removal due to min version being {}",
getColumnFamilyDescriptor().getMinVersions());
@@ -2120,7 +2121,6 @@
* @param readPt the read point of the current scane
* @param includeMemstoreScanner whether the current scanner should include memstorescanner
* @return list of scanners recreated on the current Scanners
- * @throws IOException
*/
public List<KeyValueScanner> recreateScanners(List<KeyValueScanner> currentFileScanners,
boolean cacheBlocks, boolean usePread, boolean isCompaction, ScanQueryMatcher matcher,
@@ -2319,7 +2319,6 @@
* This operation is atomic on each KeyValue (row/family/qualifier) but not necessarily atomic
* across all of them.
* @param readpoint readpoint below which we can safely remove duplicate KVs
- * @throws IOException
*/
public void upsert(Iterable<Cell> cells, long readpoint, MemStoreSizing memstoreSizing)
throws IOException {
@@ -2433,7 +2432,6 @@
* snapshot depending on dropMemstoreSnapshot argument.
* @param fileNames names of the flushed files
* @param dropMemstoreSnapshot whether to drop the prepared memstore snapshot
- * @throws IOException
*/
@Override
public void replayFlush(List<String> fileNames, boolean dropMemstoreSnapshot)
@@ -2465,7 +2463,6 @@
/**
* Abort the snapshot preparation. Drops the snapshot if any.
- * @throws IOException
*/
@Override
public void abort() throws IOException {
@@ -2720,7 +2717,8 @@
LOG.debug("Moving the files {} to archive", filesToRemove);
// Only if this is successful it has to be removed
try {
- this.fs.removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(), filesToRemove);
+ this.fs.removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(),
+ filesToRemove);
} catch (FailedArchiveException fae) {
// Even if archiving some files failed, we still need to clear out any of the
// files which were successfully archived. Otherwise we will receive a
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
index 7b4fb4a..02665a3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
@@ -26,7 +26,6 @@
import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAX_SEQ_ID_KEY;
import static org.apache.hadoop.hbase.regionserver.HStoreFile.MOB_CELLS_COUNT;
import static org.apache.hadoop.hbase.regionserver.HStoreFile.TIMERANGE_KEY;
-
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Collection;
@@ -37,15 +36,13 @@
import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
@@ -62,9 +59,7 @@
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
-
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
/**
@@ -94,7 +89,6 @@
* @param fs file system to write to
* @param path file name to create
* @param conf user configuration
- * @param comparator key comparator
* @param bloomType bloom filter setting
* @param maxKeys the expected maximum number of keys to be added. Was used
* for Bloom filter size in {@link HFile} format version 1.
@@ -105,15 +99,14 @@
* @throws IOException problem writing to FS
*/
private StoreFileWriter(FileSystem fs, Path path, final Configuration conf, CacheConfig cacheConf,
- final CellComparator comparator, BloomType bloomType, long maxKeys,
- InetSocketAddress[] favoredNodes, HFileContext fileContext, boolean shouldDropCacheBehind,
- Supplier<Collection<HStoreFile>> compactedFilesSupplier) throws IOException {
+ BloomType bloomType, long maxKeys, InetSocketAddress[] favoredNodes, HFileContext fileContext,
+ boolean shouldDropCacheBehind, Supplier<Collection<HStoreFile>> compactedFilesSupplier)
+ throws IOException {
this.compactedFilesSupplier = compactedFilesSupplier;
this.timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC);
// TODO : Change all writers to be specifically created for compaction context
writer = HFile.getWriterFactory(conf, cacheConf)
.withPath(fs, path)
- .withComparator(comparator)
.withFavoredNodes(favoredNodes)
.withFileContext(fileContext)
.withShouldDropCacheBehind(shouldDropCacheBehind)
@@ -135,14 +128,16 @@
// init bloom context
switch (bloomType) {
case ROW:
- bloomContext = new RowBloomContext(generalBloomFilterWriter, comparator);
+ bloomContext =
+ new RowBloomContext(generalBloomFilterWriter, fileContext.getCellComparator());
break;
case ROWCOL:
- bloomContext = new RowColBloomContext(generalBloomFilterWriter, comparator);
+ bloomContext =
+ new RowColBloomContext(generalBloomFilterWriter, fileContext.getCellComparator());
break;
case ROWPREFIX_FIXED_LENGTH:
- bloomContext = new RowPrefixFixedLengthBloomContext(generalBloomFilterWriter, comparator,
- Bytes.toInt(bloomParam));
+ bloomContext = new RowPrefixFixedLengthBloomContext(generalBloomFilterWriter,
+ fileContext.getCellComparator(), Bytes.toInt(bloomParam));
break;
default:
throw new IOException(
@@ -159,7 +154,8 @@
this.deleteFamilyBloomFilterWriter = BloomFilterFactory
.createDeleteBloomAtWrite(conf, cacheConf,
(int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
- deleteFamilyBloomContext = new RowBloomContext(deleteFamilyBloomFilterWriter, comparator);
+ deleteFamilyBloomContext =
+ new RowBloomContext(deleteFamilyBloomFilterWriter, fileContext.getCellComparator());
} else {
deleteFamilyBloomFilterWriter = null;
}
@@ -396,7 +392,6 @@
}
/**
- * @param fs
* @param dir Directory to create file in.
* @return random filename inside passed <code>dir</code>
*/
@@ -414,7 +409,6 @@
private final CacheConfig cacheConf;
private final FileSystem fs;
- private CellComparator comparator = CellComparator.getInstance();
private BloomType bloomType = BloomType.NONE;
private long maxKeyCount = 0;
private Path dir;
@@ -473,12 +467,6 @@
return this;
}
- public Builder withComparator(CellComparator comparator) {
- Preconditions.checkNotNull(comparator);
- this.comparator = comparator;
- return this;
- }
-
public Builder withBloomType(BloomType bloomType) {
Preconditions.checkNotNull(bloomType);
this.bloomType = bloomType;
@@ -545,11 +533,7 @@
}
}
- if (comparator == null) {
- comparator = CellComparator.getInstance();
- }
-
- return new StoreFileWriter(fs, filePath, conf, cacheConf, comparator, bloomType, maxKeyCount,
+ return new StoreFileWriter(fs, filePath, conf, cacheConf, bloomType, maxKeyCount,
favoredNodes, fileContext, shouldDropCacheBehind, compactedFilesSupplier);
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRecoveredHFilesOutputSink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRecoveredHFilesOutputSink.java
index 96ab8e6..406e90d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRecoveredHFilesOutputSink.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRecoveredHFilesOutputSink.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.wal;
import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
-
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.HashMap;
@@ -29,7 +28,6 @@
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
-
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -208,7 +206,7 @@
.withFilePath(outputFile);
HFileContextBuilder hFileContextBuilder = new HFileContextBuilder();
if (isMetaTable) {
- writerBuilder.withComparator(CellComparatorImpl.META_COMPARATOR);
+ hFileContextBuilder.withCellComparator(CellComparatorImpl.META_COMPARATOR);
} else {
configContextForNonMetaWriter(tableName, familyName, hFileContextBuilder, writerBuilder);
}
@@ -224,9 +222,9 @@
TableDescriptor tableDesc = tableDescCache.get(tableName);
ColumnFamilyDescriptor cfd = tableDesc.getColumnFamily(Bytes.toBytesBinary(familyName));
hFileContextBuilder.withCompression(cfd.getCompressionType()).withBlockSize(cfd.getBlocksize())
- .withCompressTags(cfd.isCompressTags()).withDataBlockEncoding(cfd.getDataBlockEncoding());
- writerBuilder.withBloomType(cfd.getBloomFilterType())
- .withComparator(CellComparatorImpl.COMPARATOR);
+ .withCompressTags(cfd.isCompressTags()).withDataBlockEncoding(cfd.getDataBlockEncoding())
+ .withCellComparator(CellComparatorImpl.COMPARATOR);
+ writerBuilder.withBloomType(cfd.getBloomFilterType());
}
private void checkPathValid(Path outputFile) throws IOException {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
index 90fb8a3..2c4209c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
@@ -380,7 +380,6 @@
writer = HFile.getWriterFactoryNoCache(conf)
.withPath(fs, mf)
.withFileContext(hFileContext)
- .withComparator(CellComparator.getInstance())
.create();
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
index 569ede2..3f21f0c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -206,8 +206,8 @@
.withIncludesTags(includesTags)
.withCompression(Compression.Algorithm.NONE)
.build();
- DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR,
- encoder.newDataBlockDecodingContext(meta));
+ DataBlockEncoder.EncodedSeeker seeker =
+ encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
encodedSeekers.add(seeker);
}
@@ -279,8 +279,8 @@
.withIncludesTags(includesTags)
.withCompression(Compression.Algorithm.NONE)
.build();
- DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR,
- encoder.newDataBlockDecodingContext(meta));
+ DataBlockEncoder.EncodedSeeker seeker =
+ encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
int i = 0;
do {
@@ -341,8 +341,8 @@
HFileContext meta =
new HFileContextBuilder().withHBaseCheckSum(false).withIncludesMvcc(includesMemstoreTS)
.withIncludesTags(includesTags).withCompression(Compression.Algorithm.NONE).build();
- DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR,
- encoder.newDataBlockDecodingContext(meta));
+ DataBlockEncoder.EncodedSeeker seeker =
+ encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
Cell cell = seeker.getCell();
Assert.assertEquals(expectedKV.getLength(), ((KeyValue) cell).getLength());
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekToBlockWithEncoders.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekToBlockWithEncoders.java
index e7b1426..d9f22bc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekToBlockWithEncoders.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekToBlockWithEncoders.java
@@ -25,7 +25,6 @@
import java.util.Collection;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
import org.apache.hadoop.hbase.HConstants;
@@ -290,8 +289,8 @@
HFILEBLOCK_DUMMY_HEADER, meta);
ByteBuffer encodedBuffer = TestDataBlockEncoders.encodeKeyValues(encoding, kvs,
encodingContext, this.useOffheapData);
- DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR,
- encoder.newDataBlockDecodingContext(meta));
+ DataBlockEncoder.EncodedSeeker seeker =
+ encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
encodedSeekers.add(seeker);
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
index cb7042a..609ff9d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
@@ -37,7 +37,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ArrayBackedTag;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -394,7 +393,7 @@
.withDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding())
.withIncludesTags(useTags).build();
StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConf, fs)
- .withOutputDir(storeFileParentDir).withComparator(CellComparatorImpl.COMPARATOR)
+ .withOutputDir(storeFileParentDir)
.withFileContext(meta)
.withBloomType(BLOOM_TYPE).withMaxKeyCount(NUM_KV).build();
byte[] cf = Bytes.toBytes("fam");
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
index d25ce47..14006a8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
@@ -131,6 +131,10 @@
t.createComparator(KeyValue.META_COMPARATOR.getClass().getName()).getClass());
assertEquals(CellComparatorImpl.MetaCellComparator.class, t.createComparator(
CellComparatorImpl.MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
+ assertEquals(CellComparatorImpl.META_COMPARATOR.getClass(), t.createComparator(
+ CellComparatorImpl.MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
+ assertEquals(CellComparatorImpl.COMPARATOR.getClass(), t.createComparator(
+ CellComparatorImpl.MetaCellComparator.COMPARATOR.getClass().getName()).getClass());
assertNull(t.createComparator(Bytes.BYTES_RAWCOMPARATOR.getClass().getName()));
assertNull(t.createComparator("org.apache.hadoop.hbase.KeyValue$RawBytesComparator"));
} catch (IOException e) {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
index ef7400d..b3010ed 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
@@ -50,7 +50,6 @@
import org.apache.hadoop.hbase.CellBuilder;
import org.apache.hadoop.hbase.CellBuilderFactory;
import org.apache.hadoop.hbase.CellBuilderType;
-import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
@@ -277,7 +276,7 @@
HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build();
StoreFileWriter sfw =
new StoreFileWriter.Builder(conf, fs).withOutputDir(storeFileParentDir)
- .withComparator(CellComparatorImpl.COMPARATOR).withFileContext(meta).build();
+ .withFileContext(meta).build();
final int rowLen = 32;
Random RNG = new Random();
@@ -360,7 +359,7 @@
.withCompressTags(false)
.build();
HFileWriterImpl writer = new HFileWriterImpl(conf, cacheConf, path, mockedOutputStream,
- CellComparator.getInstance(), fileContext);
+ fileContext);
CellBuilder cellBuilder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY);
byte[] row = Bytes.toBytes("foo");
byte[] qualifier = Bytes.toBytes("qualifier");
@@ -516,7 +515,6 @@
Writer writer = HFile.getWriterFactory(conf, cacheConf)
.withOutputStream(fout)
.withFileContext(meta)
- .withComparator(CellComparatorImpl.COMPARATOR)
.create();
LOG.info(Objects.toString(writer));
writeRecords(writer, useTags);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java
index 41568b3..8c3a632 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java
@@ -24,7 +24,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.KeyValue;
@@ -66,9 +65,8 @@
HFileContext context =
new HFileContextBuilder().withBlockSize(blocksize).withIncludesTags(true).build();
Configuration conf = TEST_UTIL.getConfiguration();
- HFile.Writer writer =
- HFile.getWriterFactoryNoCache(conf).withOutputStream(fout).withFileContext(context)
- .withComparator(CellComparatorImpl.COMPARATOR).create();
+ HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
+ .withOutputStream(fout).withFileContext(context).create();
// 4 bytes * 3 * 2 for each key/value +
// 3 for keys, 15 for values = 42 (woot)
writer.append(toKV("c"));
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileScannerImplReferenceCount.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileScannerImplReferenceCount.java
index 47cbd85..e326ecc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileScannerImplReferenceCount.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileScannerImplReferenceCount.java
@@ -37,7 +37,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
@@ -165,7 +164,7 @@
.withCompression(compression).withDataBlockEncoding(encoding).build();
try (HFile.Writer writer =
new HFile.WriterFactory(conf, new CacheConfig(conf)).withPath(fs, hfilePath)
- .withFileContext(context).withComparator(CellComparatorImpl.COMPARATOR).create()) {
+ .withFileContext(context).create()) {
Random rand = new Random(9713312); // Just a fixed seed.
for (int i = 0; i < cellCount; ++i) {
byte[] keyBytes = Bytes.add(Bytes.toBytes(i), SUFFIX);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
index 0dc3539..d6a97c3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
@@ -27,7 +27,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.KeyValue;
@@ -142,7 +141,6 @@
Writer writer = HFile.getWriterFactoryNoCache(conf)
.withOutputStream(fout)
.withFileContext(context)
- .withComparator(CellComparatorImpl.COMPARATOR)
.create();
try {
BytesWritable key = new BytesWritable();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java
index 4f4d36b..3cec836 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java
@@ -36,7 +36,6 @@
import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -132,7 +131,6 @@
HFile.Writer writer = new HFile.WriterFactory(conf, cacheConfig)
.withPath(fs, hfilePath)
.withFileContext(context)
- .withComparator(CellComparatorImpl.COMPARATOR)
.create();
Random rand = new Random(9713312); // Just a fixed seed.
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
index 1365680..da62e88 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
@@ -26,7 +26,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -145,7 +144,6 @@
.build();
StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConf, fs)
.withOutputDir(storeFileParentDir)
- .withComparator(CellComparatorImpl.COMPARATOR)
.withFileContext(meta)
.build();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
index 1ae861c..de68578 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
@@ -25,7 +25,6 @@
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ArrayBackedTag;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
@@ -69,8 +68,6 @@
TEST_UTIL.getConfiguration(), cacheConf)
.withOutputStream(fout)
.withFileContext(context)
- // NOTE: This test is dependent on this deprecated nonstandard comparator
- .withComparator(CellComparatorImpl.COMPARATOR)
.create();
int numberOfKeys = 1000;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
index 9ab1f24..e45383a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
@@ -33,7 +33,6 @@
import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.ByteBufferKeyValue;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
@@ -123,8 +122,7 @@
.withIncludesTags(true).build();
Configuration conf = TEST_UTIL.getConfiguration();
HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout)
- .withFileContext(context)
- .withComparator(CellComparatorImpl.COMPARATOR).create();
+ .withFileContext(context).create();
// 4 bytes * 3 * 2 for each key/value +
// 3 for keys, 15 for values = 42 (woot)
writer.append(toKV("c", tagUsage));
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestIgnoreUnknownFamily.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestIgnoreUnknownFamily.java
index a99ea7a..89ec68a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestIgnoreUnknownFamily.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestIgnoreUnknownFamily.java
@@ -23,7 +23,7 @@
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
-import org.apache.hadoop.hbase.io.hfile.HFileContext;
+import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -84,7 +84,7 @@
Path familyDir = new Path(regionDir, Bytes.toString(UNKNOWN_FAMILY));
StoreFileWriter writer =
new StoreFileWriter.Builder(mfs.getConfiguration(), mfs.getFileSystem())
- .withOutputDir(familyDir).withFileContext(new HFileContext()).build();
+ .withOutputDir(familyDir).withFileContext(new HFileContextBuilder().build()).build();
writer.close();
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java
index 52d1ce7..ed9b33e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java
@@ -52,7 +52,7 @@
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.io.hfile.HFileContext;
+import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -281,7 +281,7 @@
FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(hFileLocation), null);
try {
hFileFactory.withOutputStream(out);
- hFileFactory.withFileContext(new HFileContext());
+ hFileFactory.withFileContext(new HFileContextBuilder().build());
HFile.Writer writer = hFileFactory.create();
try {
writer.append(new KeyValue(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY)
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java
index 063c70b..33ab4e3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java
@@ -335,7 +335,7 @@
new FSDataOutputStream(new FileOutputStream(hFileLocation), null);
try {
hFileFactory.withOutputStream(out);
- hFileFactory.withFileContext(new HFileContext());
+ hFileFactory.withFileContext(new HFileContextBuilder().build());
HFile.Writer writer = hFileFactory.create();
try {
writer.append(new KeyValue(cellBuilder.build()));
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
index 1d31a4a..31aff48 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
@@ -24,9 +24,7 @@
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
-import java.util.Objects;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
@@ -41,6 +39,8 @@
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableDescriptor;
@@ -61,7 +61,8 @@
/**
* TestGet is a medley of tests of get all done up as a single test.
- * This class
+ * It was originally written to test a method since removed, getClosestAtOrBefore
+ * but the test is retained because it runs some interesting exercises.
*/
@Category({RegionServerTests.class, MediumTests.class})
public class TestGetClosestAtOrBefore {
@@ -88,7 +89,6 @@
@Test
public void testUsingMetaAndBinary() throws IOException {
- FileSystem filesystem = FileSystem.get(conf);
Path rootdir = UTIL.getDataTestDirOnTestFS();
// Up flush size else we bind up when we use default catalog flush of 16k.
TableDescriptors tds = new FSTableDescriptors(UTIL.getConfiguration());
@@ -103,13 +103,14 @@
final int last = 128;
final int interval = 2;
for (int i = 0; i <= last; i += interval) {
- HRegionInfo hri = new HRegionInfo(htd.getTableName(),
- i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i),
- i == last ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i + interval));
-
+ RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
+ .setStartKey(i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte)i))
+ .setEndKey(i == last ? HConstants.EMPTY_BYTE_ARRAY :
+ Bytes.toBytes((byte)i + interval)).build();
Put put =
MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
put.setDurability(Durability.SKIP_WAL);
+ LOG.info("Put {}", put);
mr.put(put);
}
}
@@ -117,7 +118,7 @@
try {
List<Cell> keys = new ArrayList<>();
while (s.next(keys)) {
- LOG.info(Objects.toString(keys));
+ LOG.info("Scan {}", keys);
keys.clear();
}
} finally {
@@ -133,13 +134,14 @@
findRow(mr, 'C', 46, 46);
findRow(mr, 'C', 43, 42);
// Now delete 'C' and make sure I don't get entries from 'B'.
- byte[] firstRowInC = HRegionInfo.createRegionName(TableName.valueOf("" + 'C'),
+ byte[] firstRowInC = RegionInfo.createRegionName(TableName.valueOf("" + 'C'),
HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false);
- Scan scan = new Scan(firstRowInC);
+ Scan scan = new Scan().withStartRow(firstRowInC);
s = mr.getScanner(scan);
try {
List<Cell> keys = new ArrayList<>();
while (s.next(keys)) {
+ LOG.info("Delete {}", keys);
mr.delete(new Delete(CellUtil.cloneRow(keys.get(0))));
keys.clear();
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
index 770ee8d..726d2c4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
@@ -63,7 +63,7 @@
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.io.hfile.HFileContext;
+import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.regionserver.HRegion.FlushResultImpl;
import org.apache.hadoop.hbase.regionserver.HRegion.PrepareFlushResult;
import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
@@ -1657,7 +1657,7 @@
FSDataOutputStream out = TEST_UTIL.getTestFileSystem().create(testFile);
try {
hFileFactory.withOutputStream(out);
- hFileFactory.withFileContext(new HFileContext());
+ hFileFactory.withFileContext(new HFileContextBuilder().build());
HFile.Writer writer = hFileFactory.create();
try {
writer.append(new KeyValue(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY)
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java
index 94c8af5..aeeaf3d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java
@@ -43,6 +43,7 @@
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
+import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.tool.BulkLoadHFiles;
@@ -161,7 +162,7 @@
Path path = new Path(pathStr);
HFile.WriterFactory wf = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration());
Assert.assertNotNull(wf);
- HFileContext context = new HFileContext();
+ HFileContext context = new HFileContextBuilder().build();
HFile.Writer writer = wf.withPath(fs, path).withFileContext(context).create();
KeyValue kv = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l,
Bytes.toBytes("version2"));
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java
index 610d0b1..63e8360 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java
@@ -35,7 +35,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CellComparator;
-import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -184,7 +183,7 @@
Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile");
HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build();
StoreFileWriter sfw = new StoreFileWriter.Builder(CONF, fs).withOutputDir(storeFileParentDir)
- .withComparator(CellComparatorImpl.COMPARATOR).withFileContext(meta).build();
+ .withFileContext(meta).build();
final int rowLen = 32;
Random RNG = new Random();
