HBASE-29568 - Allow for a configurable grace period when using Time Based Priority (#7425)
Signed-off-by: Wellington Chevreuil <wchevreuil@apache.org>
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
index 2a5e2a5..6638fd2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java
@@ -47,6 +47,9 @@
"hbase.regionserver.datatiering.enable";
public static final boolean DEFAULT_GLOBAL_DATA_TIERING_ENABLED = false; // disabled by default
public static final String DATATIERING_KEY = "hbase.hstore.datatiering.type";
+ public static final String HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY =
+ "hbase.hstore.datatiering.grace.period.millis";
+ public static final long DEFAULT_DATATIERING_GRACE_PERIOD = 0;
public static final String DATATIERING_HOT_DATA_AGE_KEY =
"hbase.hstore.datatiering.hot.age.millis";
public static final DataTieringType DEFAULT_DATATIERING = DataTieringType.NONE;
@@ -139,6 +142,9 @@
* @return {@code true} if the data is hot, {@code false} otherwise
*/
public boolean isHotData(long maxTimestamp, Configuration conf) {
+ if (isWithinGracePeriod(maxTimestamp, conf)) {
+ return true;
+ }
DataTieringType dataTieringType = getDataTieringType(conf);
if (
@@ -170,8 +176,11 @@
throw new DataTieringException(
"Store file corresponding to " + hFilePath + " doesn't exist");
}
- return hotDataValidator(dataTieringType.getInstance().getTimestamp(getHStoreFile(hFilePath)),
- getDataTieringHotDataAge(configuration));
+ long maxTimestamp = dataTieringType.getInstance().getTimestamp(hStoreFile);
+ if (isWithinGracePeriod(maxTimestamp, configuration)) {
+ return true;
+ }
+ return hotDataValidator(maxTimestamp, getDataTieringHotDataAge(configuration));
}
// DataTieringType.NONE or other types are considered hot by default
return true;
@@ -189,13 +198,21 @@
public boolean isHotData(HFileInfo hFileInfo, Configuration configuration) {
DataTieringType dataTieringType = getDataTieringType(configuration);
if (hFileInfo != null && !dataTieringType.equals(DataTieringType.NONE)) {
- return hotDataValidator(dataTieringType.getInstance().getTimestamp(hFileInfo),
- getDataTieringHotDataAge(configuration));
+ long maxTimestamp = dataTieringType.getInstance().getTimestamp(hFileInfo);
+ if (isWithinGracePeriod(maxTimestamp, configuration)) {
+ return true;
+ }
+ return hotDataValidator(maxTimestamp, getDataTieringHotDataAge(configuration));
}
// DataTieringType.NONE or other types are considered hot by default
return true;
}
+ private boolean isWithinGracePeriod(long maxTimestamp, Configuration conf) {
+ long gracePeriod = getDataTieringGracePeriod(conf);
+ return gracePeriod > 0 && (getCurrentTimestamp() - maxTimestamp) < gracePeriod;
+ }
+
private boolean hotDataValidator(long maxTimestamp, long hotDataAge) {
long currentTimestamp = getCurrentTimestamp();
long diff = currentTimestamp - maxTimestamp;
@@ -275,6 +292,11 @@
conf.get(DATATIERING_HOT_DATA_AGE_KEY, String.valueOf(DEFAULT_DATATIERING_HOT_DATA_AGE)));
}
+ private long getDataTieringGracePeriod(Configuration conf) {
+ return Long.parseLong(conf.get(HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY,
+ String.valueOf(DEFAULT_DATATIERING_GRACE_PERIOD)));
+ }
+
/*
* This API traverses through the list of online regions and returns a subset of these files-names
* that are cold.
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
index bf82a53..cdf00e5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java
@@ -232,6 +232,57 @@
}
@Test
+ public void testGracePeriodMakesColdFileHot() throws IOException, DataTieringException {
+ initializeTestEnvironment();
+
+ long hotAge = 1 * DAY;
+ long gracePeriod = 3 * DAY;
+
+ long currentTime = System.currentTimeMillis();
+ long fileTimestamp = currentTime - (2 * DAY);
+
+ Configuration conf = getConfWithGracePeriod(hotAge, gracePeriod);
+ HRegion region = createHRegion("tableGracePeriod", conf);
+ HStore hStore = createHStore(region, "cf1", conf);
+
+ HStoreFile file = createHStoreFile(hStore.getStoreContext().getFamilyStoreDirectoryPath(),
+ hStore.getReadOnlyConfiguration(), fileTimestamp, region.getRegionFileSystem());
+ file.initReader();
+
+ hStore.refreshStoreFiles();
+ region.stores.put(Bytes.toBytes("cf1"), hStore);
+ testOnlineRegions.put(region.getRegionInfo().getEncodedName(), region);
+ Path hFilePath = file.getPath();
+ assertTrue("File should be hot due to grace period", dataTieringManager.isHotData(hFilePath));
+ }
+
+ @Test
+ public void testFileIsColdWithoutGracePeriod() throws IOException, DataTieringException {
+ initializeTestEnvironment();
+
+ long hotAge = 1 * DAY;
+ long gracePeriod = 0;
+ long currentTime = System.currentTimeMillis();
+ long fileTimestamp = currentTime - (2 * DAY);
+
+ Configuration conf = getConfWithGracePeriod(hotAge, gracePeriod);
+ HRegion region = createHRegion("tableNoGracePeriod", conf);
+ HStore hStore = createHStore(region, "cf1", conf);
+
+ HStoreFile file = createHStoreFile(hStore.getStoreContext().getFamilyStoreDirectoryPath(),
+ hStore.getReadOnlyConfiguration(), fileTimestamp, region.getRegionFileSystem());
+ file.initReader();
+
+ hStore.refreshStoreFiles();
+ region.stores.put(Bytes.toBytes("cf1"), hStore);
+ testOnlineRegions.put(region.getRegionInfo().getEncodedName(), region);
+
+ Path hFilePath = file.getPath();
+ assertFalse("File should be cold without grace period",
+ dataTieringManager.isHotData(hFilePath));
+ }
+
+ @Test
public void testPrefetchWhenDataTieringEnabled() throws IOException {
setPrefetchBlocksOnOpen();
initializeTestEnvironment();
@@ -770,6 +821,8 @@
.setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY))
.setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY,
conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY))
+ .setValue(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY,
+ conf.get(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY))
.build();
RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build();
@@ -797,6 +850,8 @@
.setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY))
.setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY,
conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY))
+ .setValue(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY,
+ conf.get(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY))
.build();
return new HStore(region, columnFamilyDescriptor, conf, false);
@@ -809,6 +864,13 @@
return conf;
}
+ private static Configuration getConfWithGracePeriod(long hotDataAge, long gracePeriod) {
+ Configuration conf = getConfWithTimeRangeDataTieringEnabled(hotDataAge);
+ conf.set(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY,
+ String.valueOf(gracePeriod));
+ return conf;
+ }
+
static HStoreFile createHStoreFile(Path storeDir, Configuration conf, long timestamp,
HRegionFileSystem regionFs) throws IOException {
String columnFamily = storeDir.getName();
