HBASE-27147 [HBCK2] extraRegionsInMeta does not work If RegionInfo is… (#110)
Signed-off-by: Peter Somogyi <psomogyi@apache.org>
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java
index e8a9df7..cbd2456 100644
--- a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java
@@ -86,25 +86,25 @@
return missingChecker.reportTablesRegions(namespacesOrTables, this::findMissingRegionsInMETA);
}
- public Map<TableName,List<RegionInfo>>
+ public Map<TableName,List<HBCKMetaEntry>>
reportTablesExtraRegions(final List<String> namespacesOrTables) throws IOException {
- InternalMetaChecker<RegionInfo> extraChecker = new InternalMetaChecker<>();
+ InternalMetaChecker<HBCKMetaEntry> extraChecker = new InternalMetaChecker<>();
return extraChecker.reportTablesRegions(namespacesOrTables, this::findExtraRegionsInMETA);
}
List<Path> findMissingRegionsInMETA(String table) throws IOException {
InternalMetaChecker<Path> missingChecker = new InternalMetaChecker<>();
return missingChecker.checkRegionsInMETA(table, (regions, dirs) -> {
- ListUtils<Path, RegionInfo> utils = new ListUtils<>();
- return utils.complement(dirs, regions, d -> d.getName(), r -> r.getEncodedName());
+ ListUtils<Path, HBCKMetaEntry> utils = new ListUtils<>();
+ return utils.complement(dirs, regions, d -> d.getName(), r -> r.getEncodedRegionName());
});
}
- List<RegionInfo> findExtraRegionsInMETA(String table) throws IOException {
- InternalMetaChecker<RegionInfo> extraChecker = new InternalMetaChecker<>();
+ List<HBCKMetaEntry> findExtraRegionsInMETA(String table) throws IOException {
+ InternalMetaChecker<HBCKMetaEntry> extraChecker = new InternalMetaChecker<>();
return extraChecker.checkRegionsInMETA(table, (regions,dirs) -> {
- ListUtils<RegionInfo, Path> utils = new ListUtils<>();
- return utils.complement(regions, dirs, r -> r.getEncodedName(), d -> d.getName());
+ ListUtils<HBCKMetaEntry, Path> utils = new ListUtils<>();
+ return utils.complement(regions, dirs, r -> r.getEncodedRegionName(), d -> d.getName());
});
}
@@ -132,18 +132,18 @@
public List<Future<List<String>>> removeExtraRegionsFromMetaForTables(
List<String> nameSpaceOrTable) throws IOException {
if(nameSpaceOrTable.size()>0) {
- InternalMetaChecker<RegionInfo> extraChecker = new InternalMetaChecker<>();
+ InternalMetaChecker<HBCKMetaEntry> extraChecker = new InternalMetaChecker<>();
return extraChecker.processRegionsMetaCleanup(this::reportTablesExtraRegions,
this::deleteAllRegions, nameSpaceOrTable);
}
return null;
}
- private List<String> deleteAllRegions(List<RegionInfo> regions) throws IOException {
+ private List<String> deleteAllRegions(List<HBCKMetaEntry> regions) throws IOException {
List<String> resulting = new ArrayList<>();
- for(RegionInfo r : regions){
- HBCKMetaTableAccessor.deleteRegionInfo(conn, r);
- resulting.add(r.getEncodedName());
+ for(HBCKMetaEntry r : regions){
+ HBCKMetaTableAccessor.deleteRegion(conn, r);
+ resulting.add(r.getEncodedRegionName());
}
return resulting;
}
@@ -156,11 +156,11 @@
private class InternalMetaChecker<T> {
List<T> checkRegionsInMETA(String table,
- CheckingFunction<List<RegionInfo>, List<Path>, T> checkingFunction) throws IOException {
+ CheckingFunction<List<HBCKMetaEntry>, List<Path>, T> checkingFunction) throws IOException {
final List<Path> regionsDirs = getTableRegionsDirs(table);
TableName tableName = TableName.valueOf(table);
- List<RegionInfo> regions = HBCKMetaTableAccessor.
- getTableRegions(FsRegionsMetaRecoverer.this.conn, tableName);
+ List<HBCKMetaEntry> regions = HBCKMetaTableAccessor.
+ getTableRegionsAsMetaEntries(FsRegionsMetaRecoverer.this.conn, tableName);
return checkingFunction.check(regions, regionsDirs);
}
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
index a4e180b..ca1597d 100644
--- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
@@ -344,12 +344,12 @@
new FsRegionsMetaRecoverer(this.conf)) {
List<String> namespacesTables =
getFromArgsOrFiles(commandLine.getArgList(), inputFileFlag);
- Map<TableName, List<RegionInfo>> reportMap =
+ Map<TableName, List<HBCKMetaEntry>> reportMap =
fsRegionsMetaRecoverer.reportTablesExtraRegions(namespacesTables);
final List<String> toFix = new ArrayList<>();
reportMap.entrySet().forEach(e -> {
result.put(e.getKey(),
- e.getValue().stream().map(r->r.getEncodedName()).collect(Collectors.toList()));
+ e.getValue().stream().map(r->r.getEncodedRegionName()).collect(Collectors.toList()));
if(fix && e.getValue().size()>0){
toFix.add(e.getKey().getNameWithNamespaceInclAsString());
}
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKActions.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKActions.java
index a1126b6..01dc209 100644
--- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKActions.java
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKActions.java
@@ -87,7 +87,7 @@
System.out.println(String.format("Current Regions of the table " + tn.getNameAsString()
+ " in Meta before deletion of the region are: " + ris));
RegionInfo ri = ris.get(ris.size() / 2);
- System.out.println("Deleting Region " + ri.getRegionNameAsString());
+ System.out.println("Deleting Region " + ri);
byte[] key = HBCKMetaTableAccessor.getMetaKeyForRegion(ri);
Delete delete = new Delete(key);
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaEntry.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaEntry.java
new file mode 100644
index 0000000..2c60139
--- /dev/null
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaEntry.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hbase;
+
+/**
+ * A conveninent representation of a row in meta, composing the encoded name and the related
+ * rowkey (region name).
+ */
+public class HBCKMetaEntry {
+
+ private String encodedRegionName;
+ private byte[] regionName;
+
+ public HBCKMetaEntry(byte[] regionName, String encodedRegionName) {
+ this.regionName = regionName;
+ this.encodedRegionName = encodedRegionName;
+ }
+
+ public byte[] getRegionName() {
+ return regionName;
+ }
+
+ public String getEncodedRegionName() {
+ return encodedRegionName;
+ }
+}
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java
index 8d50bb5..ed7e507 100644
--- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java
@@ -128,13 +128,44 @@
* @throws IOException if it's not able to delete the regionInfo
*/
public static void deleteRegionInfo(Connection connection, RegionInfo regionInfo)
- throws IOException {
+ throws IOException {
Delete delete = new Delete(regionInfo.getRegionName());
delete.addFamily(HConstants.CATALOG_FAMILY, HConstants.LATEST_TIMESTAMP);
deleteFromMetaTable(connection, delete);
LOG.info("Deleted {}", regionInfo.getRegionNameAsString());
}
+ /**
+ * Delete the passed <code>RegionInfo</code> from the <code>hbase:meta</code> table.
+ *
+ * @param connection connection we're using
+ * @param regionInfo the regionInfo to delete from the meta table
+ * @throws IOException if it's not able to delete the regionInfo
+ */
+ public static void deleteRegionInfoColumn(Connection connection, RegionInfo regionInfo)
+ throws IOException {
+ Delete delete = new Delete(regionInfo.getRegionName());
+ delete.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ HConstants.LATEST_TIMESTAMP);
+ deleteFromMetaTable(connection, delete);
+ LOG.info("Deleted regioninfo for {}", regionInfo.getRegionNameAsString());
+ }
+
+ /**
+ * Delete the passed <code>HBCKMetaEntry</code> from the <code>hbase:meta</code> table.
+ *
+ * @param connection connection we're using
+ * @param region the region to be deleted from the meta table
+ * @throws IOException if it's not able to delete the regionInfo
+ */
+ public static void deleteRegion(Connection connection, HBCKMetaEntry region)
+ throws IOException {
+ Delete delete = new Delete(region.getRegionName());
+ delete.addFamily(HConstants.CATALOG_FAMILY, HConstants.LATEST_TIMESTAMP);
+ deleteFromMetaTable(connection, delete);
+ LOG.info("Deleted {}", region.getEncodedRegionName());
+ }
+
// Private helper methods
// COPIED from MetaTableAccessor.isMergeQualifierPrefix()
@@ -187,11 +218,49 @@
* Returns all regions in meta for the given table.
* @param conn a valid, open connection.
* @param table the table to list regions in meta.
- * @return a list of <code>RegionInfo</code> for all table regions present in meta.
+ * @return a list of <code>HBCKMetaEntry</code> with encoded region names, and the meta row key
+ * for all table regions present in meta.
+ * @throws IOException on any issues related with scanning meta table
+ * */
+ public static List<HBCKMetaEntry> getTableRegionsAsMetaEntries(final Connection conn,
+ final TableName table) throws IOException {
+ final MetaScanner<HBCKMetaEntry> scanner = new MetaScanner<>();
+ final String startRow = Bytes.toString(table.getName()) + ",,";
+ final String stopRow = Bytes.toString(table.getName()) + " ,,";
+ return scanner.scanMeta(conn,
+ scan -> {
+ scan.withStartRow(Bytes.toBytes(startRow));
+ scan.withStopRow(Bytes.toBytes(stopRow));
+ },
+ r -> {
+ if (r.getRow() != null) {
+ boolean encodedNameOffset = false;
+ StringBuilder encodedNameBuilder = new StringBuilder();
+ for(int i=0; i<r.getRow().length; i++){
+ if (r.getRow()[i]=='.'){
+ encodedNameOffset = !encodedNameOffset;
+ continue;
+ }
+ if(encodedNameOffset){
+ encodedNameBuilder.append((char)r.getRow()[i]);
+ }
+ }
+ return new HBCKMetaEntry(r.getRow(), encodedNameBuilder.toString());
+ }
+ return null;
+ });
+ }
+
+ /**
+ * Returns all regions in meta for the given table.
+ * @param conn a valid, open connection.
+ * @param table the table to list regions in meta.
+ * @return a list of <code>String</code> of encoded region names,
+ * for all table regions present in meta.
* @throws IOException on any issues related with scanning meta table
*/
public static List<RegionInfo> getTableRegions(final Connection conn, final TableName table)
- throws IOException {
+ throws IOException {
final MetaScanner<RegionInfo> scanner = new MetaScanner<>();
final String startRow = Bytes.toString(table.getName()) + ",,";
final String stopRow = Bytes.toString(table.getName()) + " ,,";
diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java
index 2093996..68101a6 100644
--- a/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java
+++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java
@@ -210,10 +210,10 @@
@Test
public void testFindExtraRegionsInMETAOneExtra() throws Exception {
RegionInfo info = createRegionInMeta(Mockito.mock(ResultScanner.class));
- List<RegionInfo> missingRegions = fixer.findExtraRegionsInMETA("test-tbl");
+ List<HBCKMetaEntry> missingRegions = fixer.findExtraRegionsInMETA("test-tbl");
assertEquals("Should had returned 1 extra region",
1, missingRegions.size());
- assertEquals(info.getEncodedName(),missingRegions.get(0).getEncodedName());
+ assertEquals(info.getEncodedName(),missingRegions.get(0).getEncodedRegionName());
}
@Test
@@ -243,7 +243,7 @@
cells.add(createCellForTableState(TableName.valueOf("test-tbl")));
Result result = Result.create(cells);
Mockito.when(mockedRS.next()).thenReturn(result,(Result)null);
- Map<TableName, List<RegionInfo>> report = fixer.reportTablesExtraRegions(null);
+ Map<TableName, List<HBCKMetaEntry>> report = fixer.reportTablesExtraRegions(null);
assertEquals("Should had returned 1 extra region.",
1,report.size());
}
diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
index 09f9559..e795650 100644
--- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
+++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
@@ -696,6 +696,20 @@
}
}
+ @Test
+ public void testRemoveExtraRegionWithoutInfo() throws Exception {
+ TableName tableName = createTestTable(5);
+ HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
+ List<RegionInfo> regions = HBCKMetaTableAccessor
+ .getTableRegions(TEST_UTIL.getConnection(), tableName);
+ deleteRegionDir(tableName, regions.get(0).getEncodedName());
+ HBCKMetaTableAccessor.deleteRegionInfoColumn(TEST_UTIL.getConnection(), regions.get(0));
+ assertEquals(1, hbck.extraRegionsInMeta(new String[]{"-f",
+ "default:" + tableName.getNameAsString()}).get(tableName).size());
+ assertEquals("Table regions should had been removed from META.", 4,
+ HBCKMetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tableName).size());
+ }
+
private String testFormatExtraRegionsInMetaReport() throws IOException {
return testRunWithArgs(new String[]{EXTRA_REGIONS_IN_META});
}
@@ -735,7 +749,7 @@
assertEquals(extraRegions, hbck.extraRegionsInMeta(new String[]{"-f",
"default:" + tableName.getNameAsString()}).get(tableName).size());
assertEquals("Table regions should had been removed from META.", remaining,
- HBCKMetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName));
+ HBCKMetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tableName).size());
}
private void testReportExtraRegionsInMeta(int extraRegionsInTestTbl,
