| /** |
| * Copyright 2010 The Apache Software Foundation |
| * |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.hbase.util; |
| |
| import java.io.IOException; |
| import java.util.Collection; |
| import java.util.Collections; |
| import java.util.Comparator; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Set; |
| import java.util.TreeMap; |
| import java.util.TreeSet; |
| import java.util.concurrent.atomic.AtomicInteger; |
| |
| import org.apache.commons.logging.Log; |
| import org.apache.commons.logging.LogFactory; |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.fs.FileStatus; |
| import org.apache.hadoop.fs.FileSystem; |
| import org.apache.hadoop.fs.Path; |
| import org.apache.hadoop.hbase.ClusterStatus; |
| import org.apache.hadoop.hbase.HBaseConfiguration; |
| import org.apache.hadoop.hbase.HConstants; |
| import org.apache.hadoop.hbase.HRegionInfo; |
| import org.apache.hadoop.hbase.HRegionLocation; |
| import org.apache.hadoop.hbase.HServerAddress; |
| import org.apache.hadoop.hbase.HServerInfo; |
| import org.apache.hadoop.hbase.HTableDescriptor; |
| import org.apache.hadoop.hbase.KeyValue; |
| import org.apache.hadoop.hbase.MasterNotRunningException; |
| import org.apache.hadoop.hbase.ZooKeeperConnectionException; |
| import org.apache.hadoop.hbase.client.HBaseAdmin; |
| import org.apache.hadoop.hbase.client.HConnection; |
| import org.apache.hadoop.hbase.client.HConnectionManager; |
| import org.apache.hadoop.hbase.client.MetaScanner; |
| import org.apache.hadoop.hbase.client.Result; |
| import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; |
| import org.apache.hadoop.hbase.ipc.HRegionInterface; |
| import org.apache.hadoop.hbase.regionserver.wal.HLog; |
| import org.apache.hadoop.hbase.zookeeper.ZKTable; |
| import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; |
| import org.apache.zookeeper.KeeperException; |
| |
| import com.google.common.base.Joiner; |
| import com.google.common.collect.Lists; |
| |
| /** |
| * Check consistency among the in-memory states of the master and the |
| * region server(s) and the state of data in HDFS. |
| */ |
| public class HBaseFsck { |
| public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute |
| |
| private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName()); |
| private Configuration conf; |
| |
| private ClusterStatus status; |
| private HConnection connection; |
| |
| private TreeMap<String, HbckInfo> regionInfo = new TreeMap<String, HbckInfo>(); |
| private TreeMap<String, TInfo> tablesInfo = new TreeMap<String, TInfo>(); |
| private TreeSet<byte[]> disabledTables = |
| new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); |
| ErrorReporter errors = new PrintingErrorReporter(); |
| |
| private static boolean details = false; // do we display the full report |
| private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older |
| private boolean fix = false; // do we want to try fixing the errors? |
| private boolean rerun = false; // if we tried to fix something rerun hbck |
| private static boolean summary = false; // if we want to print less output |
| // Empty regioninfo qualifiers in .META. |
| private Set<Result> emptyRegionInfoQualifiers = new HashSet<Result>(); |
| |
| /** |
| * Constructor |
| * |
| * @param conf Configuration object |
| * @throws MasterNotRunningException if the master is not running |
| * @throws ZooKeeperConnectionException if unable to connect to zookeeper |
| */ |
| public HBaseFsck(Configuration conf) |
| throws MasterNotRunningException, ZooKeeperConnectionException, IOException { |
| this.conf = conf; |
| |
| HBaseAdmin admin = new HBaseAdmin(conf); |
| status = admin.getMaster().getClusterStatus(); |
| connection = admin.getConnection(); |
| } |
| |
| /** |
| * Contacts the master and prints out cluster-wide information |
| * @throws IOException if a remote or network exception occurs |
| * @return 0 on success, non-zero on failure |
| * @throws KeeperException |
| * @throws InterruptedException |
| */ |
| int doWork() throws IOException, KeeperException, InterruptedException { |
| // print hbase server version |
| errors.print("Version: " + status.getHBaseVersion()); |
| |
| // Make sure regionInfo is empty before starting |
| regionInfo.clear(); |
| tablesInfo.clear(); |
| emptyRegionInfoQualifiers.clear(); |
| disabledTables.clear(); |
| |
| // get a list of all regions from the master. This involves |
| // scanning the META table |
| if (!recordRootRegion()) { |
| // Will remove later if we can fix it |
| errors.reportError("Encountered fatal error. Exitting..."); |
| return -1; |
| } |
| getMetaEntries(); |
| |
| // Check if .META. is found only once and on the right place |
| if (!checkMetaEntries()) { |
| // Will remove later if we can fix it |
| errors.reportError("Encountered fatal error. Exitting..."); |
| return -1; |
| } |
| |
| // get a list of all tables that have not changed recently. |
| AtomicInteger numSkipped = new AtomicInteger(0); |
| HTableDescriptor[] allTables = getTables(numSkipped); |
| errors.print("Number of Tables: " + allTables.length); |
| if (details) { |
| if (numSkipped.get() > 0) { |
| errors.detail("Number of Tables in flux: " + numSkipped.get()); |
| } |
| for (HTableDescriptor td : allTables) { |
| String tableName = td.getNameAsString(); |
| errors.detail(" Table: " + tableName + "\t" + |
| (td.isReadOnly() ? "ro" : "rw") + "\t" + |
| (td.isRootRegion() ? "ROOT" : |
| (td.isMetaRegion() ? "META" : " ")) + "\t" + |
| " families: " + td.getFamilies().size()); |
| } |
| } |
| |
| // From the master, get a list of all known live region servers |
| Collection<HServerInfo> regionServers = status.getServerInfo(); |
| errors.print("Number of live region servers: " + |
| regionServers.size()); |
| if (details) { |
| for (HServerInfo rsinfo: regionServers) { |
| errors.print(" " + rsinfo.getServerName()); |
| } |
| } |
| |
| // From the master, get a list of all dead region servers |
| Collection<String> deadRegionServers = status.getDeadServerNames(); |
| errors.print("Number of dead region servers: " + |
| deadRegionServers.size()); |
| if (details) { |
| for (String name: deadRegionServers) { |
| errors.print(" " + name); |
| } |
| } |
| |
| // Determine what's deployed |
| processRegionServers(regionServers); |
| |
| // Determine what's on HDFS |
| checkHdfs(); |
| |
| // Empty cells in .META.? |
| errors.print("Number of empty REGIONINFO_QUALIFIER rows in .META.: " + |
| emptyRegionInfoQualifiers.size()); |
| if (details) { |
| for (Result r: emptyRegionInfoQualifiers) { |
| errors.print(" " + r); |
| } |
| } |
| |
| // Get disabled tables from ZooKeeper |
| loadDisabledTables(); |
| |
| // Check consistency |
| checkConsistency(); |
| |
| // Check integrity |
| checkIntegrity(); |
| |
| // Print table summary |
| printTableSummary(); |
| |
| return errors.summarize(); |
| } |
| |
| /** |
| * Load the list of disabled tables in ZK into local set. |
| * @throws ZooKeeperConnectionException |
| * @throws IOException |
| * @throws KeeperException |
| */ |
| private void loadDisabledTables() |
| throws ZooKeeperConnectionException, IOException, KeeperException { |
| ZooKeeperWatcher zkw = |
| HConnectionManager.getConnection(conf).getZooKeeperWatcher(); |
| for (String tableName : ZKTable.getDisabledOrDisablingTables(zkw)) { |
| disabledTables.add(Bytes.toBytes(tableName)); |
| } |
| } |
| |
| /** |
| * Check if the specified region's table is disabled. |
| * @throws ZooKeeperConnectionException |
| * @throws IOException |
| * @throws KeeperException |
| */ |
| private boolean isTableDisabled(HRegionInfo regionInfo) { |
| return disabledTables.contains(regionInfo.getTableDesc().getName()); |
| } |
| |
| /** |
| * Scan HDFS for all regions, recording their information into |
| * regionInfo |
| */ |
| void checkHdfs() throws IOException { |
| Path rootDir = new Path(conf.get(HConstants.HBASE_DIR)); |
| FileSystem fs = rootDir.getFileSystem(conf); |
| |
| // list all tables from HDFS |
| List<FileStatus> tableDirs = Lists.newArrayList(); |
| |
| boolean foundVersionFile = false; |
| FileStatus[] files = fs.listStatus(rootDir); |
| for (FileStatus file : files) { |
| if (file.getPath().getName().equals(HConstants.VERSION_FILE_NAME)) { |
| foundVersionFile = true; |
| } else { |
| tableDirs.add(file); |
| } |
| } |
| |
| // verify that version file exists |
| if (!foundVersionFile) { |
| errors.reportError("Version file does not exist in root dir " + rootDir); |
| } |
| |
| // level 1: <HBASE_DIR>/* |
| for (FileStatus tableDir : tableDirs) { |
| String tableName = tableDir.getPath().getName(); |
| // ignore hidden files |
| if (tableName.startsWith(".") && |
| !tableName.equals( Bytes.toString(HConstants.META_TABLE_NAME))) |
| continue; |
| // level 2: <HBASE_DIR>/<table>/* |
| FileStatus[] regionDirs = fs.listStatus(tableDir.getPath()); |
| for (FileStatus regionDir : regionDirs) { |
| String encodedName = regionDir.getPath().getName(); |
| // ignore directories that aren't hexadecimal |
| if (!encodedName.toLowerCase().matches("[0-9a-f]+")) continue; |
| |
| HbckInfo hbi = getOrCreateInfo(encodedName); |
| hbi.foundRegionDir = regionDir; |
| |
| // Set a flag if this region contains only edits |
| // This is special case if a region is left after split |
| hbi.onlyEdits = true; |
| FileStatus[] subDirs = fs.listStatus(regionDir.getPath()); |
| if (subDirs != null) { |
| Path ePath = HLog.getRegionDirRecoveredEditsDir(regionDir.getPath()); |
| for (FileStatus subDir : subDirs) { |
| String sdName = subDir.getPath().getName(); |
| if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) { |
| hbi.onlyEdits = false; |
| break; |
| } |
| } |
| } |
| } |
| } |
| } |
| |
| /** |
| * Record the location of the ROOT region as found in ZooKeeper, |
| * as if it were in a META table. This is so that we can check |
| * deployment of ROOT. |
| */ |
| boolean recordRootRegion() throws IOException { |
| HRegionLocation rootLocation = connection.locateRegion( |
| HConstants.ROOT_TABLE_NAME, HConstants.EMPTY_START_ROW); |
| |
| // Check if Root region is valid and existing |
| if (rootLocation == null || rootLocation.getRegionInfo() == null || |
| rootLocation.getServerAddress() == null) { |
| errors.reportError("Root Region or some of its attributes is null."); |
| return false; |
| } |
| |
| MetaEntry m = new MetaEntry(rootLocation.getRegionInfo(), |
| rootLocation.getServerAddress(), null, System.currentTimeMillis()); |
| HbckInfo hbInfo = new HbckInfo(m); |
| regionInfo.put(rootLocation.getRegionInfo().getEncodedName(), hbInfo); |
| return true; |
| } |
| |
| /** |
| * Contacts each regionserver and fetches metadata about regions. |
| * @param regionServerList - the list of region servers to connect to |
| * @throws IOException if a remote or network exception occurs |
| */ |
| void processRegionServers(Collection<HServerInfo> regionServerList) |
| throws IOException { |
| |
| // loop to contact each region server |
| for (HServerInfo rsinfo: regionServerList) { |
| errors.progress(); |
| try { |
| HRegionInterface server = connection.getHRegionConnection( |
| rsinfo.getServerAddress()); |
| |
| // list all online regions from this region server |
| List<HRegionInfo> regions = server.getOnlineRegions(); |
| if (details) { |
| errors.detail("RegionServer: " + rsinfo.getServerName() + |
| " number of regions: " + regions.size()); |
| for (HRegionInfo rinfo: regions) { |
| errors.detail(" " + rinfo.getRegionNameAsString() + |
| " id: " + rinfo.getRegionId() + |
| " encoded_name: " + rinfo.getEncodedName() + |
| " start: " + Bytes.toStringBinary(rinfo.getStartKey()) + |
| " end: " + Bytes.toStringBinary(rinfo.getEndKey())); |
| } |
| } |
| |
| // check to see if the existance of this region matches the region in META |
| for (HRegionInfo r:regions) { |
| HbckInfo hbi = getOrCreateInfo(r.getEncodedName()); |
| hbi.deployedOn.add(rsinfo.getServerAddress()); |
| } |
| } catch (IOException e) { // unable to connect to the region server. |
| errors.reportError("\nRegionServer:" + rsinfo.getServerName() + |
| " Unable to fetch region information. " + e); |
| } |
| } |
| } |
| |
| /** |
| * Check consistency of all regions that have been found in previous phases. |
| * @throws KeeperException |
| * @throws InterruptedException |
| */ |
| void checkConsistency() |
| throws IOException, KeeperException, InterruptedException { |
| for (java.util.Map.Entry<String, HbckInfo> e: regionInfo.entrySet()) { |
| doConsistencyCheck(e.getKey(), e.getValue()); |
| } |
| } |
| |
| /** |
| * Check a single region for consistency and correct deployment. |
| * @throws KeeperException |
| * @throws InterruptedException |
| */ |
| void doConsistencyCheck(final String key, final HbckInfo hbi) |
| throws IOException, KeeperException, InterruptedException { |
| String descriptiveName = hbi.toString(); |
| |
| boolean inMeta = hbi.metaEntry != null; |
| boolean inHdfs = hbi.foundRegionDir != null; |
| boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null; |
| boolean isDeployed = !hbi.deployedOn.isEmpty(); |
| boolean isMultiplyDeployed = hbi.deployedOn.size() > 1; |
| boolean deploymentMatchesMeta = |
| hasMetaAssignment && isDeployed && !isMultiplyDeployed && |
| hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0)); |
| boolean splitParent = |
| (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline(); |
| boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry); |
| boolean recentlyModified = hbi.foundRegionDir != null && |
| hbi.foundRegionDir.getModificationTime() + timelag > System.currentTimeMillis(); |
| |
| // ========== First the healthy cases ============= |
| if (hbi.onlyEdits) { |
| return; |
| } |
| if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) { |
| return; |
| } else if (inMeta && !isDeployed && splitParent) { |
| // Offline regions shouldn't cause complaints |
| LOG.debug("Region " + descriptiveName + " offline, split, parent, ignoring."); |
| return; |
| } else if (inMeta && !shouldBeDeployed && !isDeployed) { |
| // offline regions shouldn't cause complaints |
| LOG.debug("Region " + descriptiveName + " offline, ignoring."); |
| return; |
| } else if (recentlyModified) { |
| LOG.info("Region " + descriptiveName + " was recently modified -- skipping"); |
| return; |
| } |
| // ========== Cases where the region is not in META ============= |
| else if (!inMeta && !inHdfs && !isDeployed) { |
| // We shouldn't have record of this region at all then! |
| assert false : "Entry for region with no data"; |
| } else if (!inMeta && !inHdfs && isDeployed) { |
| errors.reportError("Region " + descriptiveName + ", key=" + key + ", not on HDFS or in META but " + |
| "deployed on " + Joiner.on(", ").join(hbi.deployedOn)); |
| } else if (!inMeta && inHdfs && !isDeployed) { |
| errors.reportError("Region " + descriptiveName + " on HDFS, but not listed in META " + |
| "or deployed on any region server."); |
| } else if (!inMeta && inHdfs && isDeployed) { |
| errors.reportError("Region " + descriptiveName + " not in META, but deployed on " + |
| Joiner.on(", ").join(hbi.deployedOn)); |
| |
| // ========== Cases where the region is in META ============= |
| } else if (inMeta && !inHdfs && !isDeployed) { |
| errors.reportError("Region " + descriptiveName + " found in META, but not in HDFS " + |
| "or deployed on any region server."); |
| } else if (inMeta && !inHdfs && isDeployed) { |
| errors.reportError("Region " + descriptiveName + " found in META, but not in HDFS, " + |
| "and deployed on " + Joiner.on(", ").join(hbi.deployedOn)); |
| } else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) { |
| errors.reportError("Region " + descriptiveName + " not deployed on any region server."); |
| // If we are trying to fix the errors |
| if (shouldFix()) { |
| errors.print("Trying to fix unassigned region..."); |
| setShouldRerun(); |
| HBaseFsckRepair.fixUnassigned(this.conf, hbi.metaEntry); |
| } |
| } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) { |
| errors.reportError("Region " + descriptiveName + " should not be deployed according " + |
| "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn)); |
| } else if (inMeta && inHdfs && isMultiplyDeployed) { |
| errors.reportError("Region " + descriptiveName + " is listed in META on region server " + |
| hbi.metaEntry.regionServer + " but is multiply assigned to region servers " + |
| Joiner.on(", ").join(hbi.deployedOn)); |
| // If we are trying to fix the errors |
| if (shouldFix()) { |
| errors.print("Trying to fix assignment error..."); |
| setShouldRerun(); |
| HBaseFsckRepair.fixDupeAssignment(this.conf, hbi.metaEntry, hbi.deployedOn); |
| } |
| } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) { |
| errors.reportError("Region " + descriptiveName + " listed in META on region server " + |
| hbi.metaEntry.regionServer + " but found on region server " + |
| hbi.deployedOn.get(0)); |
| // If we are trying to fix the errors |
| if (shouldFix()) { |
| errors.print("Trying to fix assignment error..."); |
| setShouldRerun(); |
| HBaseFsckRepair.fixDupeAssignment(this.conf, hbi.metaEntry, hbi.deployedOn); |
| } |
| } else { |
| errors.reportError("Region " + descriptiveName + " is in an unforeseen state:" + |
| " inMeta=" + inMeta + |
| " inHdfs=" + inHdfs + |
| " isDeployed=" + isDeployed + |
| " isMultiplyDeployed=" + isMultiplyDeployed + |
| " deploymentMatchesMeta=" + deploymentMatchesMeta + |
| " shouldBeDeployed=" + shouldBeDeployed); |
| } |
| } |
| |
| /** |
| * Checks tables integrity. Goes over all regions and scans the tables. |
| * Collects all the pieces for each table and checks if there are missing, |
| * repeated or overlapping ones. |
| */ |
| void checkIntegrity() { |
| for (HbckInfo hbi : regionInfo.values()) { |
| // Check only valid, working regions |
| if (hbi.metaEntry == null) continue; |
| if (hbi.metaEntry.regionServer == null) continue; |
| if (hbi.foundRegionDir == null) continue; |
| if (hbi.deployedOn.size() != 1) continue; |
| if (hbi.onlyEdits) continue; |
| |
| // We should be safe here |
| String tableName = hbi.metaEntry.getTableDesc().getNameAsString(); |
| TInfo modTInfo = tablesInfo.get(tableName); |
| if (modTInfo == null) { |
| modTInfo = new TInfo(tableName); |
| } |
| for (HServerAddress server : hbi.deployedOn) { |
| modTInfo.addServer(server); |
| } |
| modTInfo.addEdge(hbi.metaEntry.getStartKey(), hbi.metaEntry.getEndKey()); |
| tablesInfo.put(tableName, modTInfo); |
| } |
| |
| for (TInfo tInfo : tablesInfo.values()) { |
| if (!tInfo.check()) { |
| errors.reportError("Found inconsistency in table " + tInfo.getName()); |
| } |
| } |
| } |
| |
| /** |
| * Maintain information about a particular table. |
| */ |
| private class TInfo { |
| String tableName; |
| TreeMap <byte[], byte[]> edges; |
| TreeSet <HServerAddress> deployedOn; |
| |
| TInfo(String name) { |
| this.tableName = name; |
| edges = new TreeMap <byte[], byte[]> (Bytes.BYTES_COMPARATOR); |
| deployedOn = new TreeSet <HServerAddress>(); |
| } |
| |
| public void addEdge(byte[] fromNode, byte[] toNode) { |
| this.edges.put(fromNode, toNode); |
| } |
| |
| public void addServer(HServerAddress server) { |
| this.deployedOn.add(server); |
| } |
| |
| public String getName() { |
| return tableName; |
| } |
| |
| public int getNumRegions() { |
| return edges.size(); |
| } |
| |
| public boolean check() { |
| byte[] last = new byte[0]; |
| byte[] next = new byte[0]; |
| TreeSet <byte[]> visited = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); |
| // Each table should start with a zero-length byte[] and end at a |
| // zero-length byte[]. Just follow the edges to see if this is true |
| while (true) { |
| // Check if chain is broken |
| if (!edges.containsKey(last)) { |
| errors.detail("Chain of regions in table " + tableName + |
| " is broken; edges does not contain " + Bytes.toString(last)); |
| return false; |
| } |
| next = edges.get(last); |
| // Found a cycle |
| if (visited.contains(next)) { |
| errors.detail("Chain of regions in table " + tableName + |
| " has a cycle around " + Bytes.toString(next)); |
| return false; |
| } |
| // Mark next node as visited |
| visited.add(next); |
| // If next is zero-length byte[] we are possibly at the end of the chain |
| if (next.length == 0) { |
| // If we have visited all elements we are fine |
| if (edges.size() != visited.size()) { |
| errors.detail("Chain of regions in table " + tableName + |
| " contains less elements than are listed in META; visited=" + visited.size() + |
| ", edges=" + edges.size()); |
| return false; |
| } |
| return true; |
| } |
| last = next; |
| } |
| // How did we get here? |
| } |
| } |
| |
| /** |
| * Return a list of user-space table names whose metadata have not been |
| * modified in the last few milliseconds specified by timelag |
| * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER, |
| * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last |
| * milliseconds specified by timelag, then the table is a candidate to be returned. |
| * @param regionList - all entries found in .META |
| * @return tables that have not been modified recently |
| * @throws IOException if an error is encountered |
| */ |
| HTableDescriptor[] getTables(AtomicInteger numSkipped) { |
| TreeSet<HTableDescriptor> uniqueTables = new TreeSet<HTableDescriptor>(); |
| long now = System.currentTimeMillis(); |
| |
| for (HbckInfo hbi : regionInfo.values()) { |
| MetaEntry info = hbi.metaEntry; |
| |
| // if the start key is zero, then we have found the first region of a table. |
| // pick only those tables that were not modified in the last few milliseconds. |
| if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) { |
| if (info.modTime + timelag < now) { |
| uniqueTables.add(info.getTableDesc()); |
| } else { |
| numSkipped.incrementAndGet(); // one more in-flux table |
| } |
| } |
| } |
| return uniqueTables.toArray(new HTableDescriptor[uniqueTables.size()]); |
| } |
| |
| /** |
| * Gets the entry in regionInfo corresponding to the the given encoded |
| * region name. If the region has not been seen yet, a new entry is added |
| * and returned. |
| */ |
| private HbckInfo getOrCreateInfo(String name) { |
| HbckInfo hbi = regionInfo.get(name); |
| if (hbi == null) { |
| hbi = new HbckInfo(null); |
| regionInfo.put(name, hbi); |
| } |
| return hbi; |
| } |
| |
| /** |
| * Check values in regionInfo for .META. |
| * Check if zero or more than one regions with META are found. |
| * If there are inconsistencies (i.e. zero or more than one regions |
| * pretend to be holding the .META.) try to fix that and report an error. |
| * @throws IOException from HBaseFsckRepair functions |
| * @throws KeeperException |
| * @throws InterruptedException |
| */ |
| boolean checkMetaEntries() |
| throws IOException, KeeperException, InterruptedException { |
| List <HbckInfo> metaRegions = Lists.newArrayList(); |
| for (HbckInfo value : regionInfo.values()) { |
| if (value.metaEntry.isMetaTable()) { |
| metaRegions.add(value); |
| } |
| } |
| |
| // If something is wrong |
| if (metaRegions.size() != 1) { |
| HRegionLocation rootLocation = connection.locateRegion( |
| HConstants.ROOT_TABLE_NAME, HConstants.EMPTY_START_ROW); |
| HbckInfo root = |
| regionInfo.get(rootLocation.getRegionInfo().getEncodedName()); |
| |
| // If there is no region holding .META. |
| if (metaRegions.size() == 0) { |
| errors.reportError(".META. is not found on any region."); |
| if (shouldFix()) { |
| errors.print("Trying to fix a problem with .META..."); |
| setShouldRerun(); |
| // try to fix it (treat it as unassigned region) |
| HBaseFsckRepair.fixUnassigned(conf, root.metaEntry); |
| } |
| } |
| // If there are more than one regions pretending to hold the .META. |
| else if (metaRegions.size() > 1) { |
| errors.reportError(".META. is found on more than one region."); |
| if (shouldFix()) { |
| errors.print("Trying to fix a problem with .META..."); |
| setShouldRerun(); |
| // try fix it (treat is a dupe assignment) |
| List <HServerAddress> deployedOn = Lists.newArrayList(); |
| for (HbckInfo mRegion : metaRegions) { |
| deployedOn.add(mRegion.metaEntry.regionServer); |
| } |
| HBaseFsckRepair.fixDupeAssignment(conf, root.metaEntry, deployedOn); |
| } |
| } |
| // rerun hbck with hopefully fixed META |
| return false; |
| } |
| // no errors, so continue normally |
| return true; |
| } |
| |
| /** |
| * Scan .META. and -ROOT-, adding all regions found to the regionInfo map. |
| * @throws IOException if an error is encountered |
| */ |
| void getMetaEntries() throws IOException { |
| MetaScannerVisitor visitor = new MetaScannerVisitor() { |
| int countRecord = 1; |
| |
| // comparator to sort KeyValues with latest modtime |
| final Comparator<KeyValue> comp = new Comparator<KeyValue>() { |
| public int compare(KeyValue k1, KeyValue k2) { |
| return (int)(k1.getTimestamp() - k2.getTimestamp()); |
| } |
| }; |
| |
| public boolean processRow(Result result) throws IOException { |
| try { |
| |
| // record the latest modification of this META record |
| long ts = Collections.max(result.list(), comp).getTimestamp(); |
| |
| // record region details |
| byte [] value = result.getValue(HConstants.CATALOG_FAMILY, |
| HConstants.REGIONINFO_QUALIFIER); |
| if (value == null || value.length == 0) { |
| emptyRegionInfoQualifiers.add(result); |
| return true; |
| } |
| HRegionInfo info = Writables.getHRegionInfo(value); |
| HServerAddress server = null; |
| byte[] startCode = null; |
| |
| // record assigned region server |
| value = result.getValue(HConstants.CATALOG_FAMILY, |
| HConstants.SERVER_QUALIFIER); |
| if (value != null && value.length > 0) { |
| String address = Bytes.toString(value); |
| server = new HServerAddress(address); |
| } |
| |
| // record region's start key |
| value = result.getValue(HConstants.CATALOG_FAMILY, |
| HConstants.STARTCODE_QUALIFIER); |
| if (value != null) { |
| startCode = value; |
| } |
| MetaEntry m = new MetaEntry(info, server, startCode, ts); |
| HbckInfo hbInfo = new HbckInfo(m); |
| HbckInfo previous = regionInfo.put(info.getEncodedName(), hbInfo); |
| if (previous != null) { |
| throw new IOException("Two entries in META are same " + previous); |
| } |
| |
| // show proof of progress to the user, once for every 100 records. |
| if (countRecord % 100 == 0) { |
| errors.progress(); |
| } |
| countRecord++; |
| return true; |
| } catch (RuntimeException e) { |
| LOG.error("Result=" + result); |
| throw e; |
| } |
| } |
| }; |
| |
| // Scan -ROOT- to pick up META regions |
| MetaScanner.metaScan(conf, visitor, null, null, |
| Integer.MAX_VALUE, HConstants.ROOT_TABLE_NAME); |
| |
| // Scan .META. to pick up user regions |
| MetaScanner.metaScan(conf, visitor); |
| errors.print(""); |
| } |
| |
| /** |
| * Stores the entries scanned from META |
| */ |
| private static class MetaEntry extends HRegionInfo { |
| HServerAddress regionServer; // server hosting this region |
| long modTime; // timestamp of most recent modification metadata |
| |
| public MetaEntry(HRegionInfo rinfo, HServerAddress regionServer, |
| byte[] startCode, long modTime) { |
| super(rinfo); |
| this.regionServer = regionServer; |
| this.modTime = modTime; |
| } |
| } |
| |
| /** |
| * Maintain information about a particular region. |
| */ |
| static class HbckInfo { |
| boolean onlyEdits = false; |
| MetaEntry metaEntry = null; |
| FileStatus foundRegionDir = null; |
| List<HServerAddress> deployedOn = Lists.newArrayList(); |
| |
| HbckInfo(MetaEntry metaEntry) { |
| this.metaEntry = metaEntry; |
| } |
| |
| public String toString() { |
| if (metaEntry != null) { |
| return metaEntry.getRegionNameAsString(); |
| } else if (foundRegionDir != null) { |
| return foundRegionDir.getPath().toString(); |
| } else { |
| return "UNKNOWN_REGION on " + Joiner.on(", ").join(deployedOn); |
| } |
| } |
| } |
| |
| /** |
| * Prints summary of all tables found on the system. |
| */ |
| private void printTableSummary() { |
| System.out.println("Summary:"); |
| for (TInfo tInfo : tablesInfo.values()) { |
| if (tInfo.check()) { |
| System.out.println(" " + tInfo.getName() + " is okay."); |
| } else { |
| System.out.println("Table " + tInfo.getName() + " is inconsistent."); |
| } |
| System.out.println(" Number of regions: " + tInfo.getNumRegions()); |
| System.out.print(" Deployed on: "); |
| for (HServerAddress server : tInfo.deployedOn) { |
| System.out.print(" " + server.toString()); |
| } |
| System.out.println(); |
| } |
| } |
| |
| interface ErrorReporter { |
| public void reportError(String message); |
| public int summarize(); |
| public void detail(String details); |
| public void progress(); |
| public void print(String message); |
| } |
| |
| private static class PrintingErrorReporter implements ErrorReporter { |
| public int errorCount = 0; |
| private int showProgress; |
| |
| public void reportError(String message) { |
| if (!summary) { |
| System.out.println("ERROR: " + message); |
| } |
| errorCount++; |
| showProgress = 0; |
| } |
| |
| public int summarize() { |
| System.out.println(Integer.toString(errorCount) + |
| " inconsistencies detected."); |
| if (errorCount == 0) { |
| System.out.println("Status: OK"); |
| return 0; |
| } else { |
| System.out.println("Status: INCONSISTENT"); |
| return -1; |
| } |
| } |
| |
| public void print(String message) { |
| if (!summary) { |
| System.out.println(message); |
| } |
| } |
| |
| public void detail(String message) { |
| if (details) { |
| System.out.println(message); |
| } |
| showProgress = 0; |
| } |
| |
| public void progress() { |
| if (showProgress++ == 10) { |
| if (!summary) { |
| System.out.print("."); |
| } |
| showProgress = 0; |
| } |
| } |
| } |
| |
| /** |
| * Display the full report from fsck. |
| * This displays all live and dead region servers, and all known regions. |
| */ |
| void displayFullReport() { |
| details = true; |
| } |
| |
| /** |
| * Set summary mode. |
| * Print only summary of the tables and status (OK or INCONSISTENT) |
| */ |
| void setSummary() { |
| summary = true; |
| } |
| |
| /** |
| * Check if we should rerun fsck again. This checks if we've tried to |
| * fix something and we should rerun fsck tool again. |
| * Display the full report from fsck. This displays all live and dead |
| * region servers, and all known regions. |
| */ |
| void setShouldRerun() { |
| rerun = true; |
| } |
| |
| boolean shouldRerun() { |
| return rerun; |
| } |
| |
| /** |
| * Fix inconsistencies found by fsck. This should try to fix errors (if any) |
| * found by fsck utility. |
| */ |
| void setFixErrors() { |
| fix = true; |
| } |
| |
| boolean shouldFix() { |
| return fix; |
| } |
| |
| /** |
| * We are interested in only those tables that have not changed their state in |
| * META during the last few seconds specified by hbase.admin.fsck.timelag |
| * @param seconds - the time in seconds |
| */ |
| void setTimeLag(long seconds) { |
| timelag = seconds * 1000; // convert to milliseconds |
| } |
| |
| protected static void printUsageAndExit() { |
| System.err.println("Usage: fsck [opts] "); |
| System.err.println(" where [opts] are:"); |
| System.err.println(" -details Display full report of all regions."); |
| System.err.println(" -timelag {timeInSeconds} Process only regions that " + |
| " have not experienced any metadata updates in the last " + |
| " {{timeInSeconds} seconds."); |
| System.err.println(" -fix Try to fix some of the errors."); |
| System.err.println(" -summary Print only summary of the tables and status."); |
| |
| Runtime.getRuntime().exit(-2); |
| } |
| |
| /** |
| * Main program |
| * @param args |
| * @throws Exception |
| */ |
| public static void main(String [] args) throws Exception { |
| |
| // create a fsck object |
| Configuration conf = HBaseConfiguration.create(); |
| conf.set("fs.defaultFS", conf.get("hbase.rootdir")); |
| HBaseFsck fsck = new HBaseFsck(conf); |
| |
| // Process command-line args. |
| for (int i = 0; i < args.length; i++) { |
| String cmd = args[i]; |
| if (cmd.equals("-details")) { |
| fsck.displayFullReport(); |
| } else if (cmd.equals("-timelag")) { |
| if (i == args.length - 1) { |
| System.err.println("HBaseFsck: -timelag needs a value."); |
| printUsageAndExit(); |
| } |
| try { |
| long timelag = Long.parseLong(args[i+1]); |
| fsck.setTimeLag(timelag); |
| } catch (NumberFormatException e) { |
| System.err.println("-timelag needs a numeric value."); |
| printUsageAndExit(); |
| } |
| i++; |
| } else if (cmd.equals("-fix")) { |
| fsck.setFixErrors(); |
| } else if (cmd.equals("-summary")) { |
| fsck.setSummary(); |
| } else { |
| String str = "Unknown command line option : " + cmd; |
| LOG.info(str); |
| System.out.println(str); |
| printUsageAndExit(); |
| } |
| } |
| // do the real work of fsck |
| int code = fsck.doWork(); |
| // If we have changed the HBase state it is better to run fsck again |
| // to see if we haven't broken something else in the process. |
| // We run it only once more because otherwise we can easily fall into |
| // an infinite loop. |
| if (fsck.shouldRerun()) { |
| code = fsck.doWork(); |
| } |
| |
| Runtime.getRuntime().exit(code); |
| } |
| } |
| |
