| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.hadoop.hbase.io; |
| |
| import java.io.IOException; |
| import java.util.regex.Matcher; |
| import java.util.regex.Pattern; |
| |
| import org.apache.commons.logging.Log; |
| import org.apache.commons.logging.LogFactory; |
| import org.apache.hadoop.hbase.classification.InterfaceAudience; |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.fs.FileSystem; |
| import org.apache.hadoop.fs.Path; |
| import org.apache.hadoop.hbase.TableName; |
| import org.apache.hadoop.hbase.HConstants; |
| import org.apache.hadoop.hbase.HRegionInfo; |
| import org.apache.hadoop.hbase.mob.MobConstants; |
| import org.apache.hadoop.hbase.regionserver.HRegion; |
| import org.apache.hadoop.hbase.regionserver.StoreFileInfo; |
| import org.apache.hadoop.hbase.util.FSUtils; |
| import org.apache.hadoop.hbase.util.HFileArchiveUtil; |
| import org.apache.hadoop.hbase.util.Pair; |
| |
| /** |
| * HFileLink describes a link to an hfile. |
| * |
| * An hfile can be served from a region or from the hfile archive directory (/hbase/.archive) |
| * HFileLink allows to access the referenced hfile regardless of the location where it is. |
| * |
| * <p>Searches for hfiles in the following order and locations: |
| * <ul> |
| * <li>/hbase/table/region/cf/hfile</li> |
| * <li>/hbase/.archive/table/region/cf/hfile</li> |
| * </ul> |
| * |
| * The link checks first in the original path if it is not present |
| * it fallbacks to the archived path. |
| */ |
| @InterfaceAudience.Private |
| @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="EQ_DOESNT_OVERRIDE_EQUALS", |
| justification="To be fixed but warning suppressed for now") |
| public class HFileLink extends FileLink { |
| private static final Log LOG = LogFactory.getLog(HFileLink.class); |
| |
| /** |
| * A non-capture group, for HFileLink, so that this can be embedded. |
| * The HFileLink describe a link to an hfile in a different table/region |
| * and the name is in the form: table=region-hfile. |
| * <p> |
| * Table name is ([a-zA-Z_0-9][a-zA-Z_0-9.-]*), so '=' is an invalid character for the table name. |
| * Region name is ([a-f0-9]+), so '-' is an invalid character for the region name. |
| * HFile is ([0-9a-f]+(?:_SeqId_[0-9]+_)?) covering the plain hfiles (uuid) |
| * and the bulk loaded (_SeqId_[0-9]+_) hfiles. |
| */ |
| public static final String LINK_NAME_REGEX = |
| String.format("(?:(?:%s=)?)%s=%s-%s", |
| TableName.VALID_NAMESPACE_REGEX, TableName.VALID_TABLE_QUALIFIER_REGEX, |
| HRegionInfo.ENCODED_REGION_NAME_REGEX, StoreFileInfo.HFILE_NAME_REGEX); |
| |
| /** Define the HFile Link name parser in the form of: table=region-hfile */ |
| //made package private for testing |
| static final Pattern LINK_NAME_PATTERN = |
| Pattern.compile(String.format("^(?:(%s)(?:\\=))?(%s)=(%s)-(%s)$", |
| TableName.VALID_NAMESPACE_REGEX, TableName.VALID_TABLE_QUALIFIER_REGEX, |
| HRegionInfo.ENCODED_REGION_NAME_REGEX, StoreFileInfo.HFILE_NAME_REGEX)); |
| |
| /** |
| * The pattern should be used for hfile and reference links |
| * that can be found in /hbase/table/region/family/ |
| */ |
| private static final Pattern REF_OR_HFILE_LINK_PATTERN = |
| Pattern.compile(String.format("^(?:(%s)(?:=))?(%s)=(%s)-(.+)$", |
| TableName.VALID_NAMESPACE_REGEX, TableName.VALID_TABLE_QUALIFIER_REGEX, |
| HRegionInfo.ENCODED_REGION_NAME_REGEX)); |
| |
| private final Path archivePath; |
| private final Path originPath; |
| private final Path mobPath; |
| private final Path tempPath; |
| |
| /** |
| * Dead simple hfile link constructor |
| */ |
| public HFileLink(final Path originPath, final Path tempPath, final Path mobPath, |
| final Path archivePath) { |
| this.tempPath = tempPath; |
| this.originPath = originPath; |
| this.mobPath = mobPath; |
| this.archivePath = archivePath; |
| setLocations(originPath, tempPath, mobPath, archivePath); |
| } |
| |
| |
| /** |
| * @param conf {@link Configuration} from which to extract specific archive locations |
| * @param hFileLinkPattern The path ending with a HFileLink pattern. (table=region-hfile) |
| * @throws IOException on unexpected error. |
| */ |
| public static final HFileLink buildFromHFileLinkPattern(Configuration conf, Path hFileLinkPattern) |
| throws IOException { |
| return buildFromHFileLinkPattern(FSUtils.getRootDir(conf), |
| HFileArchiveUtil.getArchivePath(conf), hFileLinkPattern); |
| } |
| |
| |
| |
| /** |
| * @param rootDir Path to the root directory where hbase files are stored |
| * @param archiveDir Path to the hbase archive directory |
| * @param hFileLinkPattern The path of the HFile Link. |
| */ |
| public final static HFileLink buildFromHFileLinkPattern(final Path rootDir, |
| final Path archiveDir, |
| final Path hFileLinkPattern) { |
| Path hfilePath = getHFileLinkPatternRelativePath(hFileLinkPattern); |
| Path tempPath = new Path(new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY), hfilePath); |
| Path originPath = new Path(rootDir, hfilePath); |
| Path mobPath = new Path(new Path(rootDir, MobConstants.MOB_DIR_NAME), hfilePath); |
| Path archivePath = new Path(archiveDir, hfilePath); |
| return new HFileLink(originPath, tempPath, mobPath, archivePath); |
| } |
| |
| /** |
| * Create an HFileLink relative path for the table/region/family/hfile location |
| * @param table Table name |
| * @param region Region Name |
| * @param family Family Name |
| * @param hfile HFile Name |
| * @return the relative Path to open the specified table/region/family/hfile link |
| */ |
| public static Path createPath(final TableName table, final String region, |
| final String family, final String hfile) { |
| if (HFileLink.isHFileLink(hfile)) { |
| return new Path(family, hfile); |
| } |
| return new Path(family, HFileLink.createHFileLinkName(table, region, hfile)); |
| } |
| |
| /** |
| * Create an HFileLink instance from table/region/family/hfile location |
| * @param conf {@link Configuration} from which to extract specific archive locations |
| * @param table Table name |
| * @param region Region Name |
| * @param family Family Name |
| * @param hfile HFile Name |
| * @return Link to the file with the specified table/region/family/hfile location |
| * @throws IOException on unexpected error. |
| */ |
| public static HFileLink build(final Configuration conf, final TableName table, |
| final String region, final String family, final String hfile) |
| throws IOException { |
| return HFileLink.buildFromHFileLinkPattern(conf, createPath(table, region, family, hfile)); |
| } |
| |
| /** |
| * @return the origin path of the hfile. |
| */ |
| public Path getOriginPath() { |
| return this.originPath; |
| } |
| |
| /** |
| * @return the path of the archived hfile. |
| */ |
| public Path getArchivePath() { |
| return this.archivePath; |
| } |
| |
| /** |
| * @return the path of the mob hfiles. |
| */ |
| public Path getMobPath() { |
| return this.mobPath; |
| } |
| |
| /** |
| * @param path Path to check. |
| * @return True if the path is a HFileLink. |
| */ |
| public static boolean isHFileLink(final Path path) { |
| return isHFileLink(path.getName()); |
| } |
| |
| |
| /** |
| * @param fileName File name to check. |
| * @return True if the path is a HFileLink. |
| */ |
| public static boolean isHFileLink(String fileName) { |
| Matcher m = LINK_NAME_PATTERN.matcher(fileName); |
| if (!m.matches()) return false; |
| return m.groupCount() > 2 && m.group(4) != null && m.group(3) != null && m.group(2) != null; |
| } |
| |
| /** |
| * Convert a HFileLink path to a table relative path. |
| * e.g. the link: /hbase/test/0123/cf/testtb=4567-abcd |
| * becomes: /hbase/testtb/4567/cf/abcd |
| * |
| * @param path HFileLink path |
| * @return Relative table path |
| * @throws IOException on unexpected error. |
| */ |
| private static Path getHFileLinkPatternRelativePath(final Path path) { |
| // table=region-hfile |
| Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(path.getName()); |
| if (!m.matches()) { |
| throw new IllegalArgumentException(path.getName() + " is not a valid HFileLink pattern!"); |
| } |
| |
| // Convert the HFileLink name into a real table/region/cf/hfile path. |
| TableName tableName = TableName.valueOf(m.group(1), m.group(2)); |
| String regionName = m.group(3); |
| String hfileName = m.group(4); |
| String familyName = path.getParent().getName(); |
| Path tableDir = FSUtils.getTableDir(new Path("./"), tableName); |
| return new Path(tableDir, new Path(regionName, new Path(familyName, |
| hfileName))); |
| } |
| |
| /** |
| * Get the HFile name of the referenced link |
| * |
| * @param fileName HFileLink file name |
| * @return the name of the referenced HFile |
| */ |
| public static String getReferencedHFileName(final String fileName) { |
| Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName); |
| if (!m.matches()) { |
| throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!"); |
| } |
| return(m.group(4)); |
| } |
| |
| /** |
| * Get the Region name of the referenced link |
| * |
| * @param fileName HFileLink file name |
| * @return the name of the referenced Region |
| */ |
| public static String getReferencedRegionName(final String fileName) { |
| Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName); |
| if (!m.matches()) { |
| throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!"); |
| } |
| return(m.group(3)); |
| } |
| |
| /** |
| * Get the Table name of the referenced link |
| * |
| * @param fileName HFileLink file name |
| * @return the name of the referenced Table |
| */ |
| public static TableName getReferencedTableName(final String fileName) { |
| Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(fileName); |
| if (!m.matches()) { |
| throw new IllegalArgumentException(fileName + " is not a valid HFileLink name!"); |
| } |
| return(TableName.valueOf(m.group(1), m.group(2))); |
| } |
| |
| /** |
| * Create a new HFileLink name |
| * |
| * @param hfileRegionInfo - Linked HFile Region Info |
| * @param hfileName - Linked HFile name |
| * @return file name of the HFile Link |
| */ |
| public static String createHFileLinkName(final HRegionInfo hfileRegionInfo, |
| final String hfileName) { |
| return createHFileLinkName(hfileRegionInfo.getTable(), |
| hfileRegionInfo.getEncodedName(), hfileName); |
| } |
| |
| /** |
| * Create a new HFileLink name |
| * |
| * @param tableName - Linked HFile table name |
| * @param regionName - Linked HFile region name |
| * @param hfileName - Linked HFile name |
| * @return file name of the HFile Link |
| */ |
| public static String createHFileLinkName(final TableName tableName, |
| final String regionName, final String hfileName) { |
| String s = String.format("%s=%s-%s", |
| tableName.getNameAsString().replace(TableName.NAMESPACE_DELIM, '='), |
| regionName, hfileName); |
| return s; |
| } |
| |
| /** |
| * Create a new HFileLink |
| * |
| * <p>It also adds a back-reference to the hfile back-reference directory |
| * to simplify the reference-count and the cleaning process. |
| * |
| * @param conf {@link Configuration} to read for the archive directory name |
| * @param fs {@link FileSystem} on which to write the HFileLink |
| * @param dstFamilyPath - Destination path (table/region/cf/) |
| * @param hfileRegionInfo - Linked HFile Region Info |
| * @param hfileName - Linked HFile name |
| * @return true if the file is created, otherwise the file exists. |
| * @throws IOException on file or parent directory creation failure |
| */ |
| public static boolean create(final Configuration conf, final FileSystem fs, |
| final Path dstFamilyPath, final HRegionInfo hfileRegionInfo, |
| final String hfileName) throws IOException { |
| return create(conf, fs, dstFamilyPath, hfileRegionInfo, hfileName, true); |
| } |
| |
| /** |
| * Create a new HFileLink |
| * |
| * <p>It also adds a back-reference to the hfile back-reference directory |
| * to simplify the reference-count and the cleaning process. |
| * |
| * @param conf {@link Configuration} to read for the archive directory name |
| * @param fs {@link FileSystem} on which to write the HFileLink |
| * @param dstFamilyPath - Destination path (table/region/cf/) |
| * @param hfileRegionInfo - Linked HFile Region Info |
| * @param hfileName - Linked HFile name |
| * @param createBackRef - Whether back reference should be created. Defaults to true. |
| * @return true if the file is created, otherwise the file exists. |
| * @throws IOException on file or parent directory creation failure |
| */ |
| public static boolean create(final Configuration conf, final FileSystem fs, |
| final Path dstFamilyPath, final HRegionInfo hfileRegionInfo, |
| final String hfileName, final boolean createBackRef) throws IOException { |
| TableName linkedTable = hfileRegionInfo.getTable(); |
| String linkedRegion = hfileRegionInfo.getEncodedName(); |
| return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, createBackRef); |
| } |
| |
| /** |
| * Create a new HFileLink |
| * |
| * <p>It also adds a back-reference to the hfile back-reference directory |
| * to simplify the reference-count and the cleaning process. |
| * |
| * @param conf {@link Configuration} to read for the archive directory name |
| * @param fs {@link FileSystem} on which to write the HFileLink |
| * @param dstFamilyPath - Destination path (table/region/cf/) |
| * @param linkedTable - Linked Table Name |
| * @param linkedRegion - Linked Region Name |
| * @param hfileName - Linked HFile name |
| * @return true if the file is created, otherwise the file exists. |
| * @throws IOException on file or parent directory creation failure |
| */ |
| public static boolean create(final Configuration conf, final FileSystem fs, |
| final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, |
| final String hfileName) throws IOException { |
| return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, true); |
| } |
| |
| /** |
| * Create a new HFileLink |
| * |
| * <p>It also adds a back-reference to the hfile back-reference directory |
| * to simplify the reference-count and the cleaning process. |
| * |
| * @param conf {@link Configuration} to read for the archive directory name |
| * @param fs {@link FileSystem} on which to write the HFileLink |
| * @param dstFamilyPath - Destination path (table/region/cf/) |
| * @param linkedTable - Linked Table Name |
| * @param linkedRegion - Linked Region Name |
| * @param hfileName - Linked HFile name |
| * @param createBackRef - Whether back reference should be created. Defaults to true. |
| * @return true if the file is created, otherwise the file exists. |
| * @throws IOException on file or parent directory creation failure |
| */ |
| public static boolean create(final Configuration conf, final FileSystem fs, |
| final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, |
| final String hfileName, final boolean createBackRef) throws IOException { |
| String familyName = dstFamilyPath.getName(); |
| String regionName = dstFamilyPath.getParent().getName(); |
| String tableName = FSUtils.getTableName(dstFamilyPath.getParent().getParent()) |
| .getNameAsString(); |
| |
| String name = createHFileLinkName(linkedTable, linkedRegion, hfileName); |
| String refName = createBackReferenceName(tableName, regionName); |
| |
| // Make sure the destination directory exists |
| fs.mkdirs(dstFamilyPath); |
| |
| // Make sure the FileLink reference directory exists |
| Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, |
| linkedTable, linkedRegion, familyName); |
| Path backRefPath = null; |
| if (createBackRef) { |
| Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName); |
| fs.mkdirs(backRefssDir); |
| |
| // Create the reference for the link |
| backRefPath = new Path(backRefssDir, refName); |
| fs.createNewFile(backRefPath); |
| } |
| try { |
| // Create the link |
| return fs.createNewFile(new Path(dstFamilyPath, name)); |
| } catch (IOException e) { |
| LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e); |
| // Revert the reference if the link creation failed |
| if (createBackRef) { |
| fs.delete(backRefPath, false); |
| } |
| throw e; |
| } |
| } |
| |
| /** |
| * Create a new HFileLink starting from a hfileLink name |
| * |
| * <p>It also adds a back-reference to the hfile back-reference directory |
| * to simplify the reference-count and the cleaning process. |
| * |
| * @param conf {@link Configuration} to read for the archive directory name |
| * @param fs {@link FileSystem} on which to write the HFileLink |
| * @param dstFamilyPath - Destination path (table/region/cf/) |
| * @param hfileLinkName - HFileLink name (it contains hfile-region-table) |
| * @return true if the file is created, otherwise the file exists. |
| * @throws IOException on file or parent directory creation failure |
| */ |
| public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs, |
| final Path dstFamilyPath, final String hfileLinkName) |
| throws IOException { |
| return createFromHFileLink(conf, fs, dstFamilyPath, hfileLinkName, true); |
| } |
| |
| /** |
| * Create a new HFileLink starting from a hfileLink name |
| * |
| * <p>It also adds a back-reference to the hfile back-reference directory |
| * to simplify the reference-count and the cleaning process. |
| * |
| * @param conf {@link Configuration} to read for the archive directory name |
| * @param fs {@link FileSystem} on which to write the HFileLink |
| * @param dstFamilyPath - Destination path (table/region/cf/) |
| * @param hfileLinkName - HFileLink name (it contains hfile-region-table) |
| * @param createBackRef - Whether back reference should be created. Defaults to true. |
| * @return true if the file is created, otherwise the file exists. |
| * @throws IOException on file or parent directory creation failure |
| */ |
| public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs, |
| final Path dstFamilyPath, final String hfileLinkName, final boolean createBackRef) |
| throws IOException { |
| Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName); |
| if (!m.matches()) { |
| throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!"); |
| } |
| return create(conf, fs, dstFamilyPath, TableName.valueOf(m.group(1), m.group(2)), |
| m.group(3), m.group(4), createBackRef); |
| } |
| |
| /** |
| * Create the back reference name |
| */ |
| //package-private for testing |
| static String createBackReferenceName(final String tableNameStr, |
| final String regionName) { |
| |
| return regionName + "." + tableNameStr.replace(TableName.NAMESPACE_DELIM, '='); |
| } |
| |
| /** |
| * Get the full path of the HFile referenced by the back reference |
| * |
| * @param rootDir root hbase directory |
| * @param linkRefPath Link Back Reference path |
| * @return full path of the referenced hfile |
| */ |
| public static Path getHFileFromBackReference(final Path rootDir, final Path linkRefPath) { |
| Pair<TableName, String> p = parseBackReferenceName(linkRefPath.getName()); |
| TableName linkTableName = p.getFirst(); |
| String linkRegionName = p.getSecond(); |
| |
| String hfileName = getBackReferenceFileName(linkRefPath.getParent()); |
| Path familyPath = linkRefPath.getParent().getParent(); |
| Path regionPath = familyPath.getParent(); |
| Path tablePath = regionPath.getParent(); |
| |
| String linkName = createHFileLinkName(FSUtils.getTableName(tablePath), |
| regionPath.getName(), hfileName); |
| Path linkTableDir = FSUtils.getTableDir(rootDir, linkTableName); |
| Path regionDir = HRegion.getRegionDir(linkTableDir, linkRegionName); |
| return new Path(new Path(regionDir, familyPath.getName()), linkName); |
| } |
| |
| static Pair<TableName, String> parseBackReferenceName(String name) { |
| int separatorIndex = name.indexOf('.'); |
| String linkRegionName = name.substring(0, separatorIndex); |
| String tableSubstr = name.substring(separatorIndex + 1) |
| .replace('=', TableName.NAMESPACE_DELIM); |
| TableName linkTableName = TableName.valueOf(tableSubstr); |
| return new Pair<TableName, String>(linkTableName, linkRegionName); |
| } |
| |
| /** |
| * Get the full path of the HFile referenced by the back reference |
| * |
| * @param conf {@link Configuration} to read for the archive directory name |
| * @param linkRefPath Link Back Reference path |
| * @return full path of the referenced hfile |
| * @throws IOException on unexpected error. |
| */ |
| public static Path getHFileFromBackReference(final Configuration conf, final Path linkRefPath) |
| throws IOException { |
| return getHFileFromBackReference(FSUtils.getRootDir(conf), linkRefPath); |
| } |
| |
| } |