| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.hdfs.protocol; |
| |
| import java.util.Comparator; |
| import java.util.Map; |
| import java.util.SortedSet; |
| import java.util.TreeSet; |
| |
| import org.apache.hadoop.classification.InterfaceAudience; |
| |
| /** |
| * This class tracks changes in the layout version of HDFS. |
| * |
| * Layout version is changed for following reasons: |
| * <ol> |
| * <li>The layout of how namenode or datanode stores information |
| * on disk changes.</li> |
| * <li>A new operation code is added to the editlog.</li> |
| * <li>Modification such as format of a record, content of a record |
| * in editlog or fsimage.</li> |
| * </ol> |
| * <br> |
| * <b>How to update layout version:<br></b> |
| * When a change requires new layout version, please add an entry into |
| * {@link Feature} with a short enum name, new layout version and description |
| * of the change. Please see {@link Feature} for further details. |
| * <br> |
| */ |
| @InterfaceAudience.Private |
| public class LayoutVersion { |
| /** |
| * Version in which HDFS-2991 was fixed. This bug caused OP_ADD to |
| * sometimes be skipped for append() calls. If we see such a case when |
| * loading the edits, but the version is known to have that bug, we |
| * workaround the issue. Otherwise we should consider it a corruption |
| * and bail. |
| */ |
| public static final int BUGFIX_HDFS_2991_VERSION = -40; |
| |
| /** |
| * The interface to be implemented by NameNode and DataNode layout features |
| */ |
| public interface LayoutFeature { |
| public FeatureInfo getInfo(); |
| } |
| |
| /** |
| * Enums for features that change the layout version before rolling |
| * upgrade is supported. |
| * <br><br> |
| * To add a new layout version: |
| * <ul> |
| * <li>Define a new enum constant with a short enum name, the new layout version |
| * and description of the added feature.</li> |
| * <li>When adding a layout version with an ancestor that is not same as |
| * its immediate predecessor, use the constructor where a specific ancestor |
| * can be passed. |
| * </li> |
| * </ul> |
| */ |
| public enum Feature implements LayoutFeature { |
| NAMESPACE_QUOTA(-16, "Support for namespace quotas"), |
| FILE_ACCESS_TIME(-17, "Support for access time on files"), |
| DISKSPACE_QUOTA(-18, "Support for disk space quotas"), |
| STICKY_BIT(-19, "Support for sticky bits"), |
| APPEND_RBW_DIR(-20, "Datanode has \"rbw\" subdirectory for append"), |
| ATOMIC_RENAME(-21, "Support for atomic rename"), |
| CONCAT(-22, "Support for concat operation"), |
| SYMLINKS(-23, "Support for symbolic links"), |
| DELEGATION_TOKEN(-24, "Support for delegation tokens for security"), |
| FSIMAGE_COMPRESSION(-25, "Support for fsimage compression"), |
| FSIMAGE_CHECKSUM(-26, "Support checksum for fsimage"), |
| REMOVE_REL13_DISK_LAYOUT_SUPPORT(-27, "Remove support for 0.13 disk layout"), |
| EDITS_CHECKSUM(-28, "Support checksum for editlog"), |
| UNUSED(-29, "Skipped version"), |
| FSIMAGE_NAME_OPTIMIZATION(-30, "Store only last part of path in fsimage"), |
| RESERVED_REL20_203(-31, -19, "Reserved for release 0.20.203", true, |
| DELEGATION_TOKEN), |
| RESERVED_REL20_204(-32, -31, "Reserved for release 0.20.204", true), |
| RESERVED_REL22(-33, -27, "Reserved for release 0.22", true), |
| RESERVED_REL23(-34, -30, "Reserved for release 0.23", true), |
| FEDERATION(-35, "Support for namenode federation"), |
| LEASE_REASSIGNMENT(-36, "Support for persisting lease holder reassignment"), |
| STORED_TXIDS(-37, "Transaction IDs are stored in edits log and image files"), |
| TXID_BASED_LAYOUT(-38, "File names in NN Storage are based on transaction IDs"), |
| EDITLOG_OP_OPTIMIZATION(-39, |
| "Use LongWritable and ShortWritable directly instead of ArrayWritable of UTF8"), |
| OPTIMIZE_PERSIST_BLOCKS(-40, |
| "Serialize block lists with delta-encoded variable length ints, " + |
| "add OP_UPDATE_BLOCKS"), |
| RESERVED_REL1_2_0(-41, -32, "Reserved for release 1.2.0", true, CONCAT), |
| ADD_INODE_ID(-42, -40, "Assign a unique inode id for each inode", false), |
| SNAPSHOT(-43, "Support for snapshot feature"), |
| RESERVED_REL1_3_0(-44, -41, "Reserved for release 1.3.0", true, |
| ADD_INODE_ID, SNAPSHOT, FSIMAGE_NAME_OPTIMIZATION), |
| OPTIMIZE_SNAPSHOT_INODES(-45, -43, |
| "Reduce snapshot inode memory footprint", false), |
| SEQUENTIAL_BLOCK_ID(-46, "Allocate block IDs sequentially and store " + |
| "block IDs in the edits log and image files"), |
| EDITLOG_SUPPORT_RETRYCACHE(-47, "Record ClientId and CallId in editlog to " |
| + "enable rebuilding retry cache in case of HA failover"), |
| EDITLOG_ADD_BLOCK(-48, "Add new editlog that only records allocation of " |
| + "the new block instead of the entire block list"), |
| ADD_DATANODE_AND_STORAGE_UUIDS(-49, "Replace StorageID with DatanodeUuid." |
| + " Use distinct StorageUuid per storage directory."), |
| ADD_LAYOUT_FLAGS(-50, "Add support for layout flags."), |
| CACHING(-51, "Support for cache pools and path-based caching"), |
| // Hadoop 2.4.0 |
| PROTOBUF_FORMAT(-52, "Use protobuf to serialize FSImage"), |
| EXTENDED_ACL(-53, "Extended ACL"), |
| RESERVED_REL2_4_0(-54, -51, "Reserved for release 2.4.0", true, |
| PROTOBUF_FORMAT, EXTENDED_ACL); |
| |
| private final FeatureInfo info; |
| |
| /** |
| * Feature that is added at layout version {@code lv} - 1. |
| * @param lv new layout version with the addition of this feature |
| * @param description description of the feature |
| */ |
| Feature(final int lv, final String description) { |
| this(lv, lv + 1, description, false); |
| } |
| |
| /** |
| * Feature that is added at layout version {@code ancestoryLV}. |
| * @param lv new layout version with the addition of this feature |
| * @param ancestorLV layout version from which the new lv is derived from. |
| * @param description description of the feature |
| * @param reserved true when this is a layout version reserved for previous |
| * version |
| * @param features set of features that are to be enabled for this version |
| */ |
| Feature(final int lv, final int ancestorLV, final String description, |
| boolean reserved, Feature... features) { |
| info = new FeatureInfo(lv, ancestorLV, description, reserved, features); |
| } |
| |
| @Override |
| public FeatureInfo getInfo() { |
| return info; |
| } |
| } |
| |
| /** Feature information. */ |
| public static class FeatureInfo { |
| private final int lv; |
| private final int ancestorLV; |
| private final Integer minCompatLV; |
| private final String description; |
| private final boolean reserved; |
| private final LayoutFeature[] specialFeatures; |
| |
| public FeatureInfo(final int lv, final int ancestorLV, final String description, |
| boolean reserved, LayoutFeature... specialFeatures) { |
| this(lv, ancestorLV, null, description, reserved, specialFeatures); |
| } |
| |
| public FeatureInfo(final int lv, final int ancestorLV, Integer minCompatLV, |
| final String description, boolean reserved, |
| LayoutFeature... specialFeatures) { |
| this.lv = lv; |
| this.ancestorLV = ancestorLV; |
| this.minCompatLV = minCompatLV; |
| this.description = description; |
| this.reserved = reserved; |
| this.specialFeatures = specialFeatures; |
| } |
| |
| /** |
| * Accessor method for feature layout version |
| * @return int lv value |
| */ |
| public int getLayoutVersion() { |
| return lv; |
| } |
| |
| /** |
| * Accessor method for feature ancestor layout version |
| * @return int ancestor LV value |
| */ |
| public int getAncestorLayoutVersion() { |
| return ancestorLV; |
| } |
| |
| /** |
| * Accessor method for feature minimum compatible layout version. If the |
| * feature does not define a minimum compatible layout version, then this |
| * method returns the feature's own layout version. This would indicate |
| * that the feature cannot provide compatibility with any prior layout |
| * version. |
| * |
| * @return int minimum compatible LV value |
| */ |
| public int getMinimumCompatibleLayoutVersion() { |
| return minCompatLV != null ? minCompatLV : lv; |
| } |
| |
| /** |
| * Accessor method for feature description |
| * @return String feature description |
| */ |
| public String getDescription() { |
| return description; |
| } |
| |
| public boolean isReservedForOldRelease() { |
| return reserved; |
| } |
| |
| public LayoutFeature[] getSpecialFeatures() { |
| return specialFeatures; |
| } |
| } |
| |
| static class LayoutFeatureComparator implements Comparator<LayoutFeature> { |
| @Override |
| public int compare(LayoutFeature arg0, LayoutFeature arg1) { |
| return arg0.getInfo().getLayoutVersion() |
| - arg1.getInfo().getLayoutVersion(); |
| } |
| } |
| |
| public static void updateMap(Map<Integer, SortedSet<LayoutFeature>> map, |
| LayoutFeature[] features) { |
| // Go through all the enum constants and build a map of |
| // LayoutVersion <-> Set of all supported features in that LayoutVersion |
| SortedSet<LayoutFeature> existingFeatures = new TreeSet<LayoutFeature>( |
| new LayoutFeatureComparator()); |
| for (SortedSet<LayoutFeature> s : map.values()) { |
| existingFeatures.addAll(s); |
| } |
| LayoutFeature prevF = existingFeatures.isEmpty() ? null : |
| existingFeatures.first(); |
| for (LayoutFeature f : features) { |
| final FeatureInfo info = f.getInfo(); |
| int minCompatLV = info.getMinimumCompatibleLayoutVersion(); |
| if (prevF != null && |
| minCompatLV > prevF.getInfo().getMinimumCompatibleLayoutVersion()) { |
| throw new AssertionError(String.format( |
| "Features must be listed in order of minimum compatible layout " + |
| "version. Check features %s and %s.", prevF, f)); |
| } |
| prevF = f; |
| SortedSet<LayoutFeature> ancestorSet = map.get(info.getAncestorLayoutVersion()); |
| if (ancestorSet == null) { |
| // Empty set |
| ancestorSet = new TreeSet<LayoutFeature>(new LayoutFeatureComparator()); |
| map.put(info.getAncestorLayoutVersion(), ancestorSet); |
| } |
| SortedSet<LayoutFeature> featureSet = new TreeSet<LayoutFeature>(ancestorSet); |
| if (info.getSpecialFeatures() != null) { |
| for (LayoutFeature specialFeature : info.getSpecialFeatures()) { |
| featureSet.add(specialFeature); |
| } |
| } |
| featureSet.add(f); |
| map.put(info.getLayoutVersion(), featureSet); |
| } |
| } |
| |
| /** |
| * Gets formatted string that describes {@link LayoutVersion} information. |
| */ |
| public String getString(Map<Integer, SortedSet<LayoutFeature>> map, |
| LayoutFeature[] values) { |
| final StringBuilder buf = new StringBuilder(); |
| buf.append("Feature List:\n"); |
| for (LayoutFeature f : values) { |
| final FeatureInfo info = f.getInfo(); |
| buf.append(f).append(" introduced in layout version ") |
| .append(info.getLayoutVersion()).append(" (") |
| .append(info.getDescription()).append(")\n"); |
| } |
| |
| buf.append("\n\nLayoutVersion and supported features:\n"); |
| for (LayoutFeature f : values) { |
| final FeatureInfo info = f.getInfo(); |
| buf.append(info.getLayoutVersion()).append(": ") |
| .append(map.get(info.getLayoutVersion())).append("\n"); |
| } |
| return buf.toString(); |
| } |
| |
| /** |
| * Returns true if a given feature is supported in the given layout version |
| * @param map layout feature map |
| * @param f Feature |
| * @param lv LayoutVersion |
| * @return true if {@code f} is supported in layout version {@code lv} |
| */ |
| public static boolean supports(Map<Integer, SortedSet<LayoutFeature>> map, |
| final LayoutFeature f, final int lv) { |
| final SortedSet<LayoutFeature> set = map.get(lv); |
| return set != null && set.contains(f); |
| } |
| |
| /** |
| * Get the current layout version |
| */ |
| public static int getCurrentLayoutVersion(LayoutFeature[] features) { |
| return getLastNonReservedFeature(features).getInfo().getLayoutVersion(); |
| } |
| |
| /** |
| * Gets the minimum compatible layout version. |
| * |
| * @param features all features to check |
| * @return minimum compatible layout version |
| */ |
| public static int getMinimumCompatibleLayoutVersion( |
| LayoutFeature[] features) { |
| return getLastNonReservedFeature(features).getInfo() |
| .getMinimumCompatibleLayoutVersion(); |
| } |
| |
| static LayoutFeature getLastNonReservedFeature(LayoutFeature[] features) { |
| for (int i = features.length -1; i >= 0; i--) { |
| final FeatureInfo info = features[i].getInfo(); |
| if (!info.isReservedForOldRelease()) { |
| return features[i]; |
| } |
| } |
| throw new AssertionError("All layout versions are reserved."); |
| } |
| } |