| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.hdfs.tools.offlineImageViewer; |
| |
| import com.google.common.base.Preconditions; |
| import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_MASK; |
| import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_OFFSET; |
| import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_SCOPE_OFFSET; |
| import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_TYPE_OFFSET; |
| import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_MASK; |
| import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAME_OFFSET; |
| import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_OFFSET; |
| import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_EXT_OFFSET; |
| import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_NAMESPACE_EXT_MASK; |
| import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.*; |
| |
| import java.io.BufferedOutputStream; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.FileOutputStream; |
| import java.io.IOException; |
| import java.io.InputStreamReader; |
| import java.nio.ByteBuffer; |
| import java.nio.charset.Charset; |
| import java.nio.file.Files; |
| import java.nio.file.Paths; |
| import java.security.DigestOutputStream; |
| import java.security.MessageDigest; |
| import java.text.ParseException; |
| import java.text.SimpleDateFormat; |
| import java.util.Date; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.LinkedList; |
| import java.util.Map; |
| |
| import com.google.common.io.CountingOutputStream; |
| import com.google.common.primitives.Ints; |
| import com.google.protobuf.ByteString; |
| import com.google.protobuf.TextFormat; |
| |
| import org.apache.commons.logging.Log; |
| import org.apache.commons.logging.LogFactory; |
| import org.apache.hadoop.classification.InterfaceAudience; |
| import org.apache.hadoop.classification.InterfaceStability; |
| import org.apache.hadoop.fs.permission.AclEntry; |
| import org.apache.hadoop.fs.permission.FsPermission; |
| import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; |
| import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ECSchemaProto; |
| import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ErasureCodingPolicyProto; |
| import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto; |
| import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto; |
| import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto; |
| import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos; |
| import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName; |
| import org.apache.hadoop.hdfs.server.namenode.FSImageUtil; |
| import org.apache.hadoop.hdfs.server.namenode.FsImageProto; |
| import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection; |
| import org.apache.hadoop.hdfs.server.namenode.FsImageProto.ErasureCodingSection; |
| import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary; |
| import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry; |
| import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection; |
| import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.AclFeatureProto; |
| import org.apache.hadoop.hdfs.server.namenode.FsImageProto.NameSystemSection; |
| import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection; |
| import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection.DiffEntry; |
| import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion; |
| import org.apache.hadoop.hdfs.util.MD5FileUtils; |
| import org.apache.hadoop.hdfs.util.XMLUtils; |
| import org.apache.hadoop.io.IOUtils; |
| import org.apache.hadoop.io.MD5Hash; |
| import org.apache.hadoop.util.StringUtils; |
| |
| import javax.xml.bind.annotation.adapters.HexBinaryAdapter; |
| import javax.xml.stream.XMLEventReader; |
| import javax.xml.stream.XMLInputFactory; |
| import javax.xml.stream.XMLStreamConstants; |
| import javax.xml.stream.XMLStreamException; |
| import javax.xml.stream.events.XMLEvent; |
| |
| @InterfaceAudience.Private |
| @InterfaceStability.Unstable |
| class OfflineImageReconstructor { |
| public static final Log LOG = |
| LogFactory.getLog(OfflineImageReconstructor.class); |
| |
| /** |
| * The output stream. |
| */ |
| private final CountingOutputStream out; |
| |
| /** |
| * A source of XML events based on the input file. |
| */ |
| private final XMLEventReader events; |
| |
| /** |
| * A map of section names to section handler objects. |
| */ |
| private final HashMap<String, SectionProcessor> sections; |
| |
| /** |
| * The offset of the start of the current section. |
| */ |
| private long sectionStartOffset; |
| |
| /** |
| * The FileSummary builder, where we gather information about each section |
| * we wrote. |
| */ |
| private final FileSummary.Builder fileSummaryBld = |
| FileSummary.newBuilder(); |
| |
| /** |
| * The string table. See registerStringId for details. |
| */ |
| private final HashMap<String, Integer> stringTable = new HashMap<>(); |
| |
| /** |
| * The date formatter to use with fsimage XML files. |
| */ |
| private final SimpleDateFormat isoDateFormat; |
| |
| /** |
| * The latest string ID. See registerStringId for details. |
| */ |
| private int latestStringId = 0; |
| |
| private static final String EMPTY_STRING = ""; |
| |
| private OfflineImageReconstructor(CountingOutputStream out, |
| InputStreamReader reader) throws XMLStreamException { |
| this.out = out; |
| XMLInputFactory factory = XMLInputFactory.newInstance(); |
| this.events = factory.createXMLEventReader(reader); |
| this.sections = new HashMap<>(); |
| this.sections.put(NameSectionProcessor.NAME, new NameSectionProcessor()); |
| this.sections.put(ErasureCodingSectionProcessor.NAME, |
| new ErasureCodingSectionProcessor()); |
| this.sections.put(INodeSectionProcessor.NAME, new INodeSectionProcessor()); |
| this.sections.put(SecretManagerSectionProcessor.NAME, |
| new SecretManagerSectionProcessor()); |
| this.sections.put(CacheManagerSectionProcessor.NAME, |
| new CacheManagerSectionProcessor()); |
| this.sections.put(SnapshotDiffSectionProcessor.NAME, |
| new SnapshotDiffSectionProcessor()); |
| this.sections.put(INodeReferenceSectionProcessor.NAME, |
| new INodeReferenceSectionProcessor()); |
| this.sections.put(INodeDirectorySectionProcessor.NAME, |
| new INodeDirectorySectionProcessor()); |
| this.sections.put(FilesUnderConstructionSectionProcessor.NAME, |
| new FilesUnderConstructionSectionProcessor()); |
| this.sections.put(SnapshotSectionProcessor.NAME, |
| new SnapshotSectionProcessor()); |
| this.isoDateFormat = PBImageXmlWriter.createSimpleDateFormat(); |
| } |
| |
| /** |
| * Read the next tag start or end event. |
| * |
| * @param expected The name of the next tag we expect. |
| * We will validate that the tag has this name, |
| * unless this string is enclosed in braces. |
| * @param allowEnd If true, we will also end events. |
| * If false, end events cause an exception. |
| * |
| * @return The next tag start or end event. |
| */ |
| private XMLEvent expectTag(String expected, boolean allowEnd) |
| throws IOException { |
| XMLEvent ev = null; |
| while (true) { |
| try { |
| ev = events.nextEvent(); |
| } catch (XMLStreamException e) { |
| throw new IOException("Expecting " + expected + |
| ", but got XMLStreamException", e); |
| } |
| switch (ev.getEventType()) { |
| case XMLEvent.ATTRIBUTE: |
| throw new IOException("Got unexpected attribute: " + ev); |
| case XMLEvent.CHARACTERS: |
| if (!ev.asCharacters().isWhiteSpace()) { |
| throw new IOException("Got unxpected characters while " + |
| "looking for " + expected + ": " + |
| ev.asCharacters().getData()); |
| } |
| break; |
| case XMLEvent.END_ELEMENT: |
| if (!allowEnd) { |
| throw new IOException("Got unexpected end event " + |
| "while looking for " + expected); |
| } |
| return ev; |
| case XMLEvent.START_ELEMENT: |
| if (!expected.startsWith("[")) { |
| if (!ev.asStartElement().getName().getLocalPart(). |
| equals(expected)) { |
| throw new IOException("Failed to find <" + expected + ">; " + |
| "got " + ev.asStartElement().getName().getLocalPart() + |
| " instead."); |
| } |
| } |
| return ev; |
| default: |
| // Ignore other event types like comment, etc. |
| if (LOG.isTraceEnabled()) { |
| LOG.trace("Skipping XMLEvent of type " + |
| ev.getEventType() + "(" + ev + ")"); |
| } |
| break; |
| } |
| } |
| } |
| |
| private void expectTagEnd(String expected) throws IOException { |
| XMLEvent ev = expectTag(expected, true); |
| if (ev.getEventType() != XMLStreamConstants.END_ELEMENT) { |
| throw new IOException("Expected tag end event for " + expected + |
| ", but got: " + ev); |
| } |
| if (!expected.startsWith("[")) { |
| String tag = ev.asEndElement().getName().getLocalPart(); |
| if (!tag.equals(expected)) { |
| throw new IOException("Expected tag end event for " + expected + |
| ", but got tag end event for " + tag); |
| } |
| } |
| } |
| |
| private static class Node { |
| private static final String EMPTY = ""; |
| HashMap<String, LinkedList<Node>> children; |
| String val = EMPTY; |
| |
| void addChild(String key, Node node) { |
| if (children == null) { |
| children = new HashMap<>(); |
| } |
| LinkedList<Node> cur = children.get(key); |
| if (cur == null) { |
| cur = new LinkedList<>(); |
| children.put(key, cur); |
| } |
| cur.add(node); |
| } |
| |
| Node removeChild(String key) { |
| if (children == null) { |
| return null; |
| } |
| LinkedList<Node> cur = children.get(key); |
| if (cur == null) { |
| return null; |
| } |
| Node node = cur.remove(); |
| if ((node == null) || cur.isEmpty()) { |
| children.remove(key); |
| } |
| return node; |
| } |
| |
| String removeChildStr(String key) { |
| Node child = removeChild(key); |
| if (child == null) { |
| return null; |
| } |
| if ((child.children != null) && (!child.children.isEmpty())) { |
| throw new RuntimeException("Node " + key + " contains children " + |
| "of its own."); |
| } |
| return child.getVal(); |
| } |
| |
| Integer removeChildInt(String key) throws IOException { |
| String str = removeChildStr(key); |
| if (str == null) { |
| return null; |
| } |
| return Integer.valueOf(str); |
| } |
| |
| Long removeChildLong(String key) throws IOException { |
| String str = removeChildStr(key); |
| if (str == null) { |
| return null; |
| } |
| return Long.valueOf(str); |
| } |
| |
| boolean removeChildBool(String key) throws IOException { |
| String str = removeChildStr(key); |
| if (str == null) { |
| return false; |
| } |
| return true; |
| } |
| |
| String getRemainingKeyNames() { |
| if (children == null) { |
| return ""; |
| } |
| return StringUtils.join(", ", children.keySet()); |
| } |
| |
| void verifyNoRemainingKeys(String sectionName) throws IOException { |
| String remainingKeyNames = getRemainingKeyNames(); |
| if (!remainingKeyNames.isEmpty()) { |
| throw new IOException("Found unknown XML keys in " + |
| sectionName + ": " + remainingKeyNames); |
| } |
| } |
| |
| void setVal(String val) { |
| this.val = val; |
| } |
| |
| String getVal() { |
| return val; |
| } |
| |
| String dump() { |
| StringBuilder bld = new StringBuilder(); |
| if ((children != null) && (!children.isEmpty())) { |
| bld.append("{"); |
| } |
| if (val != null) { |
| bld.append("[").append(val).append("]"); |
| } |
| if ((children != null) && (!children.isEmpty())) { |
| String prefix = ""; |
| for (Map.Entry<String, LinkedList<Node>> entry : children.entrySet()) { |
| for (Node n : entry.getValue()) { |
| bld.append(prefix); |
| bld.append(entry.getKey()).append(": "); |
| bld.append(n.dump()); |
| prefix = ", "; |
| } |
| } |
| bld.append("}"); |
| } |
| return bld.toString(); |
| } |
| } |
| |
| private void loadNodeChildrenHelper(Node parent, String expected, |
| String terminators[]) throws IOException { |
| XMLEvent ev = null; |
| while (true) { |
| try { |
| ev = events.peek(); |
| switch (ev.getEventType()) { |
| case XMLEvent.END_ELEMENT: |
| if (terminators.length != 0) { |
| return; |
| } |
| events.nextEvent(); |
| return; |
| case XMLEvent.START_ELEMENT: |
| String key = ev.asStartElement().getName().getLocalPart(); |
| for (String terminator : terminators) { |
| if (terminator.equals(key)) { |
| return; |
| } |
| } |
| events.nextEvent(); |
| Node node = new Node(); |
| parent.addChild(key, node); |
| loadNodeChildrenHelper(node, expected, new String[0]); |
| break; |
| case XMLEvent.CHARACTERS: |
| String val = XMLUtils. |
| unmangleXmlString(ev.asCharacters().getData(), true); |
| parent.setVal(val); |
| events.nextEvent(); |
| break; |
| case XMLEvent.ATTRIBUTE: |
| throw new IOException("Unexpected XML event " + ev); |
| default: |
| // Ignore other event types like comment, etc. |
| if (LOG.isTraceEnabled()) { |
| LOG.trace("Skipping XMLEvent " + ev); |
| } |
| events.nextEvent(); |
| break; |
| } |
| } catch (XMLStreamException e) { |
| throw new IOException("Expecting " + expected + |
| ", but got XMLStreamException", e); |
| } |
| } |
| } |
| |
| /** |
| * Load a subtree of the XML into a Node structure. |
| * We will keep consuming XML events until we exit the current subtree. |
| * If there are any terminators specified, we will always leave the |
| * terminating end tag event in the stream. |
| * |
| * @param parent The node to fill in. |
| * @param expected A string to display in exceptions. |
| * @param terminators Entering any one of these XML tags terminates our |
| * traversal. |
| * @throws IOException |
| */ |
| private void loadNodeChildren(Node parent, String expected, |
| String... terminators) throws IOException { |
| loadNodeChildrenHelper(parent, expected, terminators); |
| if (LOG.isTraceEnabled()) { |
| LOG.trace("loadNodeChildren(expected=" + expected + |
| ", terminators=[" + StringUtils.join(",", terminators) + |
| "]):" + parent.dump()); |
| } |
| } |
| |
| /** |
| * A processor for an FSImage XML section. |
| */ |
| private interface SectionProcessor { |
| /** |
| * Process this section. |
| */ |
| void process() throws IOException; |
| } |
| |
| /** |
| * Processes the NameSection containing last allocated block ID, etc. |
| */ |
| private class NameSectionProcessor implements SectionProcessor { |
| static final String NAME = "NameSection"; |
| |
| @Override |
| public void process() throws IOException { |
| Node node = new Node(); |
| loadNodeChildren(node, "NameSection fields"); |
| NameSystemSection.Builder b = NameSystemSection.newBuilder(); |
| Integer namespaceId = node.removeChildInt(NAME_SECTION_NAMESPACE_ID); |
| if (namespaceId == null) { |
| throw new IOException("<NameSection> is missing <namespaceId>"); |
| } |
| b.setNamespaceId(namespaceId); |
| Long lval = node.removeChildLong(NAME_SECTION_GENSTAMPV1); |
| if (lval != null) { |
| b.setGenstampV1(lval); |
| } |
| lval = node.removeChildLong(NAME_SECTION_GENSTAMPV2); |
| if (lval != null) { |
| b.setGenstampV2(lval); |
| } |
| lval = node.removeChildLong(NAME_SECTION_GENSTAMPV1_LIMIT); |
| if (lval != null) { |
| b.setGenstampV1Limit(lval); |
| } |
| lval = node.removeChildLong(NAME_SECTION_LAST_ALLOCATED_BLOCK_ID); |
| if (lval != null) { |
| b.setLastAllocatedBlockId(lval); |
| } |
| lval = node.removeChildLong(NAME_SECTION_TXID); |
| if (lval != null) { |
| b.setTransactionId(lval); |
| } |
| lval = node.removeChildLong( |
| NAME_SECTION_ROLLING_UPGRADE_START_TIME); |
| if (lval != null) { |
| b.setRollingUpgradeStartTime(lval); |
| } |
| lval = node.removeChildLong( |
| NAME_SECTION_LAST_ALLOCATED_STRIPED_BLOCK_ID); |
| if (lval != null) { |
| b.setLastAllocatedStripedBlockId(lval); |
| } |
| node.verifyNoRemainingKeys("NameSection"); |
| NameSystemSection s = b.build(); |
| if (LOG.isDebugEnabled()) { |
| LOG.debug(SectionName.NS_INFO.name() + " writing header: {" + |
| TextFormat.printToString(s) + "}"); |
| } |
| s.writeDelimitedTo(out); |
| recordSectionLength(SectionName.NS_INFO.name()); |
| } |
| } |
| |
| private class ErasureCodingSectionProcessor implements SectionProcessor { |
| static final String NAME = "ErasureCodingSection"; |
| |
| @Override |
| public void process() throws IOException { |
| Node node = new Node(); |
| loadNodeChildren(node, "ErasureCodingSection fields"); |
| ErasureCodingSection.Builder builder = ErasureCodingSection.newBuilder(); |
| while (true) { |
| ErasureCodingPolicyProto.Builder policyBuilder = |
| ErasureCodingPolicyProto.newBuilder(); |
| Node ec = node.removeChild(ERASURE_CODING_SECTION_POLICY); |
| if (ec == null) { |
| break; |
| } |
| int policyId = ec.removeChildInt(ERASURE_CODING_SECTION_POLICY_ID); |
| policyBuilder.setId(policyId); |
| String name = ec.removeChildStr(ERASURE_CODING_SECTION_POLICY_NAME); |
| policyBuilder.setName(name); |
| Integer cellSize = |
| ec.removeChildInt(ERASURE_CODING_SECTION_POLICY_CELL_SIZE); |
| policyBuilder.setCellSize(cellSize); |
| String policyState = |
| ec.removeChildStr(ERASURE_CODING_SECTION_POLICY_STATE); |
| if (policyState != null) { |
| policyBuilder.setState( |
| HdfsProtos.ErasureCodingPolicyState.valueOf(policyState)); |
| } |
| |
| Node schema = ec.removeChild(ERASURE_CODING_SECTION_SCHEMA); |
| Preconditions.checkNotNull(schema); |
| |
| ECSchemaProto.Builder schemaBuilder = ECSchemaProto.newBuilder(); |
| String codecName = |
| schema.removeChildStr(ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME); |
| schemaBuilder.setCodecName(codecName); |
| Integer dataUnits = |
| schema.removeChildInt(ERASURE_CODING_SECTION_SCHEMA_DATA_UNITS); |
| schemaBuilder.setDataUnits(dataUnits); |
| Integer parityUnits = schema. |
| removeChildInt(ERASURE_CODING_SECTION_SCHEMA_PARITY_UNITS); |
| schemaBuilder.setParityUnits(parityUnits); |
| Node options = schema |
| .removeChild(ERASURE_CODING_SECTION_SCHEMA_OPTIONS); |
| if (options != null) { |
| while (true) { |
| Node option = |
| options.removeChild(ERASURE_CODING_SECTION_SCHEMA_OPTION); |
| if (option == null) { |
| break; |
| } |
| String key = option |
| .removeChildStr(ERASURE_CODING_SECTION_SCHEMA_OPTION_KEY); |
| String value = option |
| .removeChildStr(ERASURE_CODING_SECTION_SCHEMA_OPTION_VALUE); |
| schemaBuilder.addOptions(HdfsProtos.ECSchemaOptionEntryProto |
| .newBuilder().setKey(key).setValue(value).build()); |
| } |
| } |
| policyBuilder.setSchema(schemaBuilder.build()); |
| |
| builder.addPolicies(policyBuilder.build()); |
| } |
| ErasureCodingSection section = builder.build(); |
| section.writeDelimitedTo(out); |
| node.verifyNoRemainingKeys("ErasureCodingSection"); |
| recordSectionLength(SectionName.ERASURE_CODING.name()); |
| } |
| } |
| |
| private class INodeSectionProcessor implements SectionProcessor { |
| static final String NAME = "INodeSection"; |
| |
| @Override |
| public void process() throws IOException { |
| Node headerNode = new Node(); |
| loadNodeChildren(headerNode, "INodeSection fields", "inode"); |
| INodeSection.Builder b = INodeSection.newBuilder(); |
| Long lval = headerNode.removeChildLong(INODE_SECTION_LAST_INODE_ID); |
| if (lval != null) { |
| b.setLastInodeId(lval); |
| } |
| Integer expectedNumINodes = |
| headerNode.removeChildInt(INODE_SECTION_NUM_INODES); |
| if (expectedNumINodes == null) { |
| throw new IOException("Failed to find <numInodes> in INodeSection."); |
| } |
| b.setNumInodes(expectedNumINodes); |
| INodeSection s = b.build(); |
| s.writeDelimitedTo(out); |
| headerNode.verifyNoRemainingKeys("INodeSection"); |
| int actualNumINodes = 0; |
| while (actualNumINodes < expectedNumINodes) { |
| try { |
| expectTag(INODE_SECTION_INODE, false); |
| } catch (IOException e) { |
| throw new IOException("Only found " + actualNumINodes + |
| " <inode> entries out of " + expectedNumINodes, e); |
| } |
| actualNumINodes++; |
| Node inode = new Node(); |
| loadNodeChildren(inode, "INode fields"); |
| INodeSection.INode.Builder inodeBld = processINodeXml(inode); |
| inodeBld.build().writeDelimitedTo(out); |
| } |
| expectTagEnd(INODE_SECTION_NAME); |
| recordSectionLength(SectionName.INODE.name()); |
| } |
| } |
| |
| private INodeSection.INode.Builder processINodeXml(Node node) |
| throws IOException { |
| String type = node.removeChildStr(INODE_SECTION_TYPE); |
| if (type == null) { |
| throw new IOException("INode XML found with no <type> tag."); |
| } |
| INodeSection.INode.Builder inodeBld = INodeSection.INode.newBuilder(); |
| Long id = node.removeChildLong(SECTION_ID); |
| if (id == null) { |
| throw new IOException("<inode> found without <id>"); |
| } |
| inodeBld.setId(id); |
| String name = node.removeChildStr(SECTION_NAME); |
| if (name != null) { |
| inodeBld.setName(ByteString.copyFrom(name, "UTF8")); |
| } |
| switch (type) { |
| case "FILE": |
| processFileXml(node, inodeBld); |
| break; |
| case "DIRECTORY": |
| processDirectoryXml(node, inodeBld); |
| break; |
| case "SYMLINK": |
| processSymlinkXml(node, inodeBld); |
| break; |
| default: |
| throw new IOException("INode XML found with unknown <type> " + |
| "tag " + type); |
| } |
| node.verifyNoRemainingKeys("inode"); |
| return inodeBld; |
| } |
| |
| private void processFileXml(Node node, INodeSection.INode.Builder inodeBld) |
| throws IOException { |
| inodeBld.setType(INodeSection.INode.Type.FILE); |
| INodeSection.INodeFile.Builder bld = createINodeFileBuilder(node); |
| inodeBld.setFile(bld); |
| // Will check remaining keys and serialize in processINodeXml |
| } |
| |
| private INodeSection.INodeFile.Builder createINodeFileBuilder(Node node) |
| throws IOException { |
| INodeSection.INodeFile.Builder bld = INodeSection.INodeFile.newBuilder(); |
| Integer ival = node.removeChildInt(SECTION_REPLICATION); |
| if (ival != null) { |
| bld.setReplication(ival); |
| } |
| Long lval = node.removeChildLong(INODE_SECTION_MTIME); |
| if (lval != null) { |
| bld.setModificationTime(lval); |
| } |
| lval = node.removeChildLong(INODE_SECTION_ATIME); |
| if (lval != null) { |
| bld.setAccessTime(lval); |
| } |
| lval = node.removeChildLong(INODE_SECTION_PREFERRED_BLOCK_SIZE); |
| if (lval != null) { |
| bld.setPreferredBlockSize(lval); |
| } |
| String perm = node.removeChildStr(INODE_SECTION_PERMISSION); |
| if (perm != null) { |
| bld.setPermission(permissionXmlToU64(perm)); |
| } |
| Node blocks = node.removeChild(INODE_SECTION_BLOCKS); |
| if (blocks != null) { |
| while (true) { |
| Node block = blocks.removeChild(INODE_SECTION_BLOCK); |
| if (block == null) { |
| break; |
| } |
| bld.addBlocks(createBlockBuilder(block)); |
| } |
| } |
| Node fileUnderConstruction = |
| node.removeChild(INODE_SECTION_FILE_UNDER_CONSTRUCTION); |
| if (fileUnderConstruction != null) { |
| INodeSection.FileUnderConstructionFeature.Builder fb = |
| INodeSection.FileUnderConstructionFeature.newBuilder(); |
| String clientName = |
| fileUnderConstruction.removeChildStr(INODE_SECTION_CLIENT_NAME); |
| if (clientName == null) { |
| throw new IOException("<file-under-construction> found without " + |
| "<clientName>"); |
| } |
| fb.setClientName(clientName); |
| String clientMachine = |
| fileUnderConstruction |
| .removeChildStr(INODE_SECTION_CLIENT_MACHINE); |
| if (clientMachine == null) { |
| throw new IOException("<file-under-construction> found without " + |
| "<clientMachine>"); |
| } |
| fb.setClientMachine(clientMachine); |
| bld.setFileUC(fb); |
| } |
| Node acls = node.removeChild(INODE_SECTION_ACLS); |
| if (acls != null) { |
| bld.setAcl(aclXmlToProto(acls)); |
| } |
| Node xattrs = node.removeChild(INODE_SECTION_XATTRS); |
| if (xattrs != null) { |
| bld.setXAttrs(xattrsXmlToProto(xattrs)); |
| } |
| ival = node.removeChildInt(INODE_SECTION_STORAGE_POLICY_ID); |
| if (ival != null) { |
| bld.setStoragePolicyID(ival); |
| } |
| String blockType = node.removeChildStr(INODE_SECTION_BLOCK_TYPE); |
| if(blockType != null) { |
| switch (blockType) { |
| case "CONTIGUOUS": |
| bld.setBlockType(HdfsProtos.BlockTypeProto.CONTIGUOUS); |
| break; |
| case "STRIPED": |
| bld.setBlockType(HdfsProtos.BlockTypeProto.STRIPED); |
| ival = node.removeChildInt(INODE_SECTION_EC_POLICY_ID); |
| if (ival != null) { |
| bld.setErasureCodingPolicyID(ival); |
| } |
| break; |
| default: |
| throw new IOException("INode XML found with unknown <blocktype> " + |
| blockType); |
| } |
| } |
| return bld; |
| } |
| |
| private HdfsProtos.BlockProto.Builder createBlockBuilder(Node block) |
| throws IOException { |
| HdfsProtos.BlockProto.Builder blockBld = |
| HdfsProtos.BlockProto.newBuilder(); |
| Long id = block.removeChildLong(SECTION_ID); |
| if (id == null) { |
| throw new IOException("<block> found without <id>"); |
| } |
| blockBld.setBlockId(id); |
| Long genstamp = block.removeChildLong(INODE_SECTION_GENSTAMP); |
| if (genstamp == null) { |
| throw new IOException("<block> found without <genstamp>"); |
| } |
| blockBld.setGenStamp(genstamp); |
| Long numBytes = block.removeChildLong(INODE_SECTION_NUM_BYTES); |
| if (numBytes == null) { |
| throw new IOException("<block> found without <numBytes>"); |
| } |
| blockBld.setNumBytes(numBytes); |
| return blockBld; |
| } |
| |
| private void processDirectoryXml(Node node, |
| INodeSection.INode.Builder inodeBld) throws IOException { |
| inodeBld.setType(INodeSection.INode.Type.DIRECTORY); |
| INodeSection.INodeDirectory.Builder bld = |
| createINodeDirectoryBuilder(node); |
| inodeBld.setDirectory(bld); |
| // Will check remaining keys and serialize in processINodeXml |
| } |
| |
| private INodeSection.INodeDirectory.Builder |
| createINodeDirectoryBuilder(Node node) throws IOException { |
| INodeSection.INodeDirectory.Builder bld = |
| INodeSection.INodeDirectory.newBuilder(); |
| Long lval = node.removeChildLong(INODE_SECTION_MTIME); |
| if (lval != null) { |
| bld.setModificationTime(lval); |
| } |
| lval = node.removeChildLong(INODE_SECTION_NS_QUOTA); |
| if (lval != null) { |
| bld.setNsQuota(lval); |
| } |
| lval = node.removeChildLong(INODE_SECTION_DS_QUOTA); |
| if (lval != null) { |
| bld.setDsQuota(lval); |
| } |
| String perm = node.removeChildStr(INODE_SECTION_PERMISSION); |
| if (perm != null) { |
| bld.setPermission(permissionXmlToU64(perm)); |
| } |
| Node acls = node.removeChild(INODE_SECTION_ACLS); |
| if (acls != null) { |
| bld.setAcl(aclXmlToProto(acls)); |
| } |
| Node xattrs = node.removeChild(INODE_SECTION_XATTRS); |
| if (xattrs != null) { |
| bld.setXAttrs(xattrsXmlToProto(xattrs)); |
| } |
| INodeSection.QuotaByStorageTypeFeatureProto.Builder qf = |
| INodeSection.QuotaByStorageTypeFeatureProto.newBuilder(); |
| while (true) { |
| Node typeQuota = node.removeChild(INODE_SECTION_TYPE_QUOTA); |
| if (typeQuota == null) { |
| break; |
| } |
| INodeSection.QuotaByStorageTypeEntryProto.Builder qbld = |
| INodeSection.QuotaByStorageTypeEntryProto.newBuilder(); |
| String type = typeQuota.removeChildStr(INODE_SECTION_TYPE); |
| if (type == null) { |
| throw new IOException("<typeQuota> was missing <type>"); |
| } |
| HdfsProtos.StorageTypeProto storageType = |
| HdfsProtos.StorageTypeProto.valueOf(type); |
| if (storageType == null) { |
| throw new IOException("<typeQuota> had unknown <type> " + type); |
| } |
| qbld.setStorageType(storageType); |
| Long quota = typeQuota.removeChildLong(INODE_SECTION_QUOTA); |
| if (quota == null) { |
| throw new IOException("<typeQuota> was missing <quota>"); |
| } |
| qbld.setQuota(quota); |
| qf.addQuotas(qbld); |
| } |
| bld.setTypeQuotas(qf); |
| return bld; |
| } |
| |
| private void processSymlinkXml(Node node, |
| INodeSection.INode.Builder inodeBld) throws IOException { |
| inodeBld.setType(INodeSection.INode.Type.SYMLINK); |
| INodeSection.INodeSymlink.Builder bld = |
| INodeSection.INodeSymlink.newBuilder(); |
| String perm = node.removeChildStr(INODE_SECTION_PERMISSION); |
| if (perm != null) { |
| bld.setPermission(permissionXmlToU64(perm)); |
| } |
| String target = node.removeChildStr(INODE_SECTION_TARGET); |
| if (target != null) { |
| bld.setTarget(ByteString.copyFrom(target, "UTF8")); |
| } |
| Long lval = node.removeChildLong(INODE_SECTION_MTIME); |
| if (lval != null) { |
| bld.setModificationTime(lval); |
| } |
| lval = node.removeChildLong(INODE_SECTION_ATIME); |
| if (lval != null) { |
| bld.setAccessTime(lval); |
| } |
| inodeBld.setSymlink(bld); |
| // Will check remaining keys and serialize in processINodeXml |
| } |
| |
| private INodeSection.AclFeatureProto.Builder aclXmlToProto(Node acls) |
| throws IOException { |
| AclFeatureProto.Builder b = AclFeatureProto.newBuilder(); |
| while (true) { |
| Node acl = acls.removeChild(INODE_SECTION_ACL); |
| if (acl == null) { |
| break; |
| } |
| String val = acl.getVal(); |
| AclEntry entry = AclEntry.parseAclEntry(val, true); |
| int nameId = registerStringId(entry.getName() == null ? EMPTY_STRING |
| : entry.getName()); |
| int v = ((nameId & ACL_ENTRY_NAME_MASK) << ACL_ENTRY_NAME_OFFSET) |
| | (entry.getType().ordinal() << ACL_ENTRY_TYPE_OFFSET) |
| | (entry.getScope().ordinal() << ACL_ENTRY_SCOPE_OFFSET) |
| | (entry.getPermission().ordinal()); |
| b.addEntries(v); |
| } |
| return b; |
| } |
| |
| private INodeSection.XAttrFeatureProto.Builder xattrsXmlToProto(Node xattrs) |
| throws IOException { |
| INodeSection.XAttrFeatureProto.Builder bld = |
| INodeSection.XAttrFeatureProto.newBuilder(); |
| while (true) { |
| Node xattr = xattrs.removeChild(INODE_SECTION_XATTR); |
| if (xattr == null) { |
| break; |
| } |
| INodeSection.XAttrCompactProto.Builder b = |
| INodeSection.XAttrCompactProto.newBuilder(); |
| String ns = xattr.removeChildStr(INODE_SECTION_NS); |
| if (ns == null) { |
| throw new IOException("<xattr> had no <ns> entry."); |
| } |
| int nsIdx = XAttrProtos.XAttrProto. |
| XAttrNamespaceProto.valueOf(ns).ordinal(); |
| String name = xattr.removeChildStr(SECTION_NAME); |
| String valStr = xattr.removeChildStr(INODE_SECTION_VAL); |
| byte[] val = null; |
| if (valStr == null) { |
| String valHex = xattr.removeChildStr(INODE_SECTION_VAL_HEX); |
| if (valHex == null) { |
| throw new IOException("<xattr> had no <val> or <valHex> entry."); |
| } |
| val = new HexBinaryAdapter().unmarshal(valHex); |
| } else { |
| val = valStr.getBytes("UTF8"); |
| } |
| b.setValue(ByteString.copyFrom(val)); |
| |
| // The XAttrCompactProto name field uses a fairly complex format |
| // to encode both the string table ID of the xattr name and the |
| // namespace ID. See the protobuf file for details. |
| int nameId = registerStringId(name); |
| int encodedName = (nameId << XATTR_NAME_OFFSET) | |
| ((nsIdx & XATTR_NAMESPACE_MASK) << XATTR_NAMESPACE_OFFSET) | |
| (((nsIdx >> 2) & XATTR_NAMESPACE_EXT_MASK) |
| << XATTR_NAMESPACE_EXT_OFFSET); |
| b.setName(encodedName); |
| xattr.verifyNoRemainingKeys("xattr"); |
| bld.addXAttrs(b); |
| } |
| xattrs.verifyNoRemainingKeys("xattrs"); |
| return bld; |
| } |
| |
| private class SecretManagerSectionProcessor implements SectionProcessor { |
| static final String NAME = "SecretManagerSection"; |
| |
| @Override |
| public void process() throws IOException { |
| Node secretHeader = new Node(); |
| loadNodeChildren(secretHeader, "SecretManager fields", |
| "delegationKey", "token"); |
| SecretManagerSection.Builder b = SecretManagerSection.newBuilder(); |
| Integer currentId = |
| secretHeader.removeChildInt(SECRET_MANAGER_SECTION_CURRENT_ID); |
| if (currentId == null) { |
| throw new IOException("SecretManager section had no <currentId>"); |
| } |
| b.setCurrentId(currentId); |
| Integer tokenSequenceNumber = secretHeader.removeChildInt( |
| SECRET_MANAGER_SECTION_TOKEN_SEQUENCE_NUMBER); |
| if (tokenSequenceNumber == null) { |
| throw new IOException("SecretManager section had no " + |
| "<tokenSequenceNumber>"); |
| } |
| b.setTokenSequenceNumber(tokenSequenceNumber); |
| Integer expectedNumKeys = secretHeader.removeChildInt( |
| SECRET_MANAGER_SECTION_NUM_DELEGATION_KEYS); |
| if (expectedNumKeys == null) { |
| throw new IOException("SecretManager section had no " + |
| "<numDelegationKeys>"); |
| } |
| b.setNumKeys(expectedNumKeys); |
| Integer expectedNumTokens = |
| secretHeader.removeChildInt(SECRET_MANAGER_SECTION_NUM_TOKENS); |
| if (expectedNumTokens == null) { |
| throw new IOException("SecretManager section had no " + |
| "<numTokens>"); |
| } |
| b.setNumTokens(expectedNumTokens); |
| secretHeader.verifyNoRemainingKeys("SecretManager"); |
| b.build().writeDelimitedTo(out); |
| for (int actualNumKeys = 0; actualNumKeys < expectedNumKeys; |
| actualNumKeys++) { |
| try { |
| expectTag(SECRET_MANAGER_SECTION_DELEGATION_KEY, false); |
| } catch (IOException e) { |
| throw new IOException("Only read " + actualNumKeys + |
| " delegation keys out of " + expectedNumKeys, e); |
| } |
| SecretManagerSection.DelegationKey.Builder dbld = |
| SecretManagerSection.DelegationKey.newBuilder(); |
| Node dkey = new Node(); |
| loadNodeChildren(dkey, "Delegation key fields"); |
| Integer id = dkey.removeChildInt(SECTION_ID); |
| if (id == null) { |
| throw new IOException("Delegation key stanza <delegationKey> " + |
| "lacked an <id> field."); |
| } |
| dbld.setId(id); |
| String expiry = dkey.removeChildStr(SECRET_MANAGER_SECTION_EXPIRY); |
| if (expiry == null) { |
| throw new IOException("Delegation key stanza <delegationKey> " + |
| "lacked an <expiry> field."); |
| } |
| dbld.setExpiryDate(dateStrToLong(expiry)); |
| String keyHex = dkey.removeChildStr(SECRET_MANAGER_SECTION_KEY); |
| if (keyHex == null) { |
| throw new IOException("Delegation key stanza <delegationKey> " + |
| "lacked a <key> field."); |
| } |
| byte[] key = new HexBinaryAdapter().unmarshal(keyHex); |
| dkey.verifyNoRemainingKeys(SECRET_MANAGER_SECTION_DELEGATION_KEY); |
| dbld.setKey(ByteString.copyFrom(key)); |
| dbld.build().writeDelimitedTo(out); |
| } |
| for (int actualNumTokens = 0; actualNumTokens < expectedNumTokens; |
| actualNumTokens++) { |
| try { |
| expectTag(SECRET_MANAGER_SECTION_TOKEN, false); |
| } catch (IOException e) { |
| throw new IOException("Only read " + actualNumTokens + |
| " tokens out of " + expectedNumTokens, e); |
| } |
| SecretManagerSection.PersistToken.Builder tbld = |
| SecretManagerSection.PersistToken.newBuilder(); |
| Node token = new Node(); |
| loadNodeChildren(token, "PersistToken key fields"); |
| Integer version = |
| token.removeChildInt(SECRET_MANAGER_SECTION_VERSION); |
| if (version != null) { |
| tbld.setVersion(version); |
| } |
| String owner = token.removeChildStr(SECRET_MANAGER_SECTION_OWNER); |
| if (owner != null) { |
| tbld.setOwner(owner); |
| } |
| String renewer = |
| token.removeChildStr(SECRET_MANAGER_SECTION_RENEWER); |
| if (renewer != null) { |
| tbld.setRenewer(renewer); |
| } |
| String realUser = |
| token.removeChildStr(SECRET_MANAGER_SECTION_REAL_USER); |
| if (realUser != null) { |
| tbld.setRealUser(realUser); |
| } |
| String issueDateStr = |
| token.removeChildStr(SECRET_MANAGER_SECTION_ISSUE_DATE); |
| if (issueDateStr != null) { |
| tbld.setIssueDate(dateStrToLong(issueDateStr)); |
| } |
| String maxDateStr = |
| token.removeChildStr(SECRET_MANAGER_SECTION_MAX_DATE); |
| if (maxDateStr != null) { |
| tbld.setMaxDate(dateStrToLong(maxDateStr)); |
| } |
| Integer seqNo = |
| token.removeChildInt(SECRET_MANAGER_SECTION_SEQUENCE_NUMBER); |
| if (seqNo != null) { |
| tbld.setSequenceNumber(seqNo); |
| } |
| Integer masterKeyId = |
| token.removeChildInt(SECRET_MANAGER_SECTION_MASTER_KEY_ID); |
| if (masterKeyId != null) { |
| tbld.setMasterKeyId(masterKeyId); |
| } |
| String expiryDateStr = |
| token.removeChildStr(SECRET_MANAGER_SECTION_EXPIRY_DATE); |
| if (expiryDateStr != null) { |
| tbld.setExpiryDate(dateStrToLong(expiryDateStr)); |
| } |
| token.verifyNoRemainingKeys("token"); |
| tbld.build().writeDelimitedTo(out); |
| } |
| expectTagEnd(SECRET_MANAGER_SECTION_NAME); |
| recordSectionLength(SectionName.SECRET_MANAGER.name()); |
| } |
| |
| private long dateStrToLong(String dateStr) throws IOException { |
| try { |
| Date date = isoDateFormat.parse(dateStr); |
| return date.getTime(); |
| } catch (ParseException e) { |
| throw new IOException("Failed to parse ISO date string " + dateStr, e); |
| } |
| } |
| } |
| |
| private class CacheManagerSectionProcessor implements SectionProcessor { |
| static final String NAME = "CacheManagerSection"; |
| |
| @Override |
| public void process() throws IOException { |
| Node node = new Node(); |
| loadNodeChildren(node, "CacheManager fields", "pool", "directive"); |
| CacheManagerSection.Builder b = CacheManagerSection.newBuilder(); |
| Long nextDirectiveId = |
| node.removeChildLong(CACHE_MANAGER_SECTION_NEXT_DIRECTIVE_ID); |
| if (nextDirectiveId == null) { |
| throw new IOException("CacheManager section had no <nextDirectiveId>"); |
| } |
| b.setNextDirectiveId(nextDirectiveId); |
| Integer expectedNumPools = |
| node.removeChildInt(CACHE_MANAGER_SECTION_NUM_POOLS); |
| if (expectedNumPools == null) { |
| throw new IOException("CacheManager section had no <numPools>"); |
| } |
| b.setNumPools(expectedNumPools); |
| Integer expectedNumDirectives = |
| node.removeChildInt(CACHE_MANAGER_SECTION_NUM_DIRECTIVES); |
| if (expectedNumDirectives == null) { |
| throw new IOException("CacheManager section had no <numDirectives>"); |
| } |
| b.setNumDirectives(expectedNumDirectives); |
| b.build().writeDelimitedTo(out); |
| long actualNumPools = 0; |
| while (actualNumPools < expectedNumPools) { |
| try { |
| expectTag(CACHE_MANAGER_SECTION_POOL, false); |
| } catch (IOException e) { |
| throw new IOException("Only read " + actualNumPools + |
| " cache pools out of " + expectedNumPools, e); |
| } |
| actualNumPools++; |
| Node pool = new Node(); |
| loadNodeChildren(pool, "pool fields", ""); |
| processPoolXml(node); |
| } |
| long actualNumDirectives = 0; |
| while (actualNumDirectives < expectedNumDirectives) { |
| try { |
| expectTag(CACHE_MANAGER_SECTION_DIRECTIVE, false); |
| } catch (IOException e) { |
| throw new IOException("Only read " + actualNumDirectives + |
| " cache pools out of " + expectedNumDirectives, e); |
| } |
| actualNumDirectives++; |
| Node pool = new Node(); |
| loadNodeChildren(pool, "directive fields", ""); |
| processDirectiveXml(node); |
| } |
| expectTagEnd(CACHE_MANAGER_SECTION_NAME); |
| recordSectionLength(SectionName.CACHE_MANAGER.name()); |
| } |
| |
| private void processPoolXml(Node pool) throws IOException { |
| CachePoolInfoProto.Builder bld = CachePoolInfoProto.newBuilder(); |
| String poolName = |
| pool.removeChildStr(CACHE_MANAGER_SECTION_POOL_NAME); |
| if (poolName == null) { |
| throw new IOException("<pool> found without <poolName>"); |
| } |
| bld.setPoolName(poolName); |
| String ownerName = |
| pool.removeChildStr(CACHE_MANAGER_SECTION_OWNER_NAME); |
| if (ownerName == null) { |
| throw new IOException("<pool> found without <ownerName>"); |
| } |
| bld.setOwnerName(ownerName); |
| String groupName = |
| pool.removeChildStr(CACHE_MANAGER_SECTION_GROUP_NAME); |
| if (groupName == null) { |
| throw new IOException("<pool> found without <groupName>"); |
| } |
| bld.setGroupName(groupName); |
| Integer mode = pool.removeChildInt(CACHE_MANAGER_SECTION_MODE); |
| if (mode == null) { |
| throw new IOException("<pool> found without <mode>"); |
| } |
| bld.setMode(mode); |
| Long limit = pool.removeChildLong(CACHE_MANAGER_SECTION_LIMIT); |
| if (limit == null) { |
| throw new IOException("<pool> found without <limit>"); |
| } |
| bld.setLimit(limit); |
| Long maxRelativeExpiry = |
| pool.removeChildLong(CACHE_MANAGER_SECTION_MAX_RELATIVE_EXPIRY); |
| if (maxRelativeExpiry == null) { |
| throw new IOException("<pool> found without <maxRelativeExpiry>"); |
| } |
| bld.setMaxRelativeExpiry(maxRelativeExpiry); |
| pool.verifyNoRemainingKeys("pool"); |
| bld.build().writeDelimitedTo(out); |
| } |
| |
| private void processDirectiveXml(Node directive) throws IOException { |
| CacheDirectiveInfoProto.Builder bld = |
| CacheDirectiveInfoProto.newBuilder(); |
| Long id = directive.removeChildLong(SECTION_ID); |
| if (id == null) { |
| throw new IOException("<directive> found without <id>"); |
| } |
| bld.setId(id); |
| String path = directive.removeChildStr(SECTION_PATH); |
| if (path == null) { |
| throw new IOException("<directive> found without <path>"); |
| } |
| bld.setPath(path); |
| Integer replication = directive.removeChildInt(SECTION_REPLICATION); |
| if (replication == null) { |
| throw new IOException("<directive> found without <replication>"); |
| } |
| bld.setReplication(replication); |
| String pool = directive.removeChildStr(CACHE_MANAGER_SECTION_POOL); |
| if (path == null) { |
| throw new IOException("<directive> found without <pool>"); |
| } |
| bld.setPool(pool); |
| Node expiration = |
| directive.removeChild(CACHE_MANAGER_SECTION_EXPIRATION); |
| if (expiration != null) { |
| CacheDirectiveInfoExpirationProto.Builder ebld = |
| CacheDirectiveInfoExpirationProto.newBuilder(); |
| Long millis = |
| expiration.removeChildLong(CACHE_MANAGER_SECTION_MILLIS); |
| if (millis == null) { |
| throw new IOException("cache directive <expiration> found " + |
| "without <millis>"); |
| } |
| ebld.setMillis(millis); |
| if (expiration.removeChildBool(CACHE_MANAGER_SECTION_RELATIVE)) { |
| ebld.setIsRelative(true); |
| } else { |
| ebld.setIsRelative(false); |
| } |
| bld.setExpiration(ebld); |
| } |
| directive.verifyNoRemainingKeys("directive"); |
| bld.build().writeDelimitedTo(out); |
| } |
| } |
| |
| private class INodeReferenceSectionProcessor implements SectionProcessor { |
| static final String NAME = "INodeReferenceSection"; |
| |
| @Override |
| public void process() throws IOException { |
| // There is no header for this section. |
| // We process the repeated <ref> elements. |
| while (true) { |
| XMLEvent ev = expectTag(INODE_REFERENCE_SECTION_REF, true); |
| if (ev.isEndElement()) { |
| break; |
| } |
| Node inodeRef = new Node(); |
| FsImageProto.INodeReferenceSection.INodeReference.Builder bld = |
| FsImageProto.INodeReferenceSection.INodeReference.newBuilder(); |
| loadNodeChildren(inodeRef, "INodeReference"); |
| Long referredId = |
| inodeRef.removeChildLong(INODE_REFERENCE_SECTION_REFERRED_ID); |
| if (referredId != null) { |
| bld.setReferredId(referredId); |
| } |
| String name = inodeRef.removeChildStr("name"); |
| if (name != null) { |
| bld.setName(ByteString.copyFrom(name, "UTF8")); |
| } |
| Integer dstSnapshotId = inodeRef.removeChildInt( |
| INODE_REFERENCE_SECTION_DST_SNAPSHOT_ID); |
| if (dstSnapshotId != null) { |
| bld.setDstSnapshotId(dstSnapshotId); |
| } |
| Integer lastSnapshotId = inodeRef.removeChildInt( |
| INODE_REFERENCE_SECTION_LAST_SNAPSHOT_ID); |
| if (lastSnapshotId != null) { |
| bld.setLastSnapshotId(lastSnapshotId); |
| } |
| inodeRef.verifyNoRemainingKeys("ref"); |
| bld.build().writeDelimitedTo(out); |
| } |
| recordSectionLength(SectionName.INODE_REFERENCE.name()); |
| } |
| } |
| |
| private class INodeDirectorySectionProcessor implements SectionProcessor { |
| static final String NAME = "INodeDirectorySection"; |
| |
| @Override |
| public void process() throws IOException { |
| // No header for this section |
| // Process the repeated <directory> elements. |
| while (true) { |
| XMLEvent ev = expectTag(INODE_DIRECTORY_SECTION_DIRECTORY, true); |
| if (ev.isEndElement()) { |
| break; |
| } |
| Node directory = new Node(); |
| FsImageProto.INodeDirectorySection.DirEntry.Builder bld = |
| FsImageProto.INodeDirectorySection.DirEntry.newBuilder(); |
| loadNodeChildren(directory, "directory"); |
| Long parent = directory.removeChildLong( |
| INODE_DIRECTORY_SECTION_PARENT); |
| if (parent != null) { |
| bld.setParent(parent); |
| } |
| while (true) { |
| Node child = directory.removeChild( |
| INODE_DIRECTORY_SECTION_CHILD); |
| if (child == null) { |
| break; |
| } |
| bld.addChildren(Long.parseLong(child.getVal())); |
| } |
| while (true) { |
| Node refChild = directory.removeChild( |
| INODE_DIRECTORY_SECTION_REF_CHILD); |
| if (refChild == null) { |
| break; |
| } |
| bld.addRefChildren(Integer.parseInt(refChild.getVal())); |
| } |
| directory.verifyNoRemainingKeys("directory"); |
| bld.build().writeDelimitedTo(out); |
| } |
| recordSectionLength(SectionName.INODE_DIR.name()); |
| } |
| } |
| |
| private class FilesUnderConstructionSectionProcessor |
| implements SectionProcessor { |
| static final String NAME = "FileUnderConstructionSection"; |
| |
| @Override |
| public void process() throws IOException { |
| // No header for this section type. |
| // Process the repeated files under construction elements. |
| while (true) { |
| XMLEvent ev = expectTag(INODE_SECTION_INODE, true); |
| if (ev.isEndElement()) { |
| break; |
| } |
| Node fileUnderConstruction = new Node(); |
| loadNodeChildren(fileUnderConstruction, "file under construction"); |
| FileUnderConstructionEntry.Builder bld = |
| FileUnderConstructionEntry.newBuilder(); |
| Long id = fileUnderConstruction.removeChildLong(SECTION_ID); |
| if (id != null) { |
| bld.setInodeId(id); |
| } |
| String fullpath = |
| fileUnderConstruction.removeChildStr(SECTION_PATH); |
| if (fullpath != null) { |
| bld.setFullPath(fullpath); |
| } |
| fileUnderConstruction.verifyNoRemainingKeys("inode"); |
| bld.build().writeDelimitedTo(out); |
| } |
| recordSectionLength(SectionName.FILES_UNDERCONSTRUCTION.name()); |
| } |
| } |
| |
| private class SnapshotSectionProcessor implements SectionProcessor { |
| static final String NAME = "SnapshotSection"; |
| |
| @Override |
| public void process() throws IOException { |
| FsImageProto.SnapshotSection.Builder bld = |
| FsImageProto.SnapshotSection.newBuilder(); |
| Node header = new Node(); |
| loadNodeChildren(header, "SnapshotSection fields", "snapshot"); |
| Integer snapshotCounter = header.removeChildInt( |
| SNAPSHOT_SECTION_SNAPSHOT_COUNTER); |
| if (snapshotCounter == null) { |
| throw new IOException("No <snapshotCounter> entry found in " + |
| "SnapshotSection header"); |
| } |
| bld.setSnapshotCounter(snapshotCounter); |
| Integer expectedNumSnapshots = header.removeChildInt( |
| SNAPSHOT_SECTION_NUM_SNAPSHOTS); |
| if (expectedNumSnapshots == null) { |
| throw new IOException("No <numSnapshots> entry found in " + |
| "SnapshotSection header"); |
| } |
| bld.setNumSnapshots(expectedNumSnapshots); |
| while (true) { |
| Node sd = header.removeChild(SNAPSHOT_SECTION_SNAPSHOT_TABLE_DIR); |
| if (sd == null) { |
| break; |
| } |
| Long dir = sd.removeChildLong(SNAPSHOT_SECTION_DIR); |
| sd.verifyNoRemainingKeys("<dir>"); |
| bld.addSnapshottableDir(dir); |
| } |
| header.verifyNoRemainingKeys("SnapshotSection"); |
| bld.build().writeDelimitedTo(out); |
| int actualNumSnapshots = 0; |
| while (actualNumSnapshots < expectedNumSnapshots) { |
| try { |
| expectTag(SNAPSHOT_SECTION_SNAPSHOT, false); |
| } catch (IOException e) { |
| throw new IOException("Only read " + actualNumSnapshots + |
| " <snapshot> entries out of " + expectedNumSnapshots, e); |
| } |
| actualNumSnapshots++; |
| Node snapshot = new Node(); |
| loadNodeChildren(snapshot, "snapshot fields"); |
| FsImageProto.SnapshotSection.Snapshot.Builder s = |
| FsImageProto.SnapshotSection.Snapshot.newBuilder(); |
| Integer snapshotId = snapshot.removeChildInt(SECTION_ID); |
| if (snapshotId == null) { |
| throw new IOException("<snapshot> section was missing <id>"); |
| } |
| s.setSnapshotId(snapshotId); |
| Node snapshotRoot = snapshot.removeChild(SNAPSHOT_SECTION_ROOT); |
| INodeSection.INode.Builder inodeBld = processINodeXml(snapshotRoot); |
| s.setRoot(inodeBld); |
| s.build().writeDelimitedTo(out); |
| } |
| expectTagEnd(SNAPSHOT_SECTION_NAME); |
| recordSectionLength(SectionName.SNAPSHOT.name()); |
| } |
| } |
| |
| private class SnapshotDiffSectionProcessor implements SectionProcessor { |
| static final String NAME = "SnapshotDiffSection"; |
| |
| @Override |
| public void process() throws IOException { |
| // No header for this section type. |
| LOG.debug("Processing SnapshotDiffSection"); |
| while (true) { |
| XMLEvent ev = expectTag("[diff start tag]", true); |
| if (ev.isEndElement()) { |
| String name = ev.asEndElement().getName().getLocalPart(); |
| if (name.equals(SNAPSHOT_DIFF_SECTION_NAME)) { |
| break; |
| } |
| throw new IOException("Got unexpected end tag for " + name); |
| } |
| String tagName = ev.asStartElement().getName().getLocalPart(); |
| if (tagName.equals(SNAPSHOT_DIFF_SECTION_DIR_DIFF_ENTRY)) { |
| processDirDiffEntry(); |
| } else if (tagName.equals(SNAPSHOT_DIFF_SECTION_FILE_DIFF_ENTRY)) { |
| processFileDiffEntry(); |
| } else { |
| throw new IOException("SnapshotDiffSection contained unexpected " + |
| "tag " + tagName); |
| } |
| } |
| recordSectionLength(SectionName.SNAPSHOT_DIFF.name()); |
| } |
| |
| private void processDirDiffEntry() throws IOException { |
| LOG.debug("Processing dirDiffEntry"); |
| DiffEntry.Builder headerBld = DiffEntry.newBuilder(); |
| headerBld.setType(DiffEntry.Type.DIRECTORYDIFF); |
| Node dirDiffHeader = new Node(); |
| loadNodeChildren(dirDiffHeader, "dirDiffEntry fields", "dirDiff"); |
| Long inodeId = dirDiffHeader.removeChildLong( |
| SNAPSHOT_DIFF_SECTION_INODE_ID); |
| if (inodeId == null) { |
| throw new IOException("<dirDiffEntry> contained no <inodeId> entry."); |
| } |
| headerBld.setInodeId(inodeId); |
| Integer expectedDiffs = dirDiffHeader.removeChildInt( |
| SNAPSHOT_DIFF_SECTION_COUNT); |
| if (expectedDiffs == null) { |
| throw new IOException("<dirDiffEntry> contained no <count> entry."); |
| } |
| headerBld.setNumOfDiff(expectedDiffs); |
| dirDiffHeader.verifyNoRemainingKeys("dirDiffEntry"); |
| headerBld.build().writeDelimitedTo(out); |
| for (int actualDiffs = 0; actualDiffs < expectedDiffs; actualDiffs++) { |
| try { |
| expectTag(SNAPSHOT_DIFF_SECTION_DIR_DIFF, false); |
| } catch (IOException e) { |
| throw new IOException("Only read " + (actualDiffs + 1) + |
| " diffs out of " + expectedDiffs, e); |
| } |
| Node dirDiff = new Node(); |
| loadNodeChildren(dirDiff, "dirDiff fields"); |
| FsImageProto.SnapshotDiffSection.DirectoryDiff.Builder bld = |
| FsImageProto.SnapshotDiffSection.DirectoryDiff.newBuilder(); |
| Integer snapshotId = dirDiff.removeChildInt( |
| SNAPSHOT_DIFF_SECTION_SNAPSHOT_ID); |
| if (snapshotId != null) { |
| bld.setSnapshotId(snapshotId); |
| } |
| Integer childrenSize = dirDiff.removeChildInt( |
| SNAPSHOT_DIFF_SECTION_CHILDREN_SIZE); |
| if (childrenSize == null) { |
| throw new IOException("Expected to find <childrenSize> in " + |
| "<dirDiff> section."); |
| } |
| bld.setIsSnapshotRoot(dirDiff.removeChildBool( |
| SNAPSHOT_DIFF_SECTION_IS_SNAPSHOT_ROOT)); |
| bld.setChildrenSize(childrenSize); |
| String name = dirDiff.removeChildStr(SECTION_NAME); |
| if (name != null) { |
| bld.setName(ByteString.copyFrom(name, "UTF8")); |
| } |
| Node snapshotCopy = dirDiff.removeChild( |
| SNAPSHOT_DIFF_SECTION_SNAPSHOT_COPY); |
| if (snapshotCopy != null) { |
| bld.setSnapshotCopy(createINodeDirectoryBuilder(snapshotCopy)); |
| } |
| Integer expectedCreatedListSize = dirDiff.removeChildInt( |
| SNAPSHOT_DIFF_SECTION_CREATED_LIST_SIZE); |
| if (expectedCreatedListSize == null) { |
| throw new IOException("Expected to find <createdListSize> in " + |
| "<dirDiff> section."); |
| } |
| bld.setCreatedListSize(expectedCreatedListSize); |
| while (true) { |
| Node deleted = dirDiff.removeChild( |
| SNAPSHOT_DIFF_SECTION_DELETED_INODE); |
| if (deleted == null){ |
| break; |
| } |
| bld.addDeletedINode(Long.parseLong(deleted.getVal())); |
| } |
| while (true) { |
| Node deleted = dirDiff.removeChild( |
| SNAPSHOT_DIFF_SECTION_DELETED_INODE_REF); |
| if (deleted == null){ |
| break; |
| } |
| bld.addDeletedINodeRef(Integer.parseInt(deleted.getVal())); |
| } |
| bld.build().writeDelimitedTo(out); |
| // After the DirectoryDiff header comes a list of CreatedListEntry PBs. |
| int actualCreatedListSize = 0; |
| while (true) { |
| Node created = dirDiff.removeChild( |
| SNAPSHOT_DIFF_SECTION_CREATED); |
| if (created == null){ |
| break; |
| } |
| String cleName = created.removeChildStr(SECTION_NAME); |
| if (cleName == null) { |
| throw new IOException("Expected <created> entry to have " + |
| "a <name> field"); |
| } |
| created.verifyNoRemainingKeys("created"); |
| FsImageProto.SnapshotDiffSection.CreatedListEntry.newBuilder(). |
| setName(ByteString.copyFrom(cleName, "UTF8")). |
| build().writeDelimitedTo(out); |
| actualCreatedListSize++; |
| } |
| if (actualCreatedListSize != expectedCreatedListSize) { |
| throw new IOException("<createdListSize> was " + |
| expectedCreatedListSize +", but there were " + |
| actualCreatedListSize + " <created> entries."); |
| } |
| dirDiff.verifyNoRemainingKeys("dirDiff"); |
| } |
| expectTagEnd(SNAPSHOT_DIFF_SECTION_DIR_DIFF_ENTRY); |
| } |
| |
| private void processFileDiffEntry() throws IOException { |
| LOG.debug("Processing fileDiffEntry"); |
| DiffEntry.Builder headerBld = DiffEntry.newBuilder(); |
| headerBld.setType(DiffEntry.Type.FILEDIFF); |
| Node fileDiffHeader = new Node(); |
| loadNodeChildren(fileDiffHeader, "fileDiffEntry fields", "fileDiff"); |
| Long inodeId = fileDiffHeader.removeChildLong( |
| SNAPSHOT_DIFF_SECTION_INODE_ID); |
| if (inodeId == null) { |
| throw new IOException("<fileDiffEntry> contained no <inodeid> entry."); |
| } |
| headerBld.setInodeId(inodeId); |
| Integer expectedDiffs = fileDiffHeader.removeChildInt( |
| SNAPSHOT_DIFF_SECTION_COUNT); |
| if (expectedDiffs == null) { |
| throw new IOException("<fileDiffEntry> contained no <count> entry."); |
| } |
| headerBld.setNumOfDiff(expectedDiffs); |
| fileDiffHeader.verifyNoRemainingKeys("fileDiffEntry"); |
| headerBld.build().writeDelimitedTo(out); |
| for (int actualDiffs = 0; actualDiffs < expectedDiffs; actualDiffs++) { |
| try { |
| expectTag(SNAPSHOT_DIFF_SECTION_FILE_DIFF, false); |
| } catch (IOException e) { |
| throw new IOException("Only read " + (actualDiffs + 1) + |
| " diffs out of " + expectedDiffs, e); |
| } |
| Node fileDiff = new Node(); |
| loadNodeChildren(fileDiff, "fileDiff fields"); |
| FsImageProto.SnapshotDiffSection.FileDiff.Builder bld = |
| FsImageProto.SnapshotDiffSection.FileDiff.newBuilder(); |
| Integer snapshotId = fileDiff.removeChildInt( |
| SNAPSHOT_DIFF_SECTION_SNAPSHOT_ID); |
| if (snapshotId != null) { |
| bld.setSnapshotId(snapshotId); |
| } |
| Long size = fileDiff.removeChildLong( |
| SNAPSHOT_DIFF_SECTION_SIZE); |
| if (size != null) { |
| bld.setFileSize(size); |
| } |
| String name = fileDiff.removeChildStr(SECTION_NAME); |
| if (name != null) { |
| bld.setName(ByteString.copyFrom(name, "UTF8")); |
| } |
| Node snapshotCopy = fileDiff.removeChild( |
| SNAPSHOT_DIFF_SECTION_SNAPSHOT_COPY); |
| if (snapshotCopy != null) { |
| bld.setSnapshotCopy(createINodeFileBuilder(snapshotCopy)); |
| } |
| Node blocks = fileDiff.removeChild(INODE_SECTION_BLOCKS); |
| if (blocks != null) { |
| while (true) { |
| Node block = blocks.removeChild(INODE_SECTION_BLOCK); |
| if (block == null) { |
| break; |
| } |
| bld.addBlocks(createBlockBuilder(block)); |
| } |
| } |
| fileDiff.verifyNoRemainingKeys("fileDiff"); |
| bld.build().writeDelimitedTo(out); |
| } |
| expectTagEnd(SNAPSHOT_DIFF_SECTION_FILE_DIFF_ENTRY); |
| } |
| } |
| |
| /** |
| * Permission is serialized as a 64-bit long. [0:24):[25:48):[48:64) |
| * (in Big Endian). The first and the second parts are the string ids |
| * of the user and group name, and the last 16 bits are the permission bits. |
| * |
| * @param perm The permission string from the XML. |
| * @return The 64-bit value to use in the fsimage for permission. |
| * @throws IOException If we run out of string IDs in the string table. |
| */ |
| private long permissionXmlToU64(String perm) throws IOException { |
| String components[] = perm.split(":"); |
| if (components.length != 3) { |
| throw new IOException("Unable to parse permission string " + perm + |
| ": expected 3 components, but only had " + components.length); |
| } |
| String userName = components[0]; |
| String groupName = components[1]; |
| String modeString = components[2]; |
| long userNameId = registerStringId(userName); |
| long groupNameId = registerStringId(groupName); |
| long mode = new FsPermission(modeString).toShort(); |
| return (userNameId << 40) | (groupNameId << 16) | mode; |
| } |
| |
| /** |
| * The FSImage contains a string table which maps strings to IDs. |
| * This is a simple form of compression which takes advantage of the fact |
| * that the same strings tend to occur over and over again. |
| * This function will return an ID which we can use to represent the given |
| * string. If the string already exists in the string table, we will use |
| * that ID; otherwise, we will allocate a new one. |
| * |
| * @param str The string. |
| * @return The ID in the string table. |
| * @throws IOException If we run out of bits in the string table. We only |
| * have 25 bits. |
| */ |
| int registerStringId(String str) throws IOException { |
| Integer id = stringTable.get(str); |
| if (id != null) { |
| return id; |
| } |
| int latestId = latestStringId; |
| if (latestId >= 0x1ffffff) { |
| throw new IOException("Cannot have more than 2**25 " + |
| "strings in the fsimage, because of the limitation on " + |
| "the size of string table IDs."); |
| } |
| stringTable.put(str, latestId); |
| latestStringId++; |
| return latestId; |
| } |
| |
| /** |
| * Record the length of a section of the FSImage in our FileSummary object. |
| * The FileSummary appears at the end of the FSImage and acts as a table of |
| * contents for the file. |
| * |
| * @param sectionNamePb The name of the section as it should appear in |
| * the fsimage. (This is different than the XML |
| * name.) |
| * @throws IOException |
| */ |
| void recordSectionLength(String sectionNamePb) throws IOException { |
| long curSectionStartOffset = sectionStartOffset; |
| long curPos = out.getCount(); |
| //if (sectionNamePb.equals(SectionName.STRING_TABLE.name())) { |
| fileSummaryBld.addSections(FileSummary.Section.newBuilder(). |
| setName(sectionNamePb). |
| setLength(curPos - curSectionStartOffset). |
| setOffset(curSectionStartOffset)); |
| //} |
| sectionStartOffset = curPos; |
| } |
| |
| /** |
| * Read the version tag which starts the XML file. |
| */ |
| private void readVersion() throws IOException { |
| try { |
| expectTag("version", false); |
| } catch (IOException e) { |
| // Handle the case where <version> does not exist. |
| // Note: fsimage XML files which are missing <version> are also missing |
| // many other fields that oiv needs to accurately reconstruct the |
| // fsimage. |
| throw new IOException("No <version> section found at the top of " + |
| "the fsimage XML. This XML file is too old to be processed " + |
| "by oiv.", e); |
| } |
| Node version = new Node(); |
| loadNodeChildren(version, "version fields"); |
| Integer onDiskVersion = version.removeChildInt("onDiskVersion"); |
| if (onDiskVersion == null) { |
| throw new IOException("The <version> section doesn't contain " + |
| "the onDiskVersion."); |
| } |
| Integer layoutVersion = version.removeChildInt("layoutVersion"); |
| if (layoutVersion == null) { |
| throw new IOException("The <version> section doesn't contain " + |
| "the layoutVersion."); |
| } |
| if (layoutVersion.intValue() != |
| NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION) { |
| throw new IOException("Layout version mismatch. This oiv tool " + |
| "handles layout version " + |
| NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION + ", but the " + |
| "XML file has <layoutVersion> " + layoutVersion + ". Please " + |
| "either re-generate the XML file with the proper layout version, " + |
| "or manually edit the XML file to be usable with this version " + |
| "of the oiv tool."); |
| } |
| fileSummaryBld.setOndiskVersion(onDiskVersion); |
| fileSummaryBld.setLayoutVersion(layoutVersion); |
| if (LOG.isDebugEnabled()) { |
| LOG.debug("Loaded <version> with onDiskVersion=" + onDiskVersion + |
| ", layoutVersion=" + layoutVersion + "."); |
| } |
| } |
| |
| /** |
| * Write the string table to the fsimage. |
| * @throws IOException |
| */ |
| private void writeStringTableSection() throws IOException { |
| FsImageProto.StringTableSection sectionHeader = |
| FsImageProto.StringTableSection.newBuilder(). |
| setNumEntry(stringTable.size()).build(); |
| if (LOG.isDebugEnabled()) { |
| LOG.debug(SectionName.STRING_TABLE.name() + " writing header: {" + |
| TextFormat.printToString(sectionHeader) + "}"); |
| } |
| sectionHeader.writeDelimitedTo(out); |
| |
| // The entries don't have to be in any particular order, so iterating |
| // over the hash table is fine. |
| for (Map.Entry<String, Integer> entry : stringTable.entrySet()) { |
| FsImageProto.StringTableSection.Entry stEntry = |
| FsImageProto.StringTableSection.Entry.newBuilder(). |
| setStr(entry.getKey()). |
| setId(entry.getValue()). |
| build(); |
| if (LOG.isTraceEnabled()) { |
| LOG.trace("Writing string table entry: {" + |
| TextFormat.printToString(stEntry) + "}"); |
| } |
| stEntry.writeDelimitedTo(out); |
| } |
| recordSectionLength(SectionName.STRING_TABLE.name()); |
| } |
| |
| /** |
| * Processes the XML file back into an fsimage. |
| */ |
| private void processXml() throws Exception { |
| LOG.debug("Loading <fsimage>."); |
| expectTag("fsimage", false); |
| // Read the <version> tag. |
| readVersion(); |
| // Write the HDFSIMG1 magic number which begins the fsimage file. |
| out.write(FSImageUtil.MAGIC_HEADER); |
| // Write a series of fsimage sections. |
| sectionStartOffset = FSImageUtil.MAGIC_HEADER.length; |
| final HashSet<String> unprocessedSections = |
| new HashSet<>(sections.keySet()); |
| while (!unprocessedSections.isEmpty()) { |
| XMLEvent ev = expectTag("[section header]", true); |
| if (ev.getEventType() == XMLStreamConstants.END_ELEMENT) { |
| if (ev.asEndElement().getName().getLocalPart().equals("fsimage")) { |
| throw new IOException("FSImage XML ended prematurely, without " + |
| "including section(s) " + StringUtils.join(", ", |
| unprocessedSections)); |
| } |
| throw new IOException("Got unexpected tag end event for " + |
| ev.asEndElement().getName().getLocalPart() + " while looking " + |
| "for section header tag."); |
| } else if (ev.getEventType() != XMLStreamConstants.START_ELEMENT) { |
| throw new IOException("Expected section header START_ELEMENT; " + |
| "got event of type " + ev.getEventType()); |
| } |
| String sectionName = ev.asStartElement().getName().getLocalPart(); |
| if (!unprocessedSections.contains(sectionName)) { |
| throw new IOException("Unknown or duplicate section found for " + |
| sectionName); |
| } |
| SectionProcessor sectionProcessor = sections.get(sectionName); |
| if (sectionProcessor == null) { |
| throw new IOException("Unknown FSImage section " + sectionName + |
| ". Valid section names are [" + |
| StringUtils.join(", ", sections.keySet()) + "]"); |
| } |
| unprocessedSections.remove(sectionName); |
| sectionProcessor.process(); |
| } |
| |
| // Write the StringTable section to disk. |
| // This has to be done after the other sections, since some of them |
| // add entries to the string table. |
| writeStringTableSection(); |
| |
| // Write the FileSummary section to disk. |
| // This section is always last. |
| long prevOffset = out.getCount(); |
| FileSummary fileSummary = fileSummaryBld.build(); |
| if (LOG.isDebugEnabled()) { |
| LOG.debug("Writing FileSummary: {" + |
| TextFormat.printToString(fileSummary) + "}"); |
| } |
| // Even though the last 4 bytes of the file gives the FileSummary length, |
| // we still write a varint first that also contains the length. |
| fileSummary.writeDelimitedTo(out); |
| |
| // Write the length of the FileSummary section as a fixed-size big |
| // endian 4-byte quantity. |
| int summaryLen = Ints.checkedCast(out.getCount() - prevOffset); |
| byte[] summaryLenBytes = new byte[4]; |
| ByteBuffer.wrap(summaryLenBytes).asIntBuffer().put(summaryLen); |
| out.write(summaryLenBytes); |
| } |
| |
| /** |
| * Run the OfflineImageReconstructor. |
| * |
| * @param inputPath The input path to use. |
| * @param outputPath The output path to use. |
| * |
| * @throws Exception On error. |
| */ |
| public static void run(String inputPath, String outputPath) |
| throws Exception { |
| MessageDigest digester = MD5Hash.getDigester(); |
| FileOutputStream fout = null; |
| File foutHash = new File(outputPath + ".md5"); |
| Files.deleteIfExists(foutHash.toPath()); // delete any .md5 file that exists |
| CountingOutputStream out = null; |
| FileInputStream fis = null; |
| InputStreamReader reader = null; |
| try { |
| Files.deleteIfExists(Paths.get(outputPath)); |
| fout = new FileOutputStream(outputPath); |
| fis = new FileInputStream(inputPath); |
| reader = new InputStreamReader(fis, Charset.forName("UTF-8")); |
| out = new CountingOutputStream( |
| new DigestOutputStream( |
| new BufferedOutputStream(fout), digester)); |
| OfflineImageReconstructor oir = |
| new OfflineImageReconstructor(out, reader); |
| oir.processXml(); |
| } finally { |
| IOUtils.cleanup(LOG, reader, fis, out, fout); |
| } |
| // Write the md5 file |
| MD5FileUtils.saveMD5File(new File(outputPath), |
| new MD5Hash(digester.digest())); |
| } |
| } |
