| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.hdfs.server.namenode; |
| |
| import static org.junit.Assert.assertEquals; |
| import static org.junit.Assert.assertFalse; |
| import static org.junit.Assert.assertNull; |
| import static org.junit.Assert.assertTrue; |
| |
| import java.io.FileNotFoundException; |
| |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.fs.Path; |
| import org.apache.hadoop.hdfs.DFSTestUtil; |
| import org.apache.hadoop.hdfs.DFSUtil; |
| import org.apache.hadoop.hdfs.DistributedFileSystem; |
| import org.apache.hadoop.hdfs.MiniDFSCluster; |
| import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable; |
| import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot; |
| import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeFileWithSnapshot; |
| import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; |
| import org.junit.AfterClass; |
| import org.junit.Assert; |
| import org.junit.BeforeClass; |
| import org.junit.Test; |
| |
| /** Test snapshot related operations. */ |
| public class TestSnapshotPathINodes { |
| private static final long seed = 0; |
| private static final short REPLICATION = 3; |
| |
| static private final Path dir = new Path("/TestSnapshot"); |
| |
| static private final Path sub1 = new Path(dir, "sub1"); |
| static private final Path file1 = new Path(sub1, "file1"); |
| static private final Path file2 = new Path(sub1, "file2"); |
| |
| static private Configuration conf; |
| static private MiniDFSCluster cluster; |
| static private FSNamesystem fsn; |
| static private FSDirectory fsdir; |
| |
| static private DistributedFileSystem hdfs; |
| |
| @BeforeClass |
| static public void setUp() throws Exception { |
| conf = new Configuration(); |
| cluster = new MiniDFSCluster.Builder(conf) |
| .numDataNodes(REPLICATION) |
| .build(); |
| cluster.waitActive(); |
| |
| fsn = cluster.getNamesystem(); |
| fsdir = fsn.getFSDirectory(); |
| |
| hdfs = cluster.getFileSystem(); |
| DFSTestUtil.createFile(hdfs, file1, 1024, REPLICATION, seed); |
| DFSTestUtil.createFile(hdfs, file2, 1024, REPLICATION, seed); |
| } |
| |
| @AfterClass |
| static public void tearDown() throws Exception { |
| if (cluster != null) { |
| cluster.shutdown(); |
| } |
| } |
| |
| /** Test allow-snapshot operation. */ |
| @Test (timeout=15000) |
| public void testAllowSnapshot() throws Exception { |
| final String pathStr = sub1.toString(); |
| final INode before = fsdir.getINode(pathStr); |
| |
| // Before a directory is snapshottable |
| Assert.assertTrue(before instanceof INodeDirectory); |
| Assert.assertFalse(before instanceof INodeDirectorySnapshottable); |
| |
| // After a directory is snapshottable |
| final Path path = new Path(pathStr); |
| hdfs.allowSnapshot(path); |
| { |
| final INode after = fsdir.getINode(pathStr); |
| Assert.assertTrue(after instanceof INodeDirectorySnapshottable); |
| } |
| |
| hdfs.disallowSnapshot(path); |
| { |
| final INode after = fsdir.getINode(pathStr); |
| Assert.assertTrue(after instanceof INodeDirectory); |
| Assert.assertFalse(after instanceof INodeDirectorySnapshottable); |
| } |
| } |
| |
| static Snapshot getSnapshot(INodesInPath inodesInPath, String name) { |
| if (name == null) { |
| return null; |
| } |
| final int i = inodesInPath.getSnapshotRootIndex() - 1; |
| final INode inode = inodesInPath.getINodes()[i]; |
| return ((INodeDirectorySnapshottable)inode).getSnapshot( |
| DFSUtil.string2Bytes(name)); |
| } |
| |
| static void assertSnapshot(INodesInPath inodesInPath, boolean isSnapshot, |
| final Snapshot snapshot, int index) { |
| assertEquals(isSnapshot, inodesInPath.isSnapshot()); |
| assertEquals(index, inodesInPath.getSnapshotRootIndex()); |
| assertEquals(isSnapshot? snapshot: null, inodesInPath.getPathSnapshot()); |
| assertEquals(isSnapshot? null: snapshot, inodesInPath.getLatestSnapshot()); |
| if (isSnapshot && index >= 0) { |
| assertEquals(Snapshot.Root.class, inodesInPath.getINodes()[index].getClass()); |
| } |
| } |
| |
| static void assertINodeFile(INode inode, Path path) { |
| assertEquals(path.getName(), inode.getLocalName()); |
| assertEquals(INodeFile.class, inode.getClass()); |
| } |
| |
| /** |
| * Test {@link INodeDirectory#getExistingPathINodes(byte[][], int, boolean)} |
| * for normal (non-snapshot) file. |
| */ |
| @Test (timeout=15000) |
| public void testNonSnapshotPathINodes() throws Exception { |
| // Get the inodes by resolving the path of a normal file |
| String[] names = INode.getPathNames(file1.toString()); |
| byte[][] components = INode.getPathComponents(names); |
| INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); |
| INode[] inodes = nodesInPath.getINodes(); |
| // The number of inodes should be equal to components.length |
| assertEquals(inodes.length, components.length); |
| // The returned nodesInPath should be non-snapshot |
| assertSnapshot(nodesInPath, false, null, -1); |
| |
| // The last INode should be associated with file1 |
| assertTrue("file1=" + file1 + ", nodesInPath=" + nodesInPath, |
| inodes[components.length - 1] != null); |
| assertEquals(inodes[components.length - 1].getFullPathName(), |
| file1.toString()); |
| assertEquals(inodes[components.length - 2].getFullPathName(), |
| sub1.toString()); |
| assertEquals(inodes[components.length - 3].getFullPathName(), |
| dir.toString()); |
| |
| // Call getExistingPathINodes and request only one INode. This is used |
| // when identifying the INode for a given path. |
| nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, 1, false); |
| inodes = nodesInPath.getINodes(); |
| assertEquals(inodes.length, 1); |
| assertSnapshot(nodesInPath, false, null, -1); |
| assertEquals(inodes[0].getFullPathName(), file1.toString()); |
| |
| // Call getExistingPathINodes and request 2 INodes. This is usually used |
| // when identifying the parent INode of a given path. |
| nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, 2, false); |
| inodes = nodesInPath.getINodes(); |
| assertEquals(inodes.length, 2); |
| assertSnapshot(nodesInPath, false, null, -1); |
| assertEquals(inodes[1].getFullPathName(), file1.toString()); |
| assertEquals(inodes[0].getFullPathName(), sub1.toString()); |
| } |
| |
| /** |
| * Test {@link INodeDirectory#getExistingPathINodes(byte[][], int, boolean)} |
| * for snapshot file. |
| */ |
| @Test (timeout=15000) |
| public void testSnapshotPathINodes() throws Exception { |
| // Create a snapshot for the dir, and check the inodes for the path |
| // pointing to a snapshot file |
| hdfs.allowSnapshot(sub1); |
| hdfs.createSnapshot(sub1, "s1"); |
| // The path when accessing the snapshot file of file1 is |
| // /TestSnapshot/sub1/.snapshot/s1/file1 |
| String snapshotPath = sub1.toString() + "/.snapshot/s1/file1"; |
| String[] names = INode.getPathNames(snapshotPath); |
| byte[][] components = INode.getPathComponents(names); |
| INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); |
| INode[] inodes = nodesInPath.getINodes(); |
| // Length of inodes should be (components.length - 1), since we will ignore |
| // ".snapshot" |
| assertEquals(inodes.length, components.length - 1); |
| // SnapshotRootIndex should be 3: {root, Testsnapshot, sub1, s1, file1} |
| final Snapshot snapshot = getSnapshot(nodesInPath, "s1"); |
| assertSnapshot(nodesInPath, true, snapshot, 3); |
| // Check the INode for file1 (snapshot file) |
| INode snapshotFileNode = inodes[inodes.length - 1]; |
| assertINodeFile(snapshotFileNode, file1); |
| assertTrue(snapshotFileNode.getParent() instanceof |
| INodeDirectoryWithSnapshot); |
| |
| // Call getExistingPathINodes and request only one INode. |
| nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, 1, false); |
| inodes = nodesInPath.getINodes(); |
| assertEquals(inodes.length, 1); |
| // The snapshotroot (s1) is not included in inodes. Thus the |
| // snapshotRootIndex should be -1. |
| assertSnapshot(nodesInPath, true, snapshot, -1); |
| // Check the INode for file1 (snapshot file) |
| assertINodeFile(nodesInPath.getLastINode(), file1); |
| |
| // Call getExistingPathINodes and request 2 INodes. |
| nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, 2, false); |
| inodes = nodesInPath.getINodes(); |
| assertEquals(inodes.length, 2); |
| // There should be two INodes in inodes: s1 and snapshot of file1. Thus the |
| // SnapshotRootIndex should be 0. |
| assertSnapshot(nodesInPath, true, snapshot, 0); |
| assertINodeFile(nodesInPath.getLastINode(), file1); |
| |
| // Resolve the path "/TestSnapshot/sub1/.snapshot" |
| String dotSnapshotPath = sub1.toString() + "/.snapshot"; |
| names = INode.getPathNames(dotSnapshotPath); |
| components = INode.getPathComponents(names); |
| nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); |
| inodes = nodesInPath.getINodes(); |
| // The number of INodes returned should be components.length - 1 since we |
| // will ignore ".snapshot" |
| assertEquals(inodes.length, components.length - 1); |
| |
| // No SnapshotRoot dir is included in the resolved inodes |
| assertSnapshot(nodesInPath, true, snapshot, -1); |
| // The last INode should be the INode for sub1 |
| final INode last = nodesInPath.getLastINode(); |
| assertEquals(last.getFullPathName(), sub1.toString()); |
| assertFalse(last instanceof INodeFileWithSnapshot); |
| |
| String[] invalidPathComponent = {"invalidDir", "foo", ".snapshot", "bar"}; |
| Path invalidPath = new Path(invalidPathComponent[0]); |
| for(int i = 1; i < invalidPathComponent.length; i++) { |
| invalidPath = new Path(invalidPath, invalidPathComponent[i]); |
| try { |
| hdfs.getFileStatus(invalidPath); |
| Assert.fail(); |
| } catch(FileNotFoundException fnfe) { |
| System.out.println("The exception is expected: " + fnfe); |
| } |
| } |
| } |
| |
| /** |
| * Test {@link INodeDirectory#getExistingPathINodes(byte[][], int, boolean)} |
| * for snapshot file after deleting the original file. |
| */ |
| @Test (timeout=15000) |
| public void testSnapshotPathINodesAfterDeletion() throws Exception { |
| // Create a snapshot for the dir, and check the inodes for the path |
| // pointing to a snapshot file |
| hdfs.allowSnapshot(sub1); |
| hdfs.createSnapshot(sub1, "s2"); |
| |
| // Delete the original file /TestSnapshot/sub1/file1 |
| hdfs.delete(file1, false); |
| |
| final Snapshot snapshot; |
| { |
| // Resolve the path for the snapshot file |
| // /TestSnapshot/sub1/.snapshot/s2/file1 |
| String snapshotPath = sub1.toString() + "/.snapshot/s2/file1"; |
| String[] names = INode.getPathNames(snapshotPath); |
| byte[][] components = INode.getPathComponents(names); |
| INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); |
| INode[] inodes = nodesInPath.getINodes(); |
| // Length of inodes should be (components.length - 1), since we will ignore |
| // ".snapshot" |
| assertEquals(inodes.length, components.length - 1); |
| // SnapshotRootIndex should be 3: {root, Testsnapshot, sub1, s2, file1} |
| snapshot = getSnapshot(nodesInPath, "s2"); |
| assertSnapshot(nodesInPath, true, snapshot, 3); |
| |
| // Check the INode for file1 (snapshot file) |
| final INode inode = inodes[inodes.length - 1]; |
| assertEquals(file1.getName(), inode.getLocalName()); |
| assertEquals(INodeFileWithSnapshot.class, inode.getClass()); |
| } |
| |
| // Check the INodes for path /TestSnapshot/sub1/file1 |
| String[] names = INode.getPathNames(file1.toString()); |
| byte[][] components = INode.getPathComponents(names); |
| INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); |
| INode[] inodes = nodesInPath.getINodes(); |
| // The length of inodes should be equal to components.length |
| assertEquals(inodes.length, components.length); |
| // The number of non-null elements should be components.length - 1 since |
| // file1 has been deleted |
| assertEquals(nodesInPath.getNumNonNull(), components.length - 1); |
| // The returned nodesInPath should be non-snapshot |
| assertSnapshot(nodesInPath, false, snapshot, -1); |
| // The last INode should be null, and the one before should be associated |
| // with sub1 |
| assertNull(inodes[components.length - 1]); |
| assertEquals(inodes[components.length - 2].getFullPathName(), |
| sub1.toString()); |
| assertEquals(inodes[components.length - 3].getFullPathName(), |
| dir.toString()); |
| } |
| |
| static private Snapshot s4; |
| |
| /** |
| * Test {@link INodeDirectory#getExistingPathINodes(byte[][], int, boolean)} |
| * for snapshot file while adding a new file after snapshot. |
| */ |
| @Test (timeout=15000) |
| public void testSnapshotPathINodesWithAddedFile() throws Exception { |
| // Create a snapshot for the dir, and check the inodes for the path |
| // pointing to a snapshot file |
| hdfs.allowSnapshot(sub1); |
| hdfs.createSnapshot(sub1, "s4"); |
| |
| // Add a new file /TestSnapshot/sub1/file3 |
| final Path file3 = new Path(sub1, "file3"); |
| DFSTestUtil.createFile(hdfs, file3, 1024, REPLICATION, seed); |
| |
| { |
| // Check the inodes for /TestSnapshot/sub1/.snapshot/s4/file3 |
| String snapshotPath = sub1.toString() + "/.snapshot/s4/file3"; |
| String[] names = INode.getPathNames(snapshotPath); |
| byte[][] components = INode.getPathComponents(names); |
| INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); |
| INode[] inodes = nodesInPath.getINodes(); |
| // Length of inodes should be (components.length - 1), since we will ignore |
| // ".snapshot" |
| assertEquals(inodes.length, components.length - 1); |
| // The number of non-null inodes should be components.length - 2, since |
| // snapshot of file3 does not exist |
| assertEquals(nodesInPath.getNumNonNull(), components.length - 2); |
| s4 = getSnapshot(nodesInPath, "s4"); |
| |
| // SnapshotRootIndex should still be 3: {root, Testsnapshot, sub1, s4, null} |
| assertSnapshot(nodesInPath, true, s4, 3); |
| |
| // Check the last INode in inodes, which should be null |
| assertNull(inodes[inodes.length - 1]); |
| } |
| |
| // Check the inodes for /TestSnapshot/sub1/file3 |
| String[] names = INode.getPathNames(file3.toString()); |
| byte[][] components = INode.getPathComponents(names); |
| INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); |
| INode[] inodes = nodesInPath.getINodes(); |
| // The number of inodes should be equal to components.length |
| assertEquals(inodes.length, components.length); |
| |
| // The returned nodesInPath should be non-snapshot |
| assertSnapshot(nodesInPath, false, s4, -1); |
| |
| // The last INode should be associated with file3 |
| assertEquals(inodes[components.length - 1].getFullPathName(), |
| file3.toString()); |
| assertEquals(inodes[components.length - 2].getFullPathName(), |
| sub1.toString()); |
| assertEquals(inodes[components.length - 3].getFullPathName(), |
| dir.toString()); |
| } |
| |
| /** |
| * Test {@link INodeDirectory#getExistingPathINodes(byte[][], int, boolean)} |
| * for snapshot file while modifying file after snapshot. |
| */ |
| @Test (timeout=15000) |
| public void testSnapshotPathINodesAfterModification() throws Exception { |
| //file1 was deleted, create it again. |
| DFSTestUtil.createFile(hdfs, file1, 1024, REPLICATION, seed); |
| |
| // First check the INode for /TestSnapshot/sub1/file1 |
| String[] names = INode.getPathNames(file1.toString()); |
| byte[][] components = INode.getPathComponents(names); |
| INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components); |
| INode[] inodes = nodesInPath.getINodes(); |
| // The number of inodes should be equal to components.length |
| assertEquals(inodes.length, components.length); |
| assertSnapshot(nodesInPath, false, s4, -1); |
| |
| // The last INode should be associated with file1 |
| assertEquals(inodes[components.length - 1].getFullPathName(), |
| file1.toString()); |
| |
| // Create a snapshot for the dir, and check the inodes for the path |
| // pointing to a snapshot file |
| hdfs.allowSnapshot(sub1); |
| hdfs.createSnapshot(sub1, "s3"); |
| |
| // Modify file1 |
| DFSTestUtil.appendFile(hdfs, file1, "the content for appending"); |
| |
| // Check the INodes for snapshot of file1 |
| String snapshotPath = sub1.toString() + "/.snapshot/s3/file1"; |
| names = INode.getPathNames(snapshotPath); |
| components = INode.getPathComponents(names); |
| INodesInPath ssNodesInPath = INodesInPath.resolve(fsdir.rootDir, components); |
| INode[] ssInodes = ssNodesInPath.getINodes(); |
| // Length of ssInodes should be (components.length - 1), since we will |
| // ignore ".snapshot" |
| assertEquals(ssInodes.length, components.length - 1); |
| final Snapshot s3 = getSnapshot(ssNodesInPath, "s3"); |
| assertSnapshot(ssNodesInPath, true, s3, 3); |
| // Check the INode for snapshot of file1 |
| INode snapshotFileNode = ssInodes[ssInodes.length - 1]; |
| assertEquals(snapshotFileNode.getLocalName(), file1.getName()); |
| assertTrue(snapshotFileNode instanceof INodeFileWithSnapshot); |
| // The modification time of the snapshot INode should be the same with the |
| // original INode before modification |
| assertEquals(inodes[inodes.length - 1].getModificationTime(), |
| snapshotFileNode.getModificationTime(ssNodesInPath.getPathSnapshot())); |
| |
| // Check the INode for /TestSnapshot/sub1/file1 again |
| names = INode.getPathNames(file1.toString()); |
| components = INode.getPathComponents(names); |
| INodesInPath newNodesInPath = INodesInPath.resolve(fsdir.rootDir, components); |
| assertSnapshot(newNodesInPath, false, s3, -1); |
| INode[] newInodes = newNodesInPath.getINodes(); |
| // The number of inodes should be equal to components.length |
| assertEquals(newInodes.length, components.length); |
| // The last INode should be associated with file1 |
| final int last = components.length - 1; |
| assertEquals(newInodes[last].getFullPathName(), file1.toString()); |
| // The modification time of the INode for file3 should have been changed |
| Assert.assertFalse(inodes[last].getModificationTime() |
| == newInodes[last].getModificationTime()); |
| } |
| } |