hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotReplication.java - hadoop - Git at Google

 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.hadoop.hdfs.server.namenode.snapshot;

 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;

 import java.util.HashMap;
 import java.util.Map;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
 import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;

 /**
  * This class tests the replication handling/calculation of snapshots. In
  * particular, {@link INodeFile#getFileReplication()} and
  * {@link INodeFileWithSnapshot#getBlockReplication()} are tested to make sure
  * the number of replication is calculated correctly with/without snapshots.
  */
 public class TestSnapshotReplication {

   private static final long seed = 0;
   private static final short REPLICATION = 3;
   private static final int NUMDATANODE = 5;
   private static final long BLOCKSIZE = 1024;

   private final Path dir = new Path("/TestSnapshot");
   private final Path sub1 = new Path(dir, "sub1");
   private final Path file1 = new Path(sub1, "file1");

   Configuration conf;
   MiniDFSCluster cluster;
   FSNamesystem fsn;
   DistributedFileSystem hdfs;
   FSDirectory fsdir;

   @Before
   public void setUp() throws Exception {
     conf = new Configuration();
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUMDATANODE)
         .build();
     cluster.waitActive();
     fsn = cluster.getNamesystem();
     hdfs = cluster.getFileSystem();
     fsdir = fsn.getFSDirectory();
   }

   @After
   public void tearDown() throws Exception {
     if (cluster != null) {
       cluster.shutdown();
     }
   }

   /**
    * Check the replication of a given file. We test both
    * {@link INodeFile#getFileReplication()} and
    * {@link INodeFile#getBlockReplication()}.
    *
    * @param file The given file
    * @param replication The expected replication number
    * @param blockReplication The expected replication number for the block
    * @throws Exception
    */
   private void checkFileReplication(Path file, short replication,
       short blockReplication) throws Exception {
     // Get FileStatus of file1, and identify the replication number of file1.
     // Note that the replication number in FileStatus was derived from
     // INodeFile#getFileReplication().
     short fileReplication = hdfs.getFileStatus(file1).getReplication();
     assertEquals(replication, fileReplication);
     // Check the correctness of getBlockReplication()
     INode inode = fsdir.getINode(file1.toString());
     assertTrue(inode instanceof INodeFile);
     assertEquals(blockReplication, ((INodeFile) inode).getBlockReplication());
   }

   /**
    * Test replication number calculation for a normal file without snapshots.
    */
   @Test (timeout=60000)
   public void testReplicationWithoutSnapshot() throws Exception {
     // Create file1, set its replication to REPLICATION
     DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, seed);
     // Check the replication of file1
     checkFileReplication(file1, REPLICATION, REPLICATION);
     // Change the replication factor of file1 from 3 to 2
     hdfs.setReplication(file1, (short) (REPLICATION - 1));
     // Check the replication again
     checkFileReplication(file1, (short) (REPLICATION - 1),
         (short) (REPLICATION - 1));
   }

   INodeFile getINodeFile(Path p) throws Exception {
     final String s = p.toString();
     return INodeFile.valueOf(fsdir.getINode(s), s);
   }

   /**
    * Check the replication for both the current file and all its prior snapshots
    *
    * @param currentFile
    *          the Path of the current file
    * @param snapshotRepMap
    *          A map maintaining all the snapshots of the current file, as well
    *          as their expected replication number stored in their corresponding
    *          INodes
    * @param expectedBlockRep
    *          The expected replication number that should be returned by
    *          {@link INodeFileWithSnapshot#getBlockReplication()} of all the INodes
    * @throws Exception
    */
   private void checkSnapshotFileReplication(Path currentFile,
       Map<Path, Short> snapshotRepMap, short expectedBlockRep) throws Exception {
     // First check the getBlockReplication for the INode of the currentFile
     final INodeFile inodeOfCurrentFile = getINodeFile(currentFile);
     assertEquals(expectedBlockRep, inodeOfCurrentFile.getBlockReplication());
     // Then check replication for every snapshot
     for (Path ss : snapshotRepMap.keySet()) {
       final INodesInPath iip = fsdir.getLastINodeInPath(ss.toString());
       final INodeFile ssInode = (INodeFile)iip.getLastINode();
       // The replication number derived from the
       // INodeFileWithLink#getBlockReplication should always == expectedBlockRep
       assertEquals(expectedBlockRep, ssInode.getBlockReplication());
       // Also check the number derived from INodeFile#getFileReplication
       assertEquals(snapshotRepMap.get(ss).shortValue(),
           ssInode.getFileReplication(iip.getPathSnapshot()));
     }
   }

   /**
    * Test replication number calculation for a file with snapshots.
    */
   @Test (timeout=60000)
   public void testReplicationWithSnapshot() throws Exception {
     short fileRep = 1;
     // Create file1, set its replication to 1
     DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, fileRep, seed);
     Map<Path, Short> snapshotRepMap = new HashMap<Path, Short>();
     // Change replication factor from 1 to 5. In the meanwhile, keep taking
     // snapshots for sub1
     for (; fileRep < NUMDATANODE; ) {
       // Create snapshot for sub1
       Path snapshotRoot = SnapshotTestHelper.createSnapshot(hdfs, sub1, "s"
           + fileRep);
       Path snapshot = new Path(snapshotRoot, file1.getName());

       // Check the replication stored in the INode of the snapshot of file1
       assertEquals(fileRep, getINodeFile(snapshot).getFileReplication());
       snapshotRepMap.put(snapshot, fileRep);

       // Increase the replication factor by 1
       hdfs.setReplication(file1, ++fileRep);
       // Check the replication for file1
       checkFileReplication(file1, fileRep, fileRep);
       // Also check the replication for all the prior snapshots of file1
       checkSnapshotFileReplication(file1, snapshotRepMap, fileRep);
     }

     // Change replication factor back to 3.
     hdfs.setReplication(file1, REPLICATION);
     // Check the replication for file1
     // Currently the max replication among snapshots should be 4
     checkFileReplication(file1, REPLICATION, (short) (NUMDATANODE - 1));
     // Also check the replication for all the prior snapshots of file1.
     // Currently the max replication among snapshots should be 4
     checkSnapshotFileReplication(file1, snapshotRepMap,
         (short) (NUMDATANODE - 1));
   }

   /**
    * Test replication for a file with snapshots, also including the scenario
    * where the original file is deleted
    */
   @Test (timeout=60000)
   public void testReplicationAfterDeletion() throws Exception {
     // Create file1, set its replication to 3
     DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, seed);
     Map<Path, Short> snapshotRepMap = new HashMap<Path, Short>();
     // Take 3 snapshots of sub1
     for (int i = 1; i <= 3; i++) {
       Path root = SnapshotTestHelper.createSnapshot(hdfs, sub1, "s" + i);
       Path ssFile = new Path(root, file1.getName());
       snapshotRepMap.put(ssFile, REPLICATION);
     }
     // Check replication
     checkFileReplication(file1, REPLICATION, REPLICATION);
     checkSnapshotFileReplication(file1, snapshotRepMap, REPLICATION);

     // Delete file1
     hdfs.delete(file1, true);
     // Check replication of snapshots
     for (Path ss : snapshotRepMap.keySet()) {
       final INodeFile ssInode = getINodeFile(ss);
       // The replication number derived from the
       // INodeFileWithLink#getBlockReplication should always == expectedBlockRep
       assertEquals(REPLICATION, ssInode.getBlockReplication());
       // Also check the number derived from INodeFile#getFileReplication
       assertEquals(snapshotRepMap.get(ss).shortValue(),
           ssInode.getFileReplication());
     }
   }

 }
	/**
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.hadoop.hdfs.server.namenode.snapshot;

	import static org.junit.Assert.assertEquals;
	import static org.junit.Assert.assertTrue;

	import java.util.HashMap;
	import java.util.Map;

	import org.apache.hadoop.conf.Configuration;
	import org.apache.hadoop.fs.Path;
	import org.apache.hadoop.hdfs.DFSTestUtil;
	import org.apache.hadoop.hdfs.DistributedFileSystem;
	import org.apache.hadoop.hdfs.MiniDFSCluster;
	import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
	import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
	import org.apache.hadoop.hdfs.server.namenode.INode;
	import org.apache.hadoop.hdfs.server.namenode.INodeFile;
	import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
	import org.junit.After;
	import org.junit.Before;
	import org.junit.Test;

	/**
	* This class tests the replication handling/calculation of snapshots. In
	* particular, {@link INodeFile#getFileReplication()} and
	* {@link INodeFileWithSnapshot#getBlockReplication()} are tested to make sure
	* the number of replication is calculated correctly with/without snapshots.
	*/
	public class TestSnapshotReplication {

	private static final long seed = 0;
	private static final short REPLICATION = 3;
	private static final int NUMDATANODE = 5;
	private static final long BLOCKSIZE = 1024;

	private final Path dir = new Path("/TestSnapshot");
	private final Path sub1 = new Path(dir, "sub1");
	private final Path file1 = new Path(sub1, "file1");

	Configuration conf;
	MiniDFSCluster cluster;
	FSNamesystem fsn;
	DistributedFileSystem hdfs;
	FSDirectory fsdir;

	@Before
	public void setUp() throws Exception {
	conf = new Configuration();
	cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUMDATANODE)
	.build();
	cluster.waitActive();
	fsn = cluster.getNamesystem();
	hdfs = cluster.getFileSystem();
	fsdir = fsn.getFSDirectory();
	}

	@After
	public void tearDown() throws Exception {
	if (cluster != null) {
	cluster.shutdown();
	}
	}

	/**
	* Check the replication of a given file. We test both
	* {@link INodeFile#getFileReplication()} and
	* {@link INodeFile#getBlockReplication()}.
	*
	* @param file The given file
	* @param replication The expected replication number
	* @param blockReplication The expected replication number for the block
	* @throws Exception
	*/
	private void checkFileReplication(Path file, short replication,
	short blockReplication) throws Exception {
	// Get FileStatus of file1, and identify the replication number of file1.
	// Note that the replication number in FileStatus was derived from
	// INodeFile#getFileReplication().
	short fileReplication = hdfs.getFileStatus(file1).getReplication();
	assertEquals(replication, fileReplication);
	// Check the correctness of getBlockReplication()
	INode inode = fsdir.getINode(file1.toString());
	assertTrue(inode instanceof INodeFile);
	assertEquals(blockReplication, ((INodeFile) inode).getBlockReplication());
	}

	/**
	* Test replication number calculation for a normal file without snapshots.
	*/
	@Test (timeout=60000)
	public void testReplicationWithoutSnapshot() throws Exception {
	// Create file1, set its replication to REPLICATION
	DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, seed);
	// Check the replication of file1
	checkFileReplication(file1, REPLICATION, REPLICATION);
	// Change the replication factor of file1 from 3 to 2
	hdfs.setReplication(file1, (short) (REPLICATION - 1));
	// Check the replication again
	checkFileReplication(file1, (short) (REPLICATION - 1),
	(short) (REPLICATION - 1));
	}

	INodeFile getINodeFile(Path p) throws Exception {
	final String s = p.toString();
	return INodeFile.valueOf(fsdir.getINode(s), s);
	}

	/**
	* Check the replication for both the current file and all its prior snapshots
	*
	* @param currentFile
	* the Path of the current file
	* @param snapshotRepMap
	* A map maintaining all the snapshots of the current file, as well
	* as their expected replication number stored in their corresponding
	* INodes
	* @param expectedBlockRep
	* The expected replication number that should be returned by
	* {@link INodeFileWithSnapshot#getBlockReplication()} of all the INodes
	* @throws Exception
	*/
	private void checkSnapshotFileReplication(Path currentFile,
	Map<Path, Short> snapshotRepMap, short expectedBlockRep) throws Exception {
	// First check the getBlockReplication for the INode of the currentFile
	final INodeFile inodeOfCurrentFile = getINodeFile(currentFile);
	assertEquals(expectedBlockRep, inodeOfCurrentFile.getBlockReplication());
	// Then check replication for every snapshot
	for (Path ss : snapshotRepMap.keySet()) {
	final INodesInPath iip = fsdir.getLastINodeInPath(ss.toString());
	final INodeFile ssInode = (INodeFile)iip.getLastINode();
	// The replication number derived from the
	// INodeFileWithLink#getBlockReplication should always == expectedBlockRep
	assertEquals(expectedBlockRep, ssInode.getBlockReplication());
	// Also check the number derived from INodeFile#getFileReplication
	assertEquals(snapshotRepMap.get(ss).shortValue(),
	ssInode.getFileReplication(iip.getPathSnapshot()));
	}
	}

	/**
	* Test replication number calculation for a file with snapshots.
	*/
	@Test (timeout=60000)
	public void testReplicationWithSnapshot() throws Exception {
	short fileRep = 1;
	// Create file1, set its replication to 1
	DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, fileRep, seed);
	Map<Path, Short> snapshotRepMap = new HashMap<Path, Short>();
	// Change replication factor from 1 to 5. In the meanwhile, keep taking
	// snapshots for sub1
	for (; fileRep < NUMDATANODE; ) {
	// Create snapshot for sub1
	Path snapshotRoot = SnapshotTestHelper.createSnapshot(hdfs, sub1, "s"
	+ fileRep);
	Path snapshot = new Path(snapshotRoot, file1.getName());

	// Check the replication stored in the INode of the snapshot of file1
	assertEquals(fileRep, getINodeFile(snapshot).getFileReplication());
	snapshotRepMap.put(snapshot, fileRep);

	// Increase the replication factor by 1
	hdfs.setReplication(file1, ++fileRep);
	// Check the replication for file1
	checkFileReplication(file1, fileRep, fileRep);
	// Also check the replication for all the prior snapshots of file1
	checkSnapshotFileReplication(file1, snapshotRepMap, fileRep);
	}

	// Change replication factor back to 3.
	hdfs.setReplication(file1, REPLICATION);
	// Check the replication for file1
	// Currently the max replication among snapshots should be 4
	checkFileReplication(file1, REPLICATION, (short) (NUMDATANODE - 1));
	// Also check the replication for all the prior snapshots of file1.
	// Currently the max replication among snapshots should be 4
	checkSnapshotFileReplication(file1, snapshotRepMap,
	(short) (NUMDATANODE - 1));
	}

	/**
	* Test replication for a file with snapshots, also including the scenario
	* where the original file is deleted
	*/
	@Test (timeout=60000)
	public void testReplicationAfterDeletion() throws Exception {
	// Create file1, set its replication to 3
	DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, seed);
	Map<Path, Short> snapshotRepMap = new HashMap<Path, Short>();
	// Take 3 snapshots of sub1
	for (int i = 1; i <= 3; i++) {
	Path root = SnapshotTestHelper.createSnapshot(hdfs, sub1, "s" + i);
	Path ssFile = new Path(root, file1.getName());
	snapshotRepMap.put(ssFile, REPLICATION);
	}
	// Check replication
	checkFileReplication(file1, REPLICATION, REPLICATION);
	checkSnapshotFileReplication(file1, snapshotRepMap, REPLICATION);

	// Delete file1
	hdfs.delete(file1, true);
	// Check replication of snapshots
	for (Path ss : snapshotRepMap.keySet()) {
	final INodeFile ssInode = getINodeFile(ss);
	// The replication number derived from the
	// INodeFileWithLink#getBlockReplication should always == expectedBlockRep
	assertEquals(REPLICATION, ssInode.getBlockReplication());
	// Also check the number derived from INodeFile#getFileReplication
	assertEquals(snapshotRepMap.get(ss).shortValue(),
	ssInode.getFileReplication());
	}
	}

	}