hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java - hadoop - Git at Google

 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.hadoop.hdfs.server.namenode;

 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;

 import java.io.File;
 import java.io.IOException;
 import java.util.EnumSet;

 import org.junit.Assert;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSOutputStream;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
 import org.apache.hadoop.hdfs.util.MD5FileUtils;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.PathUtils;
 import org.junit.Test;

 public class TestFSImage {

   private static final String HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ =
       "image-with-zero-block-size.tar.gz";
   @Test
   public void testPersist() throws IOException {
     Configuration conf = new Configuration();
     testPersistHelper(conf);
   }

   @Test
   public void testCompression() throws IOException {
     Configuration conf = new Configuration();
     conf.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, true);
     conf.set(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY,
         "org.apache.hadoop.io.compress.GzipCodec");
     testPersistHelper(conf);
   }

   private void testPersistHelper(Configuration conf) throws IOException {
     MiniDFSCluster cluster = null;
     try {
       cluster = new MiniDFSCluster.Builder(conf).build();
       cluster.waitActive();
       FSNamesystem fsn = cluster.getNamesystem();
       DistributedFileSystem fs = cluster.getFileSystem();

       final Path dir = new Path("/abc/def");
       final Path file1 = new Path(dir, "f1");
       final Path file2 = new Path(dir, "f2");

       // create an empty file f1
       fs.create(file1).close();

       // create an under-construction file f2
       FSDataOutputStream out = fs.create(file2);
       out.writeBytes("hello");
       ((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet
           .of(SyncFlag.UPDATE_LENGTH));

       // checkpoint
       fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
       fs.saveNamespace();
       fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);

       cluster.restartNameNode();
       cluster.waitActive();
       fs = cluster.getFileSystem();

       assertTrue(fs.isDirectory(dir));
       assertTrue(fs.exists(file1));
       assertTrue(fs.exists(file2));

       // check internals of file2
       INodeFile file2Node = fsn.dir.getINode4Write(file2.toString()).asFile();
       assertEquals("hello".length(), file2Node.computeFileSize());
       assertTrue(file2Node.isUnderConstruction());
       BlockInfo[] blks = file2Node.getBlocks();
       assertEquals(1, blks.length);
       assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState());
       // check lease manager
       Lease lease = fsn.leaseManager.getLease(file2Node);
       Assert.assertNotNull(lease);
     } finally {
       if (cluster != null) {
         cluster.shutdown();
       }
     }
   }

    /**
    * On checkpointing , stale fsimage checkpoint file should be deleted.
    */
   @Test
   public void testRemovalStaleFsimageCkpt() throws IOException {
     MiniDFSCluster cluster = null;
     SecondaryNameNode secondary = null;
     Configuration conf = new HdfsConfiguration();
     try {
       cluster = new MiniDFSCluster.Builder(conf).
           numDataNodes(1).format(true).build();
       conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
           "0.0.0.0:0");
       secondary = new SecondaryNameNode(conf);
       // Do checkpointing
       secondary.doCheckpoint();
       NNStorage storage = secondary.getFSImage().storage;
       File currentDir = FSImageTestUtil.
           getCurrentDirs(storage, NameNodeDirType.IMAGE).get(0);
       // Create a stale fsimage.ckpt file
       File staleCkptFile = new File(currentDir.getPath() +
           "/fsimage.ckpt_0000000000000000002");
       staleCkptFile.createNewFile();
       assertTrue(staleCkptFile.exists());
       // After checkpoint stale fsimage.ckpt file should be deleted
       secondary.doCheckpoint();
       assertFalse(staleCkptFile.exists());
     } finally {
       if (secondary != null) {
         secondary.shutdown();
         secondary = null;
       }
       if (cluster != null) {
         cluster.shutdown();
         cluster = null;
       }
     }
   }

   /**
    * Ensure that the digest written by the saver equals to the digest of the
    * file.
    */
   @Test
   public void testDigest() throws IOException {
     Configuration conf = new Configuration();
     MiniDFSCluster cluster = null;
     try {
       cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
       DistributedFileSystem fs = cluster.getFileSystem();
       fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
       fs.saveNamespace();
       fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
       File currentDir = FSImageTestUtil.getNameNodeCurrentDirs(cluster, 0).get(
           0);
       File fsimage = FSImageTestUtil.findNewestImageFile(currentDir
           .getAbsolutePath());
       assertEquals(MD5FileUtils.readStoredMd5ForFile(fsimage),
           MD5FileUtils.computeMd5ForFile(fsimage));
     } finally {
       if (cluster != null) {
         cluster.shutdown();
       }
     }
   }

   /**
    * Ensure mtime and atime can be loaded from fsimage.
    */
   @Test(timeout=60000)
   public void testLoadMtimeAtime() throws Exception {
     Configuration conf = new Configuration();
     MiniDFSCluster cluster = null;
     try {
       cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
       cluster.waitActive();
       DistributedFileSystem hdfs = cluster.getFileSystem();
       String userDir = hdfs.getHomeDirectory().toUri().getPath().toString();
       Path file = new Path(userDir, "file");
       Path dir = new Path(userDir, "/dir");
       Path link = new Path(userDir, "/link");
       hdfs.createNewFile(file);
       hdfs.mkdirs(dir);
       hdfs.createSymlink(file, link, false);

       long mtimeFile = hdfs.getFileStatus(file).getModificationTime();
       long atimeFile = hdfs.getFileStatus(file).getAccessTime();
       long mtimeDir = hdfs.getFileStatus(dir).getModificationTime();
       long mtimeLink = hdfs.getFileLinkStatus(link).getModificationTime();
       long atimeLink = hdfs.getFileLinkStatus(link).getAccessTime();

       // save namespace and restart cluster
       hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
       hdfs.saveNamespace();
       hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
       cluster.shutdown();
       cluster = new MiniDFSCluster.Builder(conf).format(false)
           .numDataNodes(1).build();
       cluster.waitActive();
       hdfs = cluster.getFileSystem();

       assertEquals(mtimeFile, hdfs.getFileStatus(file).getModificationTime());
       assertEquals(atimeFile, hdfs.getFileStatus(file).getAccessTime());
       assertEquals(mtimeDir, hdfs.getFileStatus(dir).getModificationTime());
       assertEquals(mtimeLink, hdfs.getFileLinkStatus(link).getModificationTime());
       assertEquals(atimeLink, hdfs.getFileLinkStatus(link).getAccessTime());
     } finally {
       if (cluster != null) {
         cluster.shutdown();
       }
     }
   }

   /**
    * In this test case, I have created an image with a file having
    * preferredblockSize = 0. We are trying to read this image (since file with
    * preferredblockSize = 0 was allowed pre 2.1.0-beta version. The namenode
    * after 2.6 version will not be able to read this particular file.
    * See HDFS-7788 for more information.
    * @throws Exception
    */
   @Test
   public void testZeroBlockSize() throws Exception {
     final Configuration conf = new HdfsConfiguration();
     String tarFile = System.getProperty("test.cache.data", "build/test/cache")
       + "/" + HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ;
     String testDir = PathUtils.getTestDirName(getClass());
     File dfsDir = new File(testDir, "image-with-zero-block-size");
     if (dfsDir.exists() && !FileUtil.fullyDelete(dfsDir)) {
       throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
     }
     FileUtil.unTar(new File(tarFile), new File(testDir));
     File nameDir = new File(dfsDir, "name");
     GenericTestUtils.assertExists(nameDir);
     conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
         nameDir.getAbsolutePath());
     MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
         .format(false)
         .manageDataDfsDirs(false)
         .manageNameDfsDirs(false)
         .waitSafeMode(false)
         .startupOption(StartupOption.UPGRADE)
         .build();
     try {
       FileSystem fs = cluster.getFileSystem();
       Path testPath = new Path("/tmp/zeroBlockFile");
       assertTrue("File /tmp/zeroBlockFile doesn't exist ", fs.exists(testPath));
       assertTrue("Name node didn't come up", cluster.isNameNodeUp(0));
     } finally {
       cluster.shutdown();
       //Clean up
       FileUtil.fullyDelete(dfsDir);
     }
   }
 }
	/**
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.hadoop.hdfs.server.namenode;

	import static org.junit.Assert.assertEquals;
	import static org.junit.Assert.assertFalse;
	import static org.junit.Assert.assertTrue;

	import java.io.File;
	import java.io.IOException;
	import java.util.EnumSet;

	import org.junit.Assert;

	import org.apache.hadoop.conf.Configuration;
	import org.apache.hadoop.fs.FSDataOutputStream;
	import org.apache.hadoop.fs.FileSystem;
	import org.apache.hadoop.fs.FileUtil;
	import org.apache.hadoop.fs.Path;
	import org.apache.hadoop.hdfs.DFSConfigKeys;
	import org.apache.hadoop.hdfs.DFSOutputStream;
	import org.apache.hadoop.hdfs.DistributedFileSystem;
	import org.apache.hadoop.hdfs.HdfsConfiguration;
	import org.apache.hadoop.hdfs.MiniDFSCluster;
	import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
	import org.apache.hadoop.hdfs.protocol.HdfsConstants;
	import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
	import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
	import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
	import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
	import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
	import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
	import org.apache.hadoop.hdfs.util.MD5FileUtils;
	import org.apache.hadoop.test.GenericTestUtils;
	import org.apache.hadoop.test.PathUtils;
	import org.junit.Test;

	public class TestFSImage {

	private static final String HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ =
	"image-with-zero-block-size.tar.gz";
	@Test
	public void testPersist() throws IOException {
	Configuration conf = new Configuration();
	testPersistHelper(conf);
	}

	@Test
	public void testCompression() throws IOException {
	Configuration conf = new Configuration();
	conf.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, true);
	conf.set(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY,
	"org.apache.hadoop.io.compress.GzipCodec");
	testPersistHelper(conf);
	}

	private void testPersistHelper(Configuration conf) throws IOException {
	MiniDFSCluster cluster = null;
	try {
	cluster = new MiniDFSCluster.Builder(conf).build();
	cluster.waitActive();
	FSNamesystem fsn = cluster.getNamesystem();
	DistributedFileSystem fs = cluster.getFileSystem();

	final Path dir = new Path("/abc/def");
	final Path file1 = new Path(dir, "f1");
	final Path file2 = new Path(dir, "f2");

	// create an empty file f1
	fs.create(file1).close();

	// create an under-construction file f2
	FSDataOutputStream out = fs.create(file2);
	out.writeBytes("hello");
	((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet
	.of(SyncFlag.UPDATE_LENGTH));

	// checkpoint
	fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
	fs.saveNamespace();
	fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);

	cluster.restartNameNode();
	cluster.waitActive();
	fs = cluster.getFileSystem();

	assertTrue(fs.isDirectory(dir));
	assertTrue(fs.exists(file1));
	assertTrue(fs.exists(file2));

	// check internals of file2
	INodeFile file2Node = fsn.dir.getINode4Write(file2.toString()).asFile();
	assertEquals("hello".length(), file2Node.computeFileSize());
	assertTrue(file2Node.isUnderConstruction());
	BlockInfo[] blks = file2Node.getBlocks();
	assertEquals(1, blks.length);
	assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState());
	// check lease manager
	Lease lease = fsn.leaseManager.getLease(file2Node);
	Assert.assertNotNull(lease);
	} finally {
	if (cluster != null) {
	cluster.shutdown();
	}
	}
	}

	/**
	* On checkpointing , stale fsimage checkpoint file should be deleted.
	*/
	@Test
	public void testRemovalStaleFsimageCkpt() throws IOException {
	MiniDFSCluster cluster = null;
	SecondaryNameNode secondary = null;
	Configuration conf = new HdfsConfiguration();
	try {
	cluster = new MiniDFSCluster.Builder(conf).
	numDataNodes(1).format(true).build();
	conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
	"0.0.0.0:0");
	secondary = new SecondaryNameNode(conf);
	// Do checkpointing
	secondary.doCheckpoint();
	NNStorage storage = secondary.getFSImage().storage;
	File currentDir = FSImageTestUtil.
	getCurrentDirs(storage, NameNodeDirType.IMAGE).get(0);
	// Create a stale fsimage.ckpt file
	File staleCkptFile = new File(currentDir.getPath() +
	"/fsimage.ckpt_0000000000000000002");
	staleCkptFile.createNewFile();
	assertTrue(staleCkptFile.exists());
	// After checkpoint stale fsimage.ckpt file should be deleted
	secondary.doCheckpoint();
	assertFalse(staleCkptFile.exists());
	} finally {
	if (secondary != null) {
	secondary.shutdown();
	secondary = null;
	}
	if (cluster != null) {
	cluster.shutdown();
	cluster = null;
	}
	}
	}

	/**
	* Ensure that the digest written by the saver equals to the digest of the
	* file.
	*/
	@Test
	public void testDigest() throws IOException {
	Configuration conf = new Configuration();
	MiniDFSCluster cluster = null;
	try {
	cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
	DistributedFileSystem fs = cluster.getFileSystem();
	fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
	fs.saveNamespace();
	fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
	File currentDir = FSImageTestUtil.getNameNodeCurrentDirs(cluster, 0).get(
	0);
	File fsimage = FSImageTestUtil.findNewestImageFile(currentDir
	.getAbsolutePath());
	assertEquals(MD5FileUtils.readStoredMd5ForFile(fsimage),
	MD5FileUtils.computeMd5ForFile(fsimage));
	} finally {
	if (cluster != null) {
	cluster.shutdown();
	}
	}
	}

	/**
	* Ensure mtime and atime can be loaded from fsimage.
	*/
	@Test(timeout=60000)
	public void testLoadMtimeAtime() throws Exception {
	Configuration conf = new Configuration();
	MiniDFSCluster cluster = null;
	try {
	cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
	cluster.waitActive();
	DistributedFileSystem hdfs = cluster.getFileSystem();
	String userDir = hdfs.getHomeDirectory().toUri().getPath().toString();
	Path file = new Path(userDir, "file");
	Path dir = new Path(userDir, "/dir");
	Path link = new Path(userDir, "/link");
	hdfs.createNewFile(file);
	hdfs.mkdirs(dir);
	hdfs.createSymlink(file, link, false);

	long mtimeFile = hdfs.getFileStatus(file).getModificationTime();
	long atimeFile = hdfs.getFileStatus(file).getAccessTime();
	long mtimeDir = hdfs.getFileStatus(dir).getModificationTime();
	long mtimeLink = hdfs.getFileLinkStatus(link).getModificationTime();
	long atimeLink = hdfs.getFileLinkStatus(link).getAccessTime();

	// save namespace and restart cluster
	hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
	hdfs.saveNamespace();
	hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
	cluster.shutdown();
	cluster = new MiniDFSCluster.Builder(conf).format(false)
	.numDataNodes(1).build();
	cluster.waitActive();
	hdfs = cluster.getFileSystem();

	assertEquals(mtimeFile, hdfs.getFileStatus(file).getModificationTime());
	assertEquals(atimeFile, hdfs.getFileStatus(file).getAccessTime());
	assertEquals(mtimeDir, hdfs.getFileStatus(dir).getModificationTime());
	assertEquals(mtimeLink, hdfs.getFileLinkStatus(link).getModificationTime());
	assertEquals(atimeLink, hdfs.getFileLinkStatus(link).getAccessTime());
	} finally {
	if (cluster != null) {
	cluster.shutdown();
	}
	}
	}

	/**
	* In this test case, I have created an image with a file having
	* preferredblockSize = 0. We are trying to read this image (since file with
	* preferredblockSize = 0 was allowed pre 2.1.0-beta version. The namenode
	* after 2.6 version will not be able to read this particular file.
	* See HDFS-7788 for more information.
	* @throws Exception
	*/
	@Test
	public void testZeroBlockSize() throws Exception {
	final Configuration conf = new HdfsConfiguration();
	String tarFile = System.getProperty("test.cache.data", "build/test/cache")
	+ "/" + HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ;
	String testDir = PathUtils.getTestDirName(getClass());
	File dfsDir = new File(testDir, "image-with-zero-block-size");
	if (dfsDir.exists() && !FileUtil.fullyDelete(dfsDir)) {
	throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
	}
	FileUtil.unTar(new File(tarFile), new File(testDir));
	File nameDir = new File(dfsDir, "name");
	GenericTestUtils.assertExists(nameDir);
	conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
	nameDir.getAbsolutePath());
	MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
	.format(false)
	.manageDataDfsDirs(false)
	.manageNameDfsDirs(false)
	.waitSafeMode(false)
	.startupOption(StartupOption.UPGRADE)
	.build();
	try {
	FileSystem fs = cluster.getFileSystem();
	Path testPath = new Path("/tmp/zeroBlockFile");
	assertTrue("File /tmp/zeroBlockFile doesn't exist ", fs.exists(testPath));
	assertTrue("Name node didn't come up", cluster.isNameNodeUp(0));
	} finally {
	cluster.shutdown();
	//Clean up
	FileUtil.fullyDelete(dfsDir);
	}
	}
	}