| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.hdfs.server.namenode.snapshot; |
| |
| import com.google.common.base.Supplier; |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.fs.FileStatus; |
| import org.apache.hadoop.fs.FileSystem; |
| import org.apache.hadoop.fs.Options; |
| import org.apache.hadoop.fs.Path; |
| import org.apache.hadoop.hdfs.DFSConfigKeys; |
| import org.apache.hadoop.hdfs.DFSTestUtil; |
| import org.apache.hadoop.hdfs.DistributedFileSystem; |
| import org.apache.hadoop.hdfs.MiniDFSCluster; |
| import org.apache.hadoop.hdfs.protocol.HdfsConstants; |
| import org.apache.hadoop.hdfs.protocol.SnapshotException; |
| import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; |
| import org.apache.hadoop.test.GenericTestUtils; |
| import org.junit.After; |
| import org.junit.Before; |
| import org.junit.Test; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| import java.io.IOException; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.HashMap; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Random; |
| import java.util.UUID; |
| import java.util.concurrent.TimeoutException; |
| |
| import static org.junit.Assert.assertEquals; |
| import static org.junit.Assert.assertNull; |
| import static org.junit.Assert.assertFalse; |
| import static org.junit.Assert.assertTrue; |
| |
| /** |
| * Testing random FileSystem operations with random Snapshot operations. |
| */ |
| public class TestRandomOpsWithSnapshots { |
| |
| private static final Logger LOG = |
| LoggerFactory.getLogger(TestRandomOpsWithSnapshots.class); |
| |
| private static final short REPL = 3; |
| private static final long BLOCKSIZE = 1024; |
| |
| private static final int TOTAL_FILECOUNT = 250; |
| private static final int MAX_NUM_ITERATIONS = 10; |
| private static final int MAX_NUM_FILESYSTEM_OPERATIONS = 50; |
| private static final int MAX_NUM_SNAPSHOT_OPERATIONS = 50; |
| private static final int MAX_NUM_SUB_DIRECTORIES_LEVEL = 10; |
| private static final int MAX_NUM_FILE_LENGTH = 100; |
| private static final int MIN_NUM_OPERATIONS = 25; |
| |
| private static final String TESTDIRSTRING = "/testDir"; |
| private static final String WITNESSDIRSTRING = "/WITNESSDIR"; |
| private static final Path TESTDIR = new Path(TESTDIRSTRING); |
| private static final Path WITNESSDIR = new Path(WITNESSDIRSTRING); |
| private static List<Path> snapshottableDirectories = new ArrayList<Path>(); |
| private static Map<Path, ArrayList<String>> pathToSnapshotsMap = |
| new HashMap<Path, ArrayList<String>>(); |
| |
| private static final Configuration CONFIG = new Configuration(); |
| private MiniDFSCluster cluster; |
| private DistributedFileSystem hdfs; |
| private static Random generator = null; |
| |
| private int numberFileCreated = 0; |
| private int numberFileDeleted = 0; |
| private int numberFileRenamed = 0; |
| |
| private int numberDirectoryCreated = 0; |
| private int numberDirectoryDeleted = 0; |
| private int numberDirectoryRenamed = 0; |
| |
| private int numberSnapshotCreated = 0; |
| private int numberSnapshotDeleted = 0; |
| private int numberSnapshotRenamed = 0; |
| |
| // Operation directories |
| private enum OperationDirectories { |
| TestDir, |
| WitnessDir; |
| } |
| |
| // Operation type |
| private enum OperationType { |
| FileSystem, |
| Snapshot; |
| } |
| |
| // FileSystem & Snapshot operation |
| private enum Operations { |
| FileSystem_CreateFile(2 /*operation weight*/, OperationType.FileSystem), |
| FileSystem_DeleteFile(2, OperationType.FileSystem), |
| FileSystem_RenameFile(2, OperationType.FileSystem), |
| FileSystem_CreateDir(1, OperationType.FileSystem), |
| FileSystem_DeleteDir(1, OperationType.FileSystem), |
| FileSystem_RenameDir(2, OperationType.FileSystem), |
| |
| Snapshot_CreateSnapshot(5, OperationType.Snapshot), |
| Snapshot_DeleteSnapshot(3, OperationType.Snapshot), |
| Snapshot_RenameSnapshot(2, OperationType.Snapshot); |
| |
| private int weight; |
| private OperationType operationType; |
| |
| Operations(int weight, OperationType type) { |
| this.weight = weight; |
| this.operationType = type; |
| } |
| |
| private int getWeight() { |
| return weight; |
| } |
| |
| private static final Operations[] VALUES = values(); |
| |
| private static int sumWeights(OperationType type) { |
| int sum = 0; |
| for (Operations value: VALUES) { |
| if (value.operationType == type) { |
| sum += value.getWeight(); |
| } |
| } |
| return sum; |
| } |
| |
| private static final int TOTAL_WEIGHT_FILESYSTEM = |
| sumWeights(OperationType.FileSystem); |
| |
| private static final int TOTAL_WEIGHT_SNAPSHOT = |
| sumWeights(OperationType.Snapshot); |
| |
| public static Operations getRandomOperation(OperationType type) { |
| int randomNum = 0; |
| Operations randomOperation = null; |
| switch (type) { |
| case FileSystem: |
| randomNum = generator.nextInt(TOTAL_WEIGHT_FILESYSTEM); |
| break; |
| case Snapshot: |
| randomNum = generator.nextInt(TOTAL_WEIGHT_SNAPSHOT); |
| break; |
| default: |
| break; |
| } |
| int currentWeightSum = 0; |
| for (Operations currentValue: VALUES) { |
| if (currentValue.operationType == type) { |
| if (randomNum <= (currentWeightSum + currentValue.getWeight())) { |
| randomOperation = currentValue; |
| break; |
| } |
| currentWeightSum += currentValue.getWeight(); |
| } |
| } |
| return randomOperation; |
| } |
| } |
| |
| @Before |
| public void setUp() throws Exception { |
| CONFIG.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE); |
| cluster = new MiniDFSCluster.Builder(CONFIG).numDataNodes(REPL). |
| format(true).build(); |
| cluster.waitActive(); |
| |
| hdfs = cluster.getFileSystem(); |
| hdfs.mkdirs(TESTDIR); |
| hdfs.mkdirs(WITNESSDIR); |
| } |
| |
| @After |
| public void tearDown() throws Exception { |
| if (cluster != null) { |
| cluster.shutdown(); |
| cluster = null; |
| } |
| } |
| |
| /* |
| * Random file system operations with snapshot operations in between. |
| */ |
| @Test(timeout = 900000) |
| public void testRandomOperationsWithSnapshots() |
| throws IOException, InterruptedException, TimeoutException { |
| // Set |
| long seed = System.currentTimeMillis(); |
| LOG.info("testRandomOperationsWithSnapshots, seed to be used: " + seed); |
| generator = new Random(seed); |
| |
| int fileLen = generator.nextInt(MAX_NUM_FILE_LENGTH); |
| createFiles(TESTDIRSTRING, fileLen); |
| |
| // Get list of snapshottable directories |
| SnapshottableDirectoryStatus[] snapshottableDirectoryStatus = |
| hdfs.getSnapshottableDirListing(); |
| for (SnapshottableDirectoryStatus ssds : snapshottableDirectoryStatus) { |
| snapshottableDirectories.add(ssds.getFullPath()); |
| } |
| |
| if (snapshottableDirectories.size() == 0) { |
| hdfs.allowSnapshot(hdfs.getHomeDirectory()); |
| snapshottableDirectories.add(hdfs.getHomeDirectory()); |
| } |
| |
| int numberOfIterations = generator.nextInt(MAX_NUM_ITERATIONS); |
| LOG.info("Number of iterations: " + numberOfIterations); |
| |
| int numberFileSystemOperations = generator.nextInt( |
| MAX_NUM_FILESYSTEM_OPERATIONS-MIN_NUM_OPERATIONS+1)+MIN_NUM_OPERATIONS; |
| LOG.info("Number of FileSystem operations: "+ numberFileSystemOperations); |
| |
| int numberSnapshotOperations = generator.nextInt( |
| MAX_NUM_SNAPSHOT_OPERATIONS-MIN_NUM_OPERATIONS)+MIN_NUM_OPERATIONS; |
| LOG.info("Number of Snapshot operations: " + numberSnapshotOperations); |
| |
| // Act && Verify |
| randomOperationsWithSnapshots(numberOfIterations, |
| numberFileSystemOperations, numberSnapshotOperations); |
| } |
| |
| /* |
| * Based on input we're performing: |
| * random number of file system operations |
| * random number of snapshot operations |
| * restart name node making sure fsimage can be loaded successfully. |
| */ |
| public void randomOperationsWithSnapshots(int numberOfIterations, |
| int numberFileSystemOperations, |
| int numberSnapshotOperations) |
| throws IOException, InterruptedException, TimeoutException { |
| // random number of iterations |
| for (int i = 0; i < numberOfIterations; i++) { |
| // random number of FileSystem operations |
| for (int j = 0; j < numberFileSystemOperations; j++) { |
| Operations fsOperation = |
| Operations.getRandomOperation(OperationType.FileSystem); |
| LOG.info("fsOperation: " + fsOperation); |
| switch (fsOperation) { |
| case FileSystem_CreateDir: |
| createTestDir(); |
| break; |
| |
| case FileSystem_DeleteDir: |
| deleteTestDir(); |
| break; |
| |
| case FileSystem_RenameDir: |
| renameTestDir(); |
| break; |
| |
| case FileSystem_CreateFile: |
| createTestFile(); |
| break; |
| |
| case FileSystem_DeleteFile: |
| deleteTestFile(); |
| break; |
| |
| case FileSystem_RenameFile: |
| renameTestFile(); |
| break; |
| |
| default: |
| assertNull("Invalid FileSystem operation", fsOperation); |
| break; |
| } |
| } |
| |
| // random number of Snapshot operations |
| for (int k = 0; k < numberSnapshotOperations; k++) { |
| Operations snapshotOperation = |
| Operations.getRandomOperation(OperationType.Snapshot); |
| LOG.info("snapshotOperation: " + snapshotOperation); |
| |
| switch (snapshotOperation) { |
| case Snapshot_CreateSnapshot: |
| createSnapshot(); |
| break; |
| |
| case Snapshot_DeleteSnapshot: |
| deleteSnapshot(); |
| break; |
| |
| case Snapshot_RenameSnapshot: |
| renameSnapshot(); |
| break; |
| |
| default: |
| assertNull("Invalid Snapshot operation", snapshotOperation); |
| break; |
| } |
| } |
| // Verification |
| checkClusterHealth(); |
| } |
| } |
| |
| /* Create a new test directory. */ |
| private void createTestDir() throws IOException { |
| if (snapshottableDirectories.size() > 0) { |
| int index = generator.nextInt(snapshottableDirectories.size()); |
| Path parentDir = snapshottableDirectories.get(index); |
| Path newDir = new Path(parentDir, "createTestDir_" + |
| UUID.randomUUID().toString()); |
| |
| for (OperationDirectories dir : OperationDirectories.values()) { |
| if (dir == OperationDirectories.WitnessDir) { |
| newDir = new Path(getNewPathString(newDir.toString(), |
| TESTDIRSTRING, WITNESSDIRSTRING)); |
| } |
| hdfs.mkdirs(newDir); |
| assertTrue("Directory exists", hdfs.exists(newDir)); |
| LOG.info("Directory created: " + newDir); |
| numberDirectoryCreated++; |
| } |
| } |
| } |
| |
| /* Delete an existing test directory. */ |
| private void deleteTestDir() throws IOException { |
| if (snapshottableDirectories.size() > 0) { |
| int index = generator.nextInt(snapshottableDirectories.size()); |
| Path deleteDir = snapshottableDirectories.get(index); |
| |
| if (!pathToSnapshotsMap.containsKey(deleteDir)) { |
| boolean isWitnessDir = false; |
| for (OperationDirectories dir : OperationDirectories.values()) { |
| if (dir == OperationDirectories.WitnessDir) { |
| isWitnessDir = true; |
| deleteDir = new Path(getNewPathString(deleteDir.toString(), |
| TESTDIRSTRING, WITNESSDIRSTRING)); |
| } |
| hdfs.delete(deleteDir, true); |
| assertFalse("Directory does not exist", hdfs.exists(deleteDir)); |
| if (!isWitnessDir) { |
| snapshottableDirectories.remove(deleteDir); |
| } |
| LOG.info("Directory removed: " + deleteDir); |
| numberDirectoryDeleted++; |
| } |
| } |
| } |
| } |
| |
| /* Rename an existing test directory. */ |
| private void renameTestDir() throws IOException { |
| if (snapshottableDirectories.size() > 0) { |
| int index = generator.nextInt(snapshottableDirectories.size()); |
| Path oldDir = snapshottableDirectories.get(index); |
| |
| if (!pathToSnapshotsMap.containsKey(oldDir)) { |
| Path newDir = oldDir.suffix("_renameDir+"+UUID.randomUUID().toString()); |
| for (OperationDirectories dir : OperationDirectories.values()) { |
| if (dir == OperationDirectories.WitnessDir) { |
| oldDir = new Path(getNewPathString(oldDir.toString(), |
| TESTDIRSTRING, WITNESSDIRSTRING)); |
| newDir = new Path(getNewPathString(newDir.toString(), |
| TESTDIRSTRING, WITNESSDIRSTRING)); |
| } |
| hdfs.rename(oldDir, newDir, Options.Rename.OVERWRITE); |
| assertTrue("Target directory exists", hdfs.exists(newDir)); |
| assertFalse("Source directory does not exist", |
| hdfs.exists(oldDir)); |
| |
| if (dir == OperationDirectories.TestDir) { |
| snapshottableDirectories.remove(oldDir); |
| snapshottableDirectories.add(newDir); |
| } |
| LOG.info("Renamed directory:" + oldDir + " to directory: " + newDir); |
| numberDirectoryRenamed++; |
| } |
| } |
| } |
| } |
| |
| /* Create a new snapshot. */ |
| private void createSnapshot() throws IOException { |
| if (snapshottableDirectories.size() > 0) { |
| int index = generator.nextInt(snapshottableDirectories.size()); |
| Path randomDir = snapshottableDirectories.get(index); |
| String snapshotName = Integer.toString(generator.nextInt()) + ".ss"; |
| hdfs.createSnapshot(randomDir, snapshotName); |
| LOG.info("createSnapshot, directory: " + randomDir + |
| ", snapshot name: " + snapshotName); |
| numberSnapshotCreated++; |
| |
| if (pathToSnapshotsMap.containsKey(randomDir)) { |
| pathToSnapshotsMap.get(randomDir).add(snapshotName); |
| } else { |
| pathToSnapshotsMap.put(randomDir, |
| new ArrayList<String>(Arrays.asList(snapshotName))); |
| } |
| } |
| } |
| |
| /* Delete an existing snapshot. */ |
| private void deleteSnapshot() throws IOException { |
| if (!pathToSnapshotsMap.isEmpty()) { |
| int index = generator.nextInt(pathToSnapshotsMap.size()); |
| Object[] snapshotPaths = pathToSnapshotsMap.keySet().toArray(); |
| Path snapshotPath = (Path)snapshotPaths[index]; |
| ArrayList<String> snapshotNameList=pathToSnapshotsMap.get(snapshotPath); |
| |
| String snapshotNameToBeDeleted = snapshotNameList.get( |
| generator.nextInt(snapshotNameList.size())); |
| hdfs.deleteSnapshot(snapshotPath, snapshotNameToBeDeleted); |
| LOG.info("deleteSnapshot, directory: " + snapshotPath + |
| ", snapshot name: " + snapshotNameToBeDeleted); |
| numberSnapshotDeleted++; |
| |
| // Adjust pathToSnapshotsMap after snapshot deletion |
| if (snapshotNameList.size() == 1) { |
| pathToSnapshotsMap.remove(snapshotPath); |
| } else { |
| pathToSnapshotsMap.get(snapshotPath).remove(snapshotNameToBeDeleted); |
| } |
| } |
| } |
| |
| /* Rename an existing snapshot. */ |
| private void renameSnapshot() throws IOException { |
| if (!pathToSnapshotsMap.isEmpty()) { |
| int index = generator.nextInt(pathToSnapshotsMap.size()); |
| Object[] snapshotPaths = pathToSnapshotsMap.keySet().toArray(); |
| Path snapshotPath = (Path)snapshotPaths[index]; |
| ArrayList<String> snapshotNameList = |
| pathToSnapshotsMap.get(snapshotPath); |
| |
| String snapshotOldName = snapshotNameList.get( |
| generator.nextInt(snapshotNameList.size())); |
| |
| String snapshotOldNameNoExt = snapshotOldName.substring(0, |
| snapshotOldName.lastIndexOf('.')-1); |
| |
| String snapshotNewName = snapshotOldNameNoExt + "_rename.ss"; |
| hdfs.renameSnapshot(snapshotPath, snapshotOldName, snapshotNewName); |
| LOG.info("renameSnapshot, directory:" + snapshotPath + ", snapshot name:" |
| + snapshotOldName + " to " + snapshotNewName); |
| numberSnapshotRenamed++; |
| |
| // Adjust pathToSnapshotsMap after snapshot deletion |
| pathToSnapshotsMap.get(snapshotPath).remove(snapshotOldName); |
| pathToSnapshotsMap.get(snapshotPath).add(snapshotNewName); |
| } |
| } |
| |
| /* Create a new test file. */ |
| private void createTestFile() throws IOException { |
| if (snapshottableDirectories.size() > 0) { |
| int index = generator.nextInt(snapshottableDirectories.size()); |
| Path randomDir = snapshottableDirectories.get(index); |
| if (!randomDir.isRoot()) { |
| randomDir = randomDir.getParent(); |
| } |
| |
| Path newFile = new Path(randomDir, "createTestFile.log"); |
| for (OperationDirectories dir : OperationDirectories.values()) { |
| if (dir == OperationDirectories.WitnessDir) { |
| newFile = new Path(getNewPathString(newFile.toString(), |
| TESTDIRSTRING, WITNESSDIRSTRING)); |
| } |
| hdfs.createNewFile(newFile); |
| assertTrue("File exists", hdfs.exists(newFile)); |
| LOG.info("createTestFile, file created: " + newFile); |
| numberFileCreated++; |
| } |
| } |
| } |
| |
| /* Delete an existing test file. */ |
| private void deleteTestFile() throws IOException { |
| if (snapshottableDirectories.size() > 0) { |
| int index = generator.nextInt(snapshottableDirectories.size()); |
| Path randomDir = snapshottableDirectories.get(index); |
| |
| FileStatus[] fileStatusList = hdfs.listStatus(randomDir); |
| for (FileStatus fsEntry : fileStatusList) { |
| if (fsEntry.isFile()) { |
| Path deleteFile = fsEntry.getPath(); |
| for (OperationDirectories dir : OperationDirectories.values()) { |
| if (dir == OperationDirectories.WitnessDir) { |
| deleteFile = new Path(getNewPathString(deleteFile.toString(), |
| TESTDIRSTRING, WITNESSDIRSTRING)); |
| } |
| hdfs.delete(deleteFile, false); |
| assertFalse("File does not exists", |
| hdfs.exists(deleteFile)); |
| LOG.info("deleteTestFile, file deleted: " + deleteFile); |
| numberFileDeleted++; |
| } |
| break; |
| } |
| } |
| } |
| } |
| |
| /* Rename an existing test file. */ |
| private void renameTestFile() throws IOException { |
| if (snapshottableDirectories.size() > 0) { |
| int index = generator.nextInt(snapshottableDirectories.size()); |
| Path randomDir = snapshottableDirectories.get(index); |
| |
| FileStatus[] fileStatusList = hdfs.listStatus(randomDir); |
| for (FileStatus fsEntry : fileStatusList) { |
| if (fsEntry.isFile()) { |
| Path oldFile = fsEntry.getPath(); |
| Path newFile = oldFile.suffix("_renameFile"); |
| for (OperationDirectories dir : OperationDirectories.values()) { |
| if (dir == OperationDirectories.WitnessDir) { |
| oldFile = new Path(getNewPathString(oldFile.toString(), |
| TESTDIRSTRING, WITNESSDIRSTRING)); |
| newFile = new Path(getNewPathString(newFile.toString(), |
| TESTDIRSTRING, WITNESSDIRSTRING)); |
| } |
| |
| hdfs.rename(oldFile, newFile, Options.Rename.OVERWRITE); |
| assertTrue("Target file exists", hdfs.exists(newFile)); |
| assertFalse("Source file does not exist", hdfs.exists(oldFile)); |
| LOG.info("Renamed file: " + oldFile + " to file: " + newFile); |
| numberFileRenamed++; |
| } |
| break; |
| } |
| } |
| } |
| } |
| |
| /* Check cluster health. */ |
| private void checkClusterHealth() throws IOException, InterruptedException, |
| TimeoutException { |
| // 1. FileStatus comparison between test and witness directory |
| FileStatus[] testDirStatus = hdfs.listStatus(TESTDIR); |
| FileStatus[] witnessDirStatus = hdfs.listStatus(WITNESSDIR); |
| assertEquals(witnessDirStatus.length, testDirStatus.length); |
| LOG.info("checkClusterHealth, number of entries verified."); |
| |
| Arrays.sort(testDirStatus); |
| Arrays.sort(witnessDirStatus); |
| for (int i = 0; i < testDirStatus.length; i++) { |
| assertEquals(witnessDirStatus[i].getPermission(), |
| testDirStatus[i].getPermission()); |
| assertEquals(witnessDirStatus[i].getOwner(), |
| testDirStatus[i].getOwner()); |
| assertEquals(witnessDirStatus[i].getGroup(), |
| testDirStatus[i].getGroup()); |
| assertEquals(witnessDirStatus[i].getLen(), testDirStatus[i].getLen()); |
| assertEquals(witnessDirStatus[i].getBlockSize(), |
| testDirStatus[i].getBlockSize()); |
| assertEquals(witnessDirStatus[i].hasAcl(), testDirStatus[i].hasAcl()); |
| assertEquals(witnessDirStatus[i].isEncrypted(), |
| testDirStatus[i].isEncrypted()); |
| assertEquals(witnessDirStatus[i].isErasureCoded(), |
| testDirStatus[i].isErasureCoded()); |
| assertEquals(witnessDirStatus[i].isDirectory(), |
| testDirStatus[i].isDirectory()); |
| assertEquals(witnessDirStatus[i].isFile(), testDirStatus[i].isFile()); |
| } |
| LOG.info("checkClusterHealth, metadata verified."); |
| |
| // Randomly decide whether we want to do a check point |
| if (generator.nextBoolean()) { |
| LOG.info("checkClusterHealth, doing a checkpoint on NN."); |
| hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); |
| hdfs.saveNamespace(); |
| hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); |
| } |
| |
| /** Restart name node making sure loading from image successfully */ |
| LOG.info("checkClusterHealth, restarting NN."); |
| cluster.restartNameNodes(); |
| GenericTestUtils.waitFor(new Supplier<Boolean>() { |
| @Override |
| public Boolean get() { |
| return !cluster.getNameNode().isInSafeMode(); |
| } |
| }, 10, 100000); |
| |
| assertTrue("NameNode is up", cluster.getNameNode().isActiveState()); |
| assertTrue("DataNode is up and running", cluster.isDataNodeUp()); |
| assertTrue("Cluster is up and running", cluster.isClusterUp()); |
| LOG.info("checkClusterHealth, cluster is healthy."); |
| |
| printOperationStats(); |
| } |
| |
| /* Helper: create a file with a length of filelen. */ |
| private void createFile(final String fileName, final long filelen, |
| boolean enableSnapshot) throws IOException { |
| FileSystem fs = cluster.getFileSystem(); |
| Path filePath = new Path(fileName); |
| DFSTestUtil.createFile(fs, filePath, filelen, (short) 1, 0); |
| |
| // Randomly allow snapshot on parent directly |
| if (enableSnapshot && generator.nextBoolean()) { |
| try { |
| hdfs.allowSnapshot(filePath.getParent()); |
| } catch (SnapshotException e) { |
| LOG.info("createFile, exception setting snapshotable directory: " + |
| e.getMessage()); |
| } |
| } |
| } |
| |
| /* Helper: create a large number of directories and files. */ |
| private void createFiles(String rootDir, int fileLength) throws IOException { |
| if (!rootDir.endsWith("/")) { |
| rootDir += "/"; |
| } |
| |
| // Create files in a directory with random depth, ranging from 0-10. |
| for (int i = 0; i < TOTAL_FILECOUNT; i++) { |
| String filename = rootDir; |
| int dirs = generator.nextInt(MAX_NUM_SUB_DIRECTORIES_LEVEL); |
| |
| for (int j=i; j >=(i-dirs); j--) { |
| filename += j + "/"; |
| } |
| filename += "file" + i; |
| createFile(filename, fileLength, true); |
| assertTrue("Test file created", hdfs.exists(new Path(filename))); |
| LOG.info("createFiles, file: " + filename + "was created"); |
| |
| String witnessFile = |
| filename.replaceAll(TESTDIRSTRING, WITNESSDIRSTRING); |
| createFile(witnessFile, fileLength, false); |
| assertTrue("Witness file exists", |
| hdfs.exists(new Path(witnessFile))); |
| LOG.info("createFiles, file: " + witnessFile + "was created"); |
| } |
| } |
| |
| /* Helper: replace all target string with replacement string in original |
| * string. */ |
| private String getNewPathString(String originalString, String targetString, |
| String replacementString) { |
| String str = originalString.replaceAll(targetString, replacementString); |
| LOG.info("Original string: " + originalString); |
| LOG.info("New string: " + str); |
| return str; |
| } |
| |
| private void printOperationStats() { |
| LOG.info("Operation statistics for this iteration: "); |
| |
| LOG.info("Number of files created: " + numberFileCreated); |
| LOG.info("Number of files deleted: " + numberFileDeleted); |
| LOG.info("Number of files renamed: " + numberFileRenamed); |
| |
| LOG.info("Number of directories created: " + numberDirectoryCreated); |
| LOG.info("Number of directories deleted: " + numberDirectoryDeleted); |
| LOG.info("Number of directories renamed: " + numberDirectoryRenamed); |
| |
| LOG.info("Number of snapshots created: " + numberSnapshotCreated); |
| LOG.info("Number of snapshots deleted: " + numberSnapshotDeleted); |
| LOG.info("Number of snapshots renamed: " + numberSnapshotRenamed); |
| |
| numberFileCreated = 0; |
| numberFileDeleted = 0; |
| numberFileRenamed = 0; |
| |
| numberDirectoryCreated = 0; |
| numberDirectoryDeleted = 0; |
| numberDirectoryRenamed = 0; |
| |
| numberSnapshotCreated = 0; |
| numberSnapshotDeleted = 0; |
| numberSnapshotRenamed = 0; |
| } |
| } |
