| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.mapreduce.util; |
| |
| import java.io.File; |
| import java.io.IOException; |
| |
| import junit.framework.TestCase; |
| |
| import org.apache.commons.logging.Log; |
| import org.apache.commons.logging.LogFactory; |
| |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.fs.FileSystem; |
| import org.apache.hadoop.fs.Path; |
| import org.apache.hadoop.fs.FileUtil; |
| import org.apache.hadoop.mapreduce.util.MRAsyncDiskService; |
| import org.junit.Test; |
| |
| /** |
| * A test for MRAsyncDiskService. |
| */ |
| public class TestMRAsyncDiskService extends TestCase { |
| |
| public static final Log LOG = LogFactory.getLog(TestMRAsyncDiskService.class); |
| |
| private static String TEST_ROOT_DIR = new Path(System.getProperty( |
| "test.build.data", "/tmp")).toString(); |
| |
| @Override |
| protected void setUp() throws Exception { |
| FileUtil.fullyDelete(new File(TEST_ROOT_DIR)); |
| } |
| |
| /** |
| * Given 'pathname', compute an equivalent path relative to the cwd. |
| * @param pathname the path to a directory. |
| * @return the path to that same directory, relative to ${user.dir}. |
| */ |
| private String relativeToWorking(String pathname) { |
| String cwd = System.getProperty("user.dir", "/"); |
| |
| // normalize pathname and cwd into full directory paths. |
| pathname = (new Path(pathname)).toUri().getPath(); |
| cwd = (new Path(cwd)).toUri().getPath(); |
| |
| String [] cwdParts = cwd.split(File.separator); |
| String [] pathParts = pathname.split(File.separator); |
| |
| // There are three possible cases: |
| // 1) pathname and cwd are equal. Return '.' |
| // 2) pathname is under cwd. Return the components that are under it. |
| // e.g., cwd = /a/b, path = /a/b/c, return 'c' |
| // 3) pathname is outside of cwd. Find the common components, if any, |
| // and subtract them from the returned path, then return enough '..' |
| // components to "undo" the non-common components of cwd, then all |
| // the remaining parts of pathname. |
| // e.g., cwd = /a/b, path = /a/c, return '../c' |
| |
| if (cwd.equals(pathname)) { |
| LOG.info("relative to working: " + pathname + " -> ."); |
| return "."; // They match exactly. |
| } |
| |
| // Determine how many path components are in common between cwd and path. |
| int common = 0; |
| for (int i = 0; i < Math.min(cwdParts.length, pathParts.length); i++) { |
| if (cwdParts[i].equals(pathParts[i])) { |
| common++; |
| } else { |
| break; |
| } |
| } |
| |
| // output path stringbuilder. |
| StringBuilder sb = new StringBuilder(); |
| |
| // For everything in cwd that isn't in pathname, add a '..' to undo it. |
| int parentDirsRequired = cwdParts.length - common; |
| for (int i = 0; i < parentDirsRequired; i++) { |
| sb.append(".."); |
| sb.append(File.separator); |
| } |
| |
| // Then append all non-common parts of 'pathname' itself. |
| for (int i = common; i < pathParts.length; i++) { |
| sb.append(pathParts[i]); |
| sb.append(File.separator); |
| } |
| |
| // Don't end with a '/'. |
| String s = sb.toString(); |
| if (s.endsWith(File.separator)) { |
| s = s.substring(0, s.length() - 1); |
| } |
| |
| LOG.info("relative to working: " + pathname + " -> " + s); |
| return s; |
| } |
| |
| @Test |
| /** Test that the relativeToWorking() method above does what we expect. */ |
| public void testRelativeToWorking() { |
| assertEquals(".", relativeToWorking(System.getProperty("user.dir", "."))); |
| |
| String cwd = System.getProperty("user.dir", "."); |
| Path cwdPath = new Path(cwd); |
| |
| Path subdir = new Path(cwdPath, "foo"); |
| assertEquals("foo", relativeToWorking(subdir.toUri().getPath())); |
| |
| Path subsubdir = new Path(subdir, "bar"); |
| assertEquals("foo/bar", relativeToWorking(subsubdir.toUri().getPath())); |
| |
| Path parent = new Path(cwdPath, ".."); |
| assertEquals("..", relativeToWorking(parent.toUri().getPath())); |
| |
| Path sideways = new Path(parent, "baz"); |
| assertEquals("../baz", relativeToWorking(sideways.toUri().getPath())); |
| } |
| |
| |
| @Test |
| /** Test that volumes specified as relative paths are handled properly |
| * by MRAsyncDiskService (MAPREDUCE-1887). |
| */ |
| public void testVolumeNormalization() throws Throwable { |
| LOG.info("TEST_ROOT_DIR is " + TEST_ROOT_DIR); |
| |
| String relativeTestRoot = relativeToWorking(TEST_ROOT_DIR); |
| |
| FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); |
| String [] vols = new String[] { relativeTestRoot + "/0", |
| relativeTestRoot + "/1" }; |
| |
| // Put a file in one of the volumes to be cleared on startup. |
| Path delDir = new Path(vols[0], MRAsyncDiskService.TOBEDELETED); |
| localFileSystem.mkdirs(delDir); |
| localFileSystem.create(new Path(delDir, "foo")).close(); |
| |
| MRAsyncDiskService service = new MRAsyncDiskService( |
| localFileSystem, vols); |
| makeSureCleanedUp(vols, service); |
| } |
| |
| /** |
| * This test creates some directories and then removes them through |
| * MRAsyncDiskService. |
| */ |
| @Test |
| public void testMRAsyncDiskService() throws Throwable { |
| |
| FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); |
| String[] vols = new String[]{TEST_ROOT_DIR + "/0", |
| TEST_ROOT_DIR + "/1"}; |
| MRAsyncDiskService service = new MRAsyncDiskService( |
| localFileSystem, vols); |
| |
| String a = "a"; |
| String b = "b"; |
| String c = "b/c"; |
| String d = "d"; |
| |
| File fa = new File(vols[0], a); |
| File fb = new File(vols[1], b); |
| File fc = new File(vols[1], c); |
| File fd = new File(vols[1], d); |
| |
| // Create the directories |
| fa.mkdirs(); |
| fb.mkdirs(); |
| fc.mkdirs(); |
| fd.mkdirs(); |
| |
| assertTrue(fa.exists()); |
| assertTrue(fb.exists()); |
| assertTrue(fc.exists()); |
| assertTrue(fd.exists()); |
| |
| // Move and delete them |
| service.moveAndDeleteRelativePath(vols[0], a); |
| assertFalse(fa.exists()); |
| service.moveAndDeleteRelativePath(vols[1], b); |
| assertFalse(fb.exists()); |
| assertFalse(fc.exists()); |
| |
| assertFalse(service.moveAndDeleteRelativePath(vols[1], "not_exists")); |
| |
| // asyncDiskService is NOT able to delete files outside all volumes. |
| IOException ee = null; |
| try { |
| service.moveAndDeleteAbsolutePath(TEST_ROOT_DIR + "/2"); |
| } catch (IOException e) { |
| ee = e; |
| } |
| assertNotNull("asyncDiskService should not be able to delete files " |
| + "outside all volumes", ee); |
| // asyncDiskService is able to automatically find the file in one |
| // of the volumes. |
| assertTrue(service.moveAndDeleteAbsolutePath(vols[1] + Path.SEPARATOR_CHAR + d)); |
| |
| // Make sure everything is cleaned up |
| makeSureCleanedUp(vols, service); |
| } |
| |
| /** |
| * This test creates some directories inside the volume roots, and then |
| * call asyncDiskService.MoveAndDeleteAllVolumes. |
| * We should be able to delete all files/dirs inside the volumes except |
| * the toBeDeleted directory. |
| */ |
| @Test |
| public void testMRAsyncDiskServiceMoveAndDeleteAllVolumes() throws Throwable { |
| FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); |
| String[] vols = new String[]{TEST_ROOT_DIR + "/0", |
| TEST_ROOT_DIR + "/1"}; |
| MRAsyncDiskService service = new MRAsyncDiskService( |
| localFileSystem, vols); |
| |
| String a = "a"; |
| String b = "b"; |
| String c = "b/c"; |
| String d = "d"; |
| |
| File fa = new File(vols[0], a); |
| File fb = new File(vols[1], b); |
| File fc = new File(vols[1], c); |
| File fd = new File(vols[1], d); |
| |
| // Create the directories |
| fa.mkdirs(); |
| fb.mkdirs(); |
| fc.mkdirs(); |
| fd.mkdirs(); |
| |
| assertTrue(fa.exists()); |
| assertTrue(fb.exists()); |
| assertTrue(fc.exists()); |
| assertTrue(fd.exists()); |
| |
| // Delete all of them |
| service.cleanupAllVolumes(); |
| |
| assertFalse(fa.exists()); |
| assertFalse(fb.exists()); |
| assertFalse(fc.exists()); |
| assertFalse(fd.exists()); |
| |
| // Make sure everything is cleaned up |
| makeSureCleanedUp(vols, service); |
| } |
| |
| /** |
| * This test creates some directories inside the toBeDeleted directory and |
| * then start the asyncDiskService. |
| * AsyncDiskService will create tasks to delete the content inside the |
| * toBeDeleted directories. |
| */ |
| @Test |
| public void testMRAsyncDiskServiceStartupCleaning() throws Throwable { |
| FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); |
| String[] vols = new String[]{TEST_ROOT_DIR + "/0", |
| TEST_ROOT_DIR + "/1"}; |
| |
| String a = "a"; |
| String b = "b"; |
| String c = "b/c"; |
| String d = "d"; |
| |
| // Create directories inside SUBDIR |
| String suffix = Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED; |
| File fa = new File(vols[0] + suffix, a); |
| File fb = new File(vols[1] + suffix, b); |
| File fc = new File(vols[1] + suffix, c); |
| File fd = new File(vols[1] + suffix, d); |
| |
| // Create the directories |
| fa.mkdirs(); |
| fb.mkdirs(); |
| fc.mkdirs(); |
| fd.mkdirs(); |
| |
| assertTrue(fa.exists()); |
| assertTrue(fb.exists()); |
| assertTrue(fc.exists()); |
| assertTrue(fd.exists()); |
| |
| // Create the asyncDiskService which will delete all contents inside SUBDIR |
| MRAsyncDiskService service = new MRAsyncDiskService( |
| localFileSystem, vols); |
| |
| // Make sure everything is cleaned up |
| makeSureCleanedUp(vols, service); |
| } |
| |
| private void makeSureCleanedUp(String[] vols, MRAsyncDiskService service) |
| throws Throwable { |
| // Sleep at most 5 seconds to make sure the deleted items are all gone. |
| service.shutdown(); |
| if (!service.awaitTermination(5000)) { |
| fail("MRAsyncDiskService is still not shutdown in 5 seconds!"); |
| } |
| |
| // All contents should be gone by now. |
| for (int i = 0; i < vols.length; i++) { |
| File subDir = new File(vols[0]); |
| String[] subDirContent = subDir.list(); |
| assertEquals("Volume should contain a single child: " |
| + MRAsyncDiskService.TOBEDELETED, 1, subDirContent.length); |
| |
| File toBeDeletedDir = new File(vols[0], MRAsyncDiskService.TOBEDELETED); |
| String[] content = toBeDeletedDir.list(); |
| assertNotNull("Cannot find " + toBeDeletedDir, content); |
| assertEquals("" + toBeDeletedDir + " should be empty now.", 0, |
| content.length); |
| } |
| } |
| |
| @Test |
| public void testToleratesSomeUnwritableVolumes() throws Throwable { |
| FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); |
| String[] vols = new String[]{TEST_ROOT_DIR + "/0", |
| TEST_ROOT_DIR + "/1"}; |
| |
| assertTrue(new File(vols[0]).mkdirs()); |
| assertEquals(0, FileUtil.chmod(vols[0], "400")); // read only |
| try { |
| new MRAsyncDiskService(localFileSystem, vols); |
| } finally { |
| FileUtil.chmod(vols[0], "755"); // make writable again |
| } |
| } |
| |
| } |