| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.mapreduce.util; |
| |
| import java.io.File; |
| import java.io.IOException; |
| |
| import junit.framework.TestCase; |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.fs.FileSystem; |
| import org.apache.hadoop.fs.Path; |
| import org.apache.hadoop.mapreduce.util.MRAsyncDiskService; |
| import org.junit.Test; |
| |
| /** |
| * A test for MRAsyncDiskService. |
| */ |
| public class TestMRAsyncDiskService extends TestCase { |
| |
| private static String TEST_ROOT_DIR = new Path(System.getProperty( |
| "test.build.data", "/tmp")).toString(); |
| |
| /** |
| * This test creates some directories and then removes them through |
| * MRAsyncDiskService. |
| */ |
| @Test |
| public void testMRAsyncDiskService() throws Throwable { |
| |
| FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); |
| String[] vols = new String[]{TEST_ROOT_DIR + "/0", |
| TEST_ROOT_DIR + "/1"}; |
| MRAsyncDiskService service = new MRAsyncDiskService( |
| localFileSystem, vols); |
| |
| String a = "a"; |
| String b = "b"; |
| String c = "b/c"; |
| String d = "d"; |
| |
| File fa = new File(vols[0], a); |
| File fb = new File(vols[1], b); |
| File fc = new File(vols[1], c); |
| File fd = new File(vols[1], d); |
| |
| // Create the directories |
| fa.mkdirs(); |
| fb.mkdirs(); |
| fc.mkdirs(); |
| fd.mkdirs(); |
| |
| assertTrue(fa.exists()); |
| assertTrue(fb.exists()); |
| assertTrue(fc.exists()); |
| assertTrue(fd.exists()); |
| |
| // Move and delete them |
| service.moveAndDeleteRelativePath(vols[0], a); |
| assertFalse(fa.exists()); |
| service.moveAndDeleteRelativePath(vols[1], b); |
| assertFalse(fb.exists()); |
| assertFalse(fc.exists()); |
| |
| // asyncDiskService is NOT able to delete files outside all volumes. |
| IOException ee = null; |
| try { |
| service.moveAndDeleteAbsolutePath(TEST_ROOT_DIR + "/2"); |
| } catch (IOException e) { |
| ee = e; |
| } |
| assertNotNull("asyncDiskService should not be able to delete files " |
| + "outside all volumes", ee); |
| // asyncDiskService is able to automatically find the file in one |
| // of the volumes. |
| assertTrue(service.moveAndDeleteAbsolutePath(vols[1] + Path.SEPARATOR_CHAR + d)); |
| |
| // Make sure everything is cleaned up |
| makeSureCleanedUp(vols, service); |
| } |
| |
| /** |
| * This test creates some directories inside the volume roots, and then |
| * call asyncDiskService.MoveAndDeleteAllVolumes. |
| * We should be able to delete all files/dirs inside the volumes except |
| * the toBeDeleted directory. |
| */ |
| @Test |
| public void testMRAsyncDiskServiceMoveAndDeleteAllVolumes() throws Throwable { |
| FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); |
| String[] vols = new String[]{TEST_ROOT_DIR + "/0", |
| TEST_ROOT_DIR + "/1"}; |
| MRAsyncDiskService service = new MRAsyncDiskService( |
| localFileSystem, vols); |
| |
| String a = "a"; |
| String b = "b"; |
| String c = "b/c"; |
| String d = "d"; |
| |
| File fa = new File(vols[0], a); |
| File fb = new File(vols[1], b); |
| File fc = new File(vols[1], c); |
| File fd = new File(vols[1], d); |
| |
| // Create the directories |
| fa.mkdirs(); |
| fb.mkdirs(); |
| fc.mkdirs(); |
| fd.mkdirs(); |
| |
| assertTrue(fa.exists()); |
| assertTrue(fb.exists()); |
| assertTrue(fc.exists()); |
| assertTrue(fd.exists()); |
| |
| // Delete all of them |
| service.cleanupAllVolumes(); |
| |
| assertFalse(fa.exists()); |
| assertFalse(fb.exists()); |
| assertFalse(fc.exists()); |
| assertFalse(fd.exists()); |
| |
| // Make sure everything is cleaned up |
| makeSureCleanedUp(vols, service); |
| } |
| |
| /** |
| * This test creates some directories inside the toBeDeleted directory and |
| * then start the asyncDiskService. |
| * AsyncDiskService will create tasks to delete the content inside the |
| * toBeDeleted directories. |
| */ |
| @Test |
| public void testMRAsyncDiskServiceStartupCleaning() throws Throwable { |
| FileSystem localFileSystem = FileSystem.getLocal(new Configuration()); |
| String[] vols = new String[]{TEST_ROOT_DIR + "/0", |
| TEST_ROOT_DIR + "/1"}; |
| |
| String a = "a"; |
| String b = "b"; |
| String c = "b/c"; |
| String d = "d"; |
| |
| // Create directories inside SUBDIR |
| File fa = new File(vols[0] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, a); |
| File fb = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, b); |
| File fc = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, c); |
| File fd = new File(vols[1] + Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED, d); |
| |
| // Create the directories |
| fa.mkdirs(); |
| fb.mkdirs(); |
| fc.mkdirs(); |
| fd.mkdirs(); |
| |
| assertTrue(fa.exists()); |
| assertTrue(fb.exists()); |
| assertTrue(fc.exists()); |
| assertTrue(fd.exists()); |
| |
| // Create the asyncDiskService which will delete all contents inside SUBDIR |
| MRAsyncDiskService service = new MRAsyncDiskService( |
| localFileSystem, vols); |
| |
| // Make sure everything is cleaned up |
| makeSureCleanedUp(vols, service); |
| } |
| |
| private void makeSureCleanedUp(String[] vols, MRAsyncDiskService service) |
| throws Throwable { |
| // Sleep at most 5 seconds to make sure the deleted items are all gone. |
| service.shutdown(); |
| if (!service.awaitTermination(5000)) { |
| fail("MRAsyncDiskService is still not shutdown in 5 seconds!"); |
| } |
| |
| // All contents should be gone by now. |
| for (int i = 0; i < vols.length; i++) { |
| File subDir = new File(vols[0]); |
| String[] subDirContent = subDir.list(); |
| assertEquals("Volume should contain a single child: " |
| + MRAsyncDiskService.TOBEDELETED, 1, subDirContent.length); |
| |
| File toBeDeletedDir = new File(vols[0], MRAsyncDiskService.TOBEDELETED); |
| String[] content = toBeDeletedDir.list(); |
| assertNotNull("Cannot find " + toBeDeletedDir, content); |
| assertEquals("" + toBeDeletedDir + " should be empty now.", 0, |
| content.length); |
| } |
| } |
| |
| } |