MAPREDUCE-4349. Add test to verify Distributed Cache consistency when cached archives are deleted. Contributed by Mayank Bansal.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.22@1365292 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/mapreduce/CHANGES.txt b/mapreduce/CHANGES.txt
index cf6d062..4c146a5 100644
--- a/mapreduce/CHANGES.txt
+++ b/mapreduce/CHANGES.txt
@@ -17,6 +17,9 @@
MAPREDUCE-4405. Test case for HierarchicalQueue in TestJobQueueClient.
(Mayank Bansal via shv)
+ MAPREDUCE-4349. Add test to verify Distributed Cache consistency when
+ cached archives are deleted. (Mayank Bansal via shv)
+
OPTIMIZATIONS
BUG FIXES
diff --git a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java
index e65e968..59af1d8 100644
--- a/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java
+++ b/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java
@@ -46,7 +46,6 @@
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.filecache.TaskDistributedCacheManager.CacheFile;
-import org.apache.hadoop.mapreduce.filecache.TrackerDistributedCacheManager.CacheStatus;
import org.apache.hadoop.mapreduce.filecache.TestTrackerDistributedCacheManager;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -227,7 +226,8 @@
* @throws InterruptedException
*/
@SuppressWarnings("deprecation")
- public void testCacheConsistency() throws IOException, LoginException, InterruptedException {
+ public void testCacheConsistency() throws IOException, LoginException,
+ InterruptedException {
if (!canRun()) {
return;
}
@@ -238,6 +238,7 @@
subConf.set(MRJobConfig.USER_NAME, userName);
JobID jobid = new JobID("jt", 1);
DistributedCache.addCacheFile(firstCacheFile.toUri(), subConf);
+ DistributedCache.addCacheArchive(secondCacheFile.toUri(), subConf);
TrackerDistributedCacheManager.determineTimestamps(subConf);
TrackerDistributedCacheManager.determineCacheVisibilities(subConf);
// ****** End of imitating JobClient code
@@ -253,6 +254,7 @@
TaskDistributedCacheManager handle = manager
.newTaskDistributedCacheManager(jobid, subConf);
assertNull(null, DistributedCache.getLocalCacheFiles(subConf));
+ assertNull(null, DistributedCache.getLocalCacheArchives(subConf));
handle.setupCache(subConf, TaskTracker.getPublicDistributedCacheDir(),
TaskTracker.getPrivateDistributedCacheDir(userName));
JobLocalizer.downloadPrivateCache(subConf);
@@ -269,28 +271,52 @@
File f1 = new File(cachedFirstFile.toString());
assertTrue(f1.delete());
+ Path[] localCacheArchiveFiles = DistributedCache
+ .getLocalCacheArchives(subConf);
+ assertNotNull(null, localCacheArchiveFiles);
+ assertEquals(1, localCacheArchiveFiles.length);
+ Path cachedSecondFile = localCacheArchiveFiles[0];
+ assertFileLengthEquals(secondCacheFile, new Path(cachedSecondFile,
+ "secondcachefile"));
+ assertFalse("Paths should be different.",
+ secondCacheFile.equals(cachedSecondFile));
+ File f2 = new File(cachedSecondFile.toString());
+ FileUtil.fullyDelete(f2);
+ assertFalse(f2.exists());
// ****** Imitate JobClient code
// Configures a task/job with both a regular file and a "classpath" file.
subConf = new Configuration(conf);
userName = getJobOwnerName();
subConf.set(MRJobConfig.USER_NAME, userName);
DistributedCache.addCacheFile(firstCacheFile.toUri(), subConf);
+ DistributedCache.addCacheArchive(secondCacheFile.toUri(), subConf);
TrackerDistributedCacheManager.determineTimestamps(subConf);
TrackerDistributedCacheManager.determineCacheVisibilities(subConf);
JobID jobidnew = new JobID("jt", 2);
handle = manager.newTaskDistributedCacheManager(jobidnew, subConf);
assertNull(null, DistributedCache.getLocalCacheFiles(subConf));
+ assertNull(null, DistributedCache.getLocalCacheArchives(subConf));
handle.setupCache(subConf, TaskTracker.getPublicDistributedCacheDir(),
TaskTracker.getPrivateDistributedCacheDir(userName));
JobLocalizer.downloadPrivateCache(subConf);
// ****** End of imitating TaskRunner code
Path[] localCacheFilesagain = DistributedCache.getLocalCacheFiles(subConf);
+ Path[] localCacheArchivesagain = DistributedCache
+ .getLocalCacheArchives(subConf);
assertNotNull(null, localCacheFilesagain);
+ assertNotNull(null, localCacheArchivesagain);
assertEquals(1, localCacheFilesagain.length);
+ assertEquals(1, localCacheArchivesagain.length);
Path cachedFirstFileAgain = localCacheFilesagain[0];
assertFileLengthEquals(firstCacheFile, cachedFirstFileAgain);
assertFalse("Paths should be different.",
firstCacheFile.equals(cachedFirstFileAgain));
+
+ Path cachedSecondFileAgain = localCacheArchivesagain[0];
+ assertFileLengthEquals(secondCacheFile, new Path(cachedSecondFileAgain,
+ "secondcachefile"));
+ assertFalse("Paths should be different.",
+ secondCacheFile.equals(cachedSecondFileAgain));
}
/**
