Adding missing new files
git-svn-id: https://svn.apache.org/repos/asf/hadoop/mapreduce/branches/yahoo-merge@1082642 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java
new file mode 100644
index 0000000..bdaa4af
--- /dev/null
+++ b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSAndLocalFSDCFiles.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the emulation of HDFS and Local FS distributed cache files against
+ * the given input trace file.
+ */
+public class TestEmulationOfHDFSAndLocalFSDCFiles extends
+ GridmixSystemTestCase {
+ private static final Log LOG = LogFactory.
+ getLog("TestEmulationOfLocalFSDCFiles.class");
+ private static final String MAPREDUCE_DELEGATION_TOKEN =
+ "mapreduce.job.complete.cancel.delegation.tokens";
+
+ /**
+ * Generate the input data and distributed cache files for HDFS and
+ * local FS. Verify the gridmix emulation of HDFS and Local FS
+ * distributed cache files in RoundRobinUserResolver mode with STRESS
+ * submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGenerateDataEmulateHDFSAndLocalFSDCFiles()
+ throws Exception {
+ final long INPUT_SIZE = 1024 * 6;
+ final String tracePath = getTraceFile("distcache_case8_trace");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ INPUT_SIZE+"m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+ final String [] otherArgs = {
+ "-D",
+ MAPREDUCE_DELEGATION_TOKEN +
+ "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+ "=false"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX);
+ }
+
+ /**
+ * Use existing input and distributed cache files for HDFS and
+ * local FS. Verify the gridmix emulation of HDFS and Local FS
+ * distributed cache files in SubmitterUserResolver mode with REPLAY
+ * submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testEmulationOfHDFSAndLocalFSDCFiles()
+ throws Exception {
+ final String tracePath = getTraceFile("distcache_case8_trace");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ tracePath};
+ final String [] otherArgs = {
+ "-D", MAPREDUCE_DELEGATION_TOKEN +
+ "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+ "=false"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.RUN_GRIDMIX);
+ }
+}
diff --git a/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java
new file mode 100644
index 0000000..c7713da
--- /dev/null
+++ b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFileUsesMultipleJobs.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the Gridmix emulation of HDFS distributed cache file which uses
+ * different jobs that are submitted with different users.
+ */
+public class TestEmulationOfHDFSDCFileUsesMultipleJobs extends
+ GridmixSystemTestCase {
+ private static final Log LOG = LogFactory.
+ getLog("TestEmulationOfHDFSDCFileUsesMultipleJobs.class");
+
+ /**
+ * Generate the input data and HDFS distributed cache file based
+ * on given input trace. Verify the Gridmix emulation of HDFS
+ * distributed cache file in RoundRobinResolver mode with
+ * STRESS submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGenerateAndEmulationOfHDFSDCFile()
+ throws Exception {
+ final long INPUT_SIZE = 1024 * 6;
+ final String tracePath = getTraceFile("distcache_case9_trace");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ INPUT_SIZE+"m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+ final String [] otherArgs = {
+ "-D",
+ "mapreduce.job.complete.cancel.delegation.tokens=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX);
+ }
+
+ /**
+ * Verify the Gridmix emulation of HDFS distributed cache
+ * file in SubmitterUserResolver mode with STRESS submission policy
+ * by using the existing input data and HDFS distributed cache file.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixEmulationOfHDFSPublicDCFile()
+ throws Exception {
+ final String tracePath = getTraceFile("distcache_case9_trace");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ tracePath};
+ final String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.RUN_GRIDMIX);
+ }
+}
diff --git a/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java
new file mode 100644
index 0000000..55f8c39
--- /dev/null
+++ b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfHDFSDCFilesWithDifferentVisibilities.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the Gridmix emulation of HDFS distributed cache files of
+ * different visibilities.
+ */
+
+public class TestEmulationOfHDFSDCFilesWithDifferentVisibilities extends
+ GridmixSystemTestCase {
+ private static final Log LOG = LogFactory.
+ getLog("TestEmulationOfHDFSDCFilesWithDifferentVisibilities.class");
+
+ /**
+ * Generate input data and HDFS distributed cache files of different
+ * visibilities based on given input trace. Verify the Gridmix emulation
+ * of HDFS distributed cache files of different visibilities in
+ * RoundRobinUserResolver mode with SERIAL submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGenerateAndEmulateOfHDFSDCFilesWithDiffVisibilities()
+ throws Exception {
+ final long INPUT_SIZE = 1024 * 9;
+ final String tracePath = getTraceFile("distcache_case5_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ INPUT_SIZE+"m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+ final String [] otherArgs = {
+ "-D",
+ "mapreduce.job.complete.cancel.delegation.tokens=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX);
+ }
+
+ /**
+ * Disable the distributed cache emulation and verify the Gridmix jobs
+ * whether it emulates or not.
+ * @throws Exception
+ */
+ @Test
+ public void testHDFSDCFilesWithoutEnableDCEmulation()
+ throws Exception {
+ final String tracePath = getTraceFile("distcache_case6_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "REPLAY",
+ tracePath};
+ final String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=false"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.RUN_GRIDMIX);
+ }
+}
+
diff --git a/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java
new file mode 100644
index 0000000..6207a9e
--- /dev/null
+++ b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestEmulationOfLocalFSDCFiles.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the emulation of local FS distributed cache files.
+ *
+ */
+public class TestEmulationOfLocalFSDCFiles extends GridmixSystemTestCase {
+ private static final Log LOG = LogFactory.
+ getLog("TestEmulationOfLocalFSDCFiles.class");
+ private static final String MAPREDUCE_DELEGATION_TOKEN =
+ "mapreduce.job.complete.cancel.delegation.tokens";
+ /**
+ * Generate the input data and distributer cache files.Verify the
+ * gridmix emulation of local file system distributed cache files
+ * in RoundRobinUserResolver mode with REPLAY submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGenerateInputAndEmulateLocalFSDCFile()
+ throws Exception {
+ final long INPUT_SIZE = 1024 * 6;
+ final String tracePath = getTraceFile("distcache_case7_trace");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "REPLAY",
+ INPUT_SIZE+"m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+ final String [] otherArgs = {
+ "-D", MAPREDUCE_DELEGATION_TOKEN +
+ "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+ "=false"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX);
+ }
+
+ /**
+ * Use existing input and local distributed cache files and verify
+ * the gridmix emulation of local file system distributed cache
+ * files in SubmitterUserResolver mode with STRESS
+ * Submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testEmulationOfLocalFSDCFile()
+ throws Exception {
+ final String tracePath = getTraceFile("distcache_case7_trace");
+ Assert.assertNotNull("Trace file has not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ tracePath};
+ final String [] otherArgs = {
+ "-D", MAPREDUCE_DELEGATION_TOKEN +
+ "=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+ "=false"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.RUN_GRIDMIX);
+ }
+}
diff --git a/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java
new file mode 100644
index 0000000..936b18d
--- /dev/null
+++ b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPrivateDCFile.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the Gridmix emulation of HDFS private distributed cache file.
+ */
+public class TestGridmixEmulationOfHDFSPrivateDCFile
+ extends GridmixSystemTestCase {
+ private static final Log LOG = LogFactory.
+ getLog("TestGridmixEmulationOfHDFSPrivateDCFile.class");
+ /**
+ * Generate input data and single HDFS private distributed cache
+ * file based on given input trace.Verify the Gridmix emulation of
+ * single private HDFS distributed cache file in RoundRobinUserResolver
+ * mode with STRESS submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGenerateAndEmulateOfHDFSPrivateDCFile()
+ throws Exception {
+ final long INPUT_SIZE = 8192;
+ final String tracePath = getTraceFile("distcache_case3_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ INPUT_SIZE+"m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+ final String [] otherArgs = {
+ "-D",
+ "mapreduce.job.complete.cancel.delegation.tokens=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX);
+ }
+ /**
+ * Verify the Gridmix emulation of single HDFS private distributed
+ * cache file in SubmitterUserResolver mode with REPLAY submission
+ * policy by using the existing input data and HDFS private
+ * distributed cache file.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixEmulationOfHDFSPrivateDCFile()
+ throws Exception {
+ final String tracePath = getTraceFile("distcache_case3_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "REPLAY",
+ tracePath};
+ final String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.RUN_GRIDMIX);
+ }
+}
+
diff --git a/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java
new file mode 100644
index 0000000..bf3d6aa
--- /dev/null
+++ b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfHDFSPublicDCFile.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the Gridmix emulation of HDFS public distributed cache file.
+ */
+public class TestGridmixEmulationOfHDFSPublicDCFile extends
+ GridmixSystemTestCase {
+ private static final Log LOG = LogFactory.
+ getLog("TestGridmixEmulationOfHDFSPublicDCFile.class");
+
+ /**
+ * Generate the input data and HDFS distributed cache file based
+ * on given input trace. Verify the Gridmix emulation of single HDFS
+ * public distributed cache file in SubmitterUserResolver mode with
+ * STRESS submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGenerateAndEmulationOfSingleHDFSDCFile()
+ throws Exception {
+ final long INPUT_SIZE = 7168;
+ final String tracePath = getTraceFile("distcache_case1_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ INPUT_SIZE+"m",
+ tracePath};
+ final String [] otherArgs = {
+ "-D",
+ "mapreduce.job.complete.cancel.delegation.tokens=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX);
+ }
+
+ /**
+ * Verify the Gridmix emulation of Single HDFS public distributed cache
+ * file in RoundRobinUserResolver mode with REPLAY submission policy
+ * by using the existing input data and HDFS public distributed cache file.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixEmulationOfSingleHDFSPublicDCFile()
+ throws Exception {
+ final String tracePath = getTraceFile("distcache_case1_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "REPLAY",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+ final String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.RUN_GRIDMIX);
+ }
+}
+
diff --git a/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java
new file mode 100644
index 0000000..6602399
--- /dev/null
+++ b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Verify the Gridmix emulation of Multiple HDFS private distributed
+ * cache files.
+ */
+public class TestGridmixEmulationOfMultipleHDFSPrivateDCFiles
+ extends GridmixSystemTestCase {
+ private static final Log LOG = LogFactory.
+ getLog("TestGridmixEmulationOfMultipleHDFSPrivateDCFiles.class");
+
+ /**
+ * Generate input data and multiple HDFS private distributed cache
+ * files based on given input trace.Verify the Gridmix emulation of
+ * multiple private HDFS distributed cache files in RoundRobinUserResolver
+ * mode with SERIAL submission policy.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGenerateAndEmulationOfMultipleHDFSPrivateDCFiles()
+ throws Exception {
+ final long inputSize = 6144;
+ final String tracePath = getTraceFile("distcache_case4_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "SERIAL",
+ inputSize+"m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+ final String [] otherArgs = {
+ "-D",
+ "mapreduce.job.complete.cancel.delegation.tokens=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX);
+ }
+
+ /**
+ * Verify the Gridmix emulation of multiple HDFS private distributed
+ * cache files in SubmitterUserResolver mode with STRESS submission
+ * policy by using the existing input data and HDFS private
+ * distributed cache files.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixEmulationOfMultipleHDFSPrivateDCFiles()
+ throws Exception {
+ final String tracePath = getTraceFile("distcache_case4_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ tracePath};
+ final String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.RUN_GRIDMIX);
+ }
+}
+
diff --git a/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java
new file mode 100644
index 0000000..16e71c1
--- /dev/null
+++ b/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridmixEmulationOfMultipleHDFSPublicDCFiles.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.junit.Assert;
+import org.junit.Test;
+import java.io.IOException;
+
+public class TestGridmixEmulationOfMultipleHDFSPublicDCFiles extends
+ GridmixSystemTestCase {
+ private static final Log LOG = LogFactory.
+ getLog("TestGridmixEmulationOfMultipleHDFSPublicDCFiles.class");
+
+ /**
+ * Generate the compressed input data and dist cache files based
+ * on input trace. Verify the Gridmix emulation of
+ * multiple HDFS public distributed cache file.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGenerateAndEmulationOfMultipleHDFSDCFiles()
+ throws Exception {
+ final long inputSize = 7168;
+ final String tracePath = getTraceFile("distcache_case2_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ RoundRobinUserResolver.class.getName(),
+ "STRESS",
+ inputSize+"m",
+ "file://" + UtilsForGridmix.getProxyUsersFile(conf),
+ tracePath};
+ final String [] otherArgs = {
+ "-D",
+ "mapreduce.job.complete.cancel.delegation.tokens=false",
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath, GridMixRunMode.
+ DATA_GENERATION_AND_RUN_GRIDMIX);
+ }
+
+ /**
+ * Verify the Gridmix emulation of Single HDFS public distributed cache file
+ * by using an existing input compressed data and HDFS dist cache file.
+ * @throws Exception - if an error occurs.
+ */
+ @Test
+ public void testGridmixEmulationOfMulitpleHDFSPublicDCFile()
+ throws Exception {
+ final String tracePath = getTraceFile("distcache_case2_trace");
+ Assert.assertNotNull("Trace file was not found.", tracePath);
+ final String [] runtimeValues ={"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "SERIAL",
+ tracePath};
+ final String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=true"
+ };
+ runGridmixAndVerify(runtimeValues, otherArgs, tracePath,
+ GridMixRunMode.RUN_GRIDMIX);
+ }
+}
+
diff --git a/src/java/org/apache/hadoop/mapred/MROutputFiles.java b/src/java/org/apache/hadoop/mapred/MROutputFiles.java
new file mode 100644
index 0000000..3f64a10
--- /dev/null
+++ b/src/java/org/apache/hadoop/mapred/MROutputFiles.java
@@ -0,0 +1,226 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.fs.LocalDirAllocator;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.MRConfig;
+
+/**
+ * Manipulate the working area for the transient store for maps and reduces.
+ *
+ * This class is used by map and reduce tasks to identify the directories that
+ * they need to write to/read from for intermediate files. The callers of
+ * these methods are from the Child running the Task.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class MROutputFiles extends MapOutputFile {
+
+ private LocalDirAllocator lDirAlloc =
+ new LocalDirAllocator(MRConfig.LOCAL_DIR);
+
+ public MROutputFiles() {
+ }
+
+ /**
+ * Return the path to local map output file created earlier
+ *
+ * @return path
+ * @throws IOException
+ */
+ @Override
+ public Path getOutputFile()
+ throws IOException {
+ return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + Path.SEPARATOR
+ + MAP_OUTPUT_FILENAME_STRING, getConf());
+ }
+
+ /**
+ * Create a local map output file name.
+ *
+ * @param size the size of the file
+ * @return path
+ * @throws IOException
+ */
+ @Override
+ public Path getOutputFileForWrite(long size)
+ throws IOException {
+ return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + Path.SEPARATOR
+ + MAP_OUTPUT_FILENAME_STRING, size, getConf());
+ }
+
+ /**
+ * Create a local map output file name on the same volume.
+ */
+ @Override
+ public Path getOutputFileForWriteInVolume(Path existing) {
+ return new Path(existing.getParent(), MAP_OUTPUT_FILENAME_STRING);
+ }
+
+ /**
+ * Return the path to a local map output index file created earlier
+ *
+ * @return path
+ * @throws IOException
+ */
+ @Override
+ public Path getOutputIndexFile()
+ throws IOException {
+ return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + Path.SEPARATOR
+ + MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING,
+ getConf());
+ }
+
+ /**
+ * Create a local map output index file name.
+ *
+ * @param size the size of the file
+ * @return path
+ * @throws IOException
+ */
+ @Override
+ public Path getOutputIndexFileForWrite(long size)
+ throws IOException {
+ return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + Path.SEPARATOR
+ + MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING,
+ size, getConf());
+ }
+
+ /**
+ * Create a local map output index file name on the same volume.
+ */
+ @Override
+ public Path getOutputIndexFileForWriteInVolume(Path existing) {
+ return new Path(existing.getParent(),
+ MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING);
+ }
+
+ /**
+ * Return a local map spill file created earlier.
+ *
+ * @param spillNumber the number
+ * @return path
+ * @throws IOException
+ */
+ @Override
+ public Path getSpillFile(int spillNumber)
+ throws IOException {
+ return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill"
+ + spillNumber + ".out", getConf());
+ }
+
+ /**
+ * Create a local map spill file name.
+ *
+ * @param spillNumber the number
+ * @param size the size of the file
+ * @return path
+ * @throws IOException
+ */
+ @Override
+ public Path getSpillFileForWrite(int spillNumber, long size)
+ throws IOException {
+ return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + "/spill"
+ + spillNumber + ".out", size, getConf());
+ }
+
+ /**
+ * Return a local map spill index file created earlier
+ *
+ * @param spillNumber the number
+ * @return path
+ * @throws IOException
+ */
+ @Override
+ public Path getSpillIndexFile(int spillNumber)
+ throws IOException {
+ return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill"
+ + spillNumber + ".out.index", getConf());
+ }
+
+ /**
+ * Create a local map spill index file name.
+ *
+ * @param spillNumber the number
+ * @param size the size of the file
+ * @return path
+ * @throws IOException
+ */
+ @Override
+ public Path getSpillIndexFileForWrite(int spillNumber, long size)
+ throws IOException {
+ return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + "/spill"
+ + spillNumber + ".out.index", size, getConf());
+ }
+
+ /**
+ * Return a local reduce input file created earlier
+ *
+ * @param mapId a map task id
+ * @return path
+ * @throws IOException
+ */
+ @Override
+ public Path getInputFile(int mapId)
+ throws IOException {
+ return lDirAlloc.getLocalPathToRead(String.format(
+ REDUCE_INPUT_FILE_FORMAT_STRING, TaskTracker.OUTPUT, Integer
+ .valueOf(mapId)), getConf());
+ }
+
+ /**
+ * Create a local reduce input file name.
+ *
+ * @param mapId a map task id
+ * @param size the size of the file
+ * @return path
+ * @throws IOException
+ */
+ @Override
+ public Path getInputFileForWrite(org.apache.hadoop.mapreduce.TaskID mapId,
+ long size)
+ throws IOException {
+ return lDirAlloc.getLocalPathForWrite(String.format(
+ REDUCE_INPUT_FILE_FORMAT_STRING, TaskTracker.OUTPUT, mapId.getId()),
+ size, getConf());
+ }
+
+ /** Removes all of the files related to a task. */
+ @Override
+ public void removeAll()
+ throws IOException {
+ ((JobConf)getConf()).deleteLocalFiles(TaskTracker.OUTPUT);
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ if (!(conf instanceof JobConf)) {
+ conf = new JobConf(conf);
+ }
+ super.setConf(conf);
+ }
+
+}
