| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.mapred; |
| |
| import java.io.BufferedReader; |
| import java.io.File; |
| import java.io.FileReader; |
| import java.io.IOException; |
| import java.io.PrintWriter; |
| |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.fs.Path; |
| import org.apache.hadoop.mapreduce.TaskType; |
| import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; |
| import org.apache.hadoop.security.Groups; |
| import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; |
| import org.apache.hadoop.security.UserGroupInformation; |
| import static org.junit.Assert.*; |
| |
| import org.junit.After; |
| import org.junit.Before; |
| import org.junit.Test; |
| |
| public class TestDebugScript { |
| |
| // base directory which is used by the debug script |
| private static final String BASE_DIR = new File(System.getProperty( |
| "test.build.data", "/tmp")).getAbsolutePath(); |
| |
| // script directory which is passed as dummy input + where debugscript |
| // is written. |
| private static final String SCRIPT_DIR = new File(BASE_DIR, "debugscript") |
| .getAbsolutePath(); |
| |
| // script which is used as debug script. |
| private static final String SCRIPT_FILE = new File(SCRIPT_DIR, |
| "debugscript.sh").getAbsolutePath(); |
| |
| // making an assumption we have bash in path. Same as TaskLog. |
| // The debug script just accesses the stderr file of the task |
| // and does a 'cat' on it |
| private static final String SCRIPT_CONTENT = "cat $2"; |
| |
| @Before |
| public void setup() throws Exception { |
| setupDebugScriptDirs(); |
| } |
| |
| @After |
| public void tearDown() throws Exception { |
| cleanupDebugScriptDirs(); |
| } |
| |
| /** |
| * Cleanup method which is used to delete the files folder which are generated |
| * by the testcase. |
| * |
| */ |
| static void cleanupDebugScriptDirs() { |
| File scriptFile = new File(SCRIPT_FILE); |
| scriptFile.delete(); |
| File scriptDir = new File(SCRIPT_DIR); |
| scriptDir.delete(); |
| } |
| |
| /** |
| * Setup method which is used to create necessary files and folder for the |
| * testcase. |
| * |
| * @throws Exception |
| */ |
| static void setupDebugScriptDirs() throws Exception { |
| File scriptDir = new File(SCRIPT_DIR); |
| if (!scriptDir.exists()) { |
| scriptDir.mkdirs(); |
| } |
| scriptDir.setExecutable(true, false); |
| scriptDir.setReadable(true, false); |
| scriptDir.setWritable(true, false); |
| File scriptFile = new File(SCRIPT_FILE); |
| PrintWriter writer = new PrintWriter(scriptFile); |
| writer.println(SCRIPT_CONTENT); |
| writer.flush(); |
| writer.close(); |
| scriptFile.setExecutable(true, false); |
| scriptFile.setReadable(true, false); |
| } |
| |
| /** |
| * Main test case which checks proper execution of the testcase. |
| * |
| * @throws Exception |
| */ |
| @Test |
| public void testDebugScript() throws Exception { |
| JobConf conf = new JobConf(); |
| conf.setLong(TTConfig.TT_SLEEP_TIME_BEFORE_SIG_KILL, 0L); |
| MiniMRCluster mrCluster = new MiniMRCluster(1, "file:///", 1, null, null, conf); |
| Path inputPath = new Path(SCRIPT_DIR); |
| Path outputPath = new Path(SCRIPT_DIR, "task_output"); |
| |
| // Run a failing mapper so debug script is launched. |
| JobID jobId = runFailingMapJob(mrCluster.createJobConf(), inputPath, |
| outputPath); |
| // construct the task id of first map task of failmap |
| TaskAttemptID taskId = new TaskAttemptID( |
| new TaskID(jobId,TaskType.MAP, 0), 0); |
| // verify if debug script was launched correctly and ran correctly. |
| verifyDebugScriptOutput(taskId); |
| } |
| |
| /** |
| * Method which verifies if debug script ran and ran correctly. |
| * |
| * @param taskId |
| * @param expectedUser |
| * expected user id from debug script |
| * @throws Exception |
| */ |
| static void verifyDebugScriptOutput(TaskAttemptID taskId) throws Exception { |
| verifyDebugScriptOutput(taskId, null, null, null); |
| } |
| /** |
| * Method which verifies if debug script ran and ran correctly. |
| * |
| * @param taskId |
| * @param expectedUser |
| * expected user id from debug script |
| * @param expectedPerms the expected permissions on the debugout file |
| * @throws Exception |
| */ |
| static void verifyDebugScriptOutput(TaskAttemptID taskId, String expectedUser, |
| String expectedGroup, String expectedPerms) throws Exception { |
| File output = TaskLog.getRealTaskLogFileLocation(taskId, false, |
| TaskLog.LogName.DEBUGOUT); |
| // Check the presence of the output file if the script is to be run. |
| assertTrue("Output file does not exists. DebugScript has not been run", |
| output.exists()); |
| // slurp the output from file, which is one line |
| BufferedReader reader = new BufferedReader(new FileReader(output)); |
| String out = reader.readLine(); |
| // close the file. |
| reader.close(); |
| // Check if there is any output |
| assertNotNull("DebugScript didn't generate output.", out); |
| assertTrue(out.contains("failing map")); |
| if (expectedPerms != null && expectedUser != null) { |
| //check whether the debugout file ownership/permissions are as expected |
| TestTaskTrackerLocalization.checkFilePermissions(output.getAbsolutePath(), |
| expectedPerms, expectedUser, expectedGroup); |
| } |
| } |
| |
| /** |
| * Method to run a failing mapper on a given Cluster. |
| * |
| * @param conf |
| * the JobConf for the job |
| * @param inputPath |
| * input path for the job. |
| * @param outputDir |
| * output directory for job. |
| * @throws IOException |
| */ |
| static JobID runFailingMapJob(JobConf conf, Path inputPath, Path outputDir) |
| throws IOException { |
| conf.setMapDebugScript(SCRIPT_FILE); |
| conf.setMaxMapAttempts(0); |
| conf.set("mapred.committer.job.setup.cleanup.needed", "false"); |
| RunningJob rJob = UtilsForTests.runJobFail(conf, inputPath, outputDir); |
| return rJob.getID(); |
| } |
| } |