mapreduce/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.mapred;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
import org.apache.hadoop.security.UnixUserGroupInformation;
import static org.junit.Assert.*;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;

public class TestDebugScript {

  // base directory which is used by the debug script
  private static final String BASE_DIR = new File(System.getProperty(
      "test.build.data", "/tmp")).getAbsolutePath();

  // script directory which is passed as dummy input + where debugscript
  // is written.
  private static final String SCRIPT_DIR = new File(BASE_DIR, "debugscript")
      .getAbsolutePath();

  // script which is used as debug script.
  private static final String SCRIPT_FILE = new File(SCRIPT_DIR,
      "debugscript.sh").getAbsolutePath();

  // making an assumption we have bash in path. Same as TaskLog. 
  // The debug script just accesses the stderr file of the task
  // and does a 'cat' on it
  private static final String SCRIPT_CONTENT = "cat $2";

  @Before
  public void setup() throws Exception {
    setupDebugScriptDirs();
  }

  @After
  public void tearDown() throws Exception {
    cleanupDebugScriptDirs();
  }

  /**
   * Cleanup method which is used to delete the files folder which are generated
   * by the testcase.
   * 
   */
  static void cleanupDebugScriptDirs() {
    File scriptFile = new File(SCRIPT_FILE);
    scriptFile.delete();
    File scriptDir = new File(SCRIPT_DIR);
    scriptDir.delete();
  }

  /**
   * Setup method which is used to create necessary files and folder for the
   * testcase.
   * 
   * @throws Exception
   */
  static void setupDebugScriptDirs() throws Exception {
    File scriptDir = new File(SCRIPT_DIR);
    if (!scriptDir.exists()) {
      scriptDir.mkdirs();
    }
    scriptDir.setExecutable(true, false);
    scriptDir.setReadable(true, false);
    scriptDir.setWritable(true, false);
    File scriptFile = new File(SCRIPT_FILE);
    PrintWriter writer = new PrintWriter(scriptFile);
    writer.println(SCRIPT_CONTENT);
    writer.flush();
    writer.close();
    scriptFile.setExecutable(true, false);
    scriptFile.setReadable(true, false);
  }

  /**
   * Main test case which checks proper execution of the testcase.
   * 
   * @throws Exception
   */
  @Test
  public void testDebugScript() throws Exception {
    JobConf conf = new JobConf();
    conf.setLong(TTConfig.TT_SLEEP_TIME_BEFORE_SIG_KILL, 0L);
    MiniMRCluster mrCluster = new MiniMRCluster(1, "file:///", 1, null, null, conf);
    Path inputPath = new Path(SCRIPT_DIR);
    Path outputPath = new Path(SCRIPT_DIR, "task_output");
    
    // Run a failing mapper so debug script is launched.
    JobID jobId = runFailingMapJob(mrCluster.createJobConf(), inputPath,
        outputPath);
    // construct the task id of first map task of failmap
    TaskAttemptID taskId = new TaskAttemptID(
        new TaskID(jobId,TaskType.MAP, 0), 0);
    // verify if debug script was launched correctly and ran correctly.
    verifyDebugScriptOutput(taskId);
  }

  /**
   * Method which verifies if debug script ran and ran correctly.
   * 
   * @param taskId
   * @param expectedUser
   *          expected user id from debug script
   * @throws Exception
   */
  static void verifyDebugScriptOutput(TaskAttemptID taskId) throws Exception {
    verifyDebugScriptOutput(taskId, null, null);
  }
  /**
   * Method which verifies if debug script ran and ran correctly.
   * 
   * @param taskId
   * @param expectedUser
   *          expected user id from debug script
   * @param expectedPerms the expected permissions on the debugout file
   * @throws Exception
   */
  static void verifyDebugScriptOutput(TaskAttemptID taskId, String expectedUser, 
      String expectedPerms) throws Exception {
    File output = TaskLog.getRealTaskLogFileLocation(taskId, 
        TaskLog.LogName.DEBUGOUT);
    // Check the presence of the output file if the script is to be run.
    assertTrue("Output file does not exists. DebugScript has not been run",
          output.exists());
    // slurp the output from file, which is one line
    BufferedReader reader = new BufferedReader(new FileReader(output));
    String out = reader.readLine();
    // close the file.
    reader.close();
    // Check if there is any output
    assertNotNull("DebugScript didn't generate output.", out);
    assertTrue(out.contains("failing map"));
    if (expectedPerms != null && expectedUser != null) {
      //check whether the debugout file ownership/permissions are as expected
      String ttGroup = UnixUserGroupInformation.login().getGroupNames()[0];
      TestTaskTrackerLocalization.checkFilePermissions(output.getAbsolutePath(),
          expectedPerms, expectedUser, ttGroup);
    }
  }

  /**
   * Method to run a failing mapper on a given Cluster.
   * 
   * @param conf
   *          the JobConf for the job
   * @param inputPath
   *          input path for the job.
   * @param outputDir
   *          output directory for job.
   * @throws IOException
   */
  static JobID runFailingMapJob(JobConf conf, Path inputPath, Path outputDir)
      throws IOException {
    conf.setMapDebugScript(SCRIPT_FILE);
    conf.setMaxMapAttempts(0);
    conf.set("mapred.committer.job.setup.cleanup.needed", "false");
    RunningJob rJob = UtilsForTests.runJobFail(conf, inputPath, outputDir);
    return rJob.getID();
  }
}