src/mapred/org/apache/hadoop/mapred/JobHistory.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.mapred;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;

/**
 * Provides methods for writing to and reading from job history. 
 * Job History works in an append mode, JobHistory and its inner classes provide methods 
 * to log job events. 
 * 
 * JobHistory is split into multiple files, format of each file is plain text where each line 
 * is of the format [type (key=value)*], where type identifies the type of the record. 
 * Type maps to UID of one of the inner classes of this class. 
 * 
 * Job history is maintained in a master index which contains star/stop times of all jobs with
 * a few other job level properties. Apart from this each job's history is maintained in a seperate history 
 * file. name of job history files follows the format jobtrackerId_jobid
 *  
 * For parsing the job history it supports a listener based interface where each line is parsed
 * and passed to listener. The listener can create an object model of history or look for specific 
 * events and discard rest of the history.  
 * 
 * CHANGE LOG :
 * Version 0 : The history has the following format : 
 *             TAG KEY1="VALUE1" KEY2="VALUE2" and so on. 
               TAG can be Job, Task, MapAttempt or ReduceAttempt. 
               Note that a '"' is the line delimiter.
 * Version 1 : Changes the line delimiter to '.'
               Values are now escaped for unambiguous parsing. 
               Added the Meta tag to store version info.
 */
public class JobHistory {
  
  static final long VERSION = 1L;

  static final int DONE_DIRECTORY_FORMAT_VERSION = 1;

  static final String DONE_DIRECTORY_FORMAT_DIRNAME
    = "version-" + DONE_DIRECTORY_FORMAT_VERSION;

  static final String UNDERSCORE_ESCAPE = "%5F";

  public static final Log LOG = LogFactory.getLog(JobHistory.class);
  private static final char DELIMITER = ' ';
  static final char LINE_DELIMITER_CHAR = '.';
  static final char[] charsToEscape = new char[] {'"', '=', 
                                                LINE_DELIMITER_CHAR};
  static final String DIGITS = "[0-9]+";

  static final String KEY = "(\\w+)";
  // value is any character other than quote, but escaped quotes can be there
  static final String VALUE = "[^\"\\\\]*+(?:\\\\.[^\"\\\\]*+)*+";
  
  static final Pattern pattern = Pattern.compile(KEY + "=" + "\"" + VALUE + "\"");

  static final int MAXIMUM_DATESTRING_COUNT = 200000;
  
  public static final int JOB_NAME_TRIM_LENGTH = 50;
  private static String JOBTRACKER_UNIQUE_STRING = null;
  private static final String JOBHISTORY_DEBUG_MODE =
    "mapreduce.jobhistory.debug.mode";
  private static String LOG_DIR = null;
  private static final String SECONDARY_FILE_SUFFIX = ".recover";
  private static long jobHistoryBlockSize = 0;
  private static String jobtrackerHostname;
  private static JobHistoryFilesManager fileManager = null;
  final static FsPermission HISTORY_DIR_PERMISSION =
    FsPermission.createImmutable((short) 0755); // rwxr-xr-x
  final static FsPermission HISTORY_FILE_PERMISSION =
    FsPermission.createImmutable((short) 0744); // rwxr--r--
  private static FileSystem LOGDIR_FS; // log dir filesystem
  private static FileSystem DONEDIR_FS; // Done dir filesystem
  private static JobConf jtConf;
  private static Path DONE = null; // folder for completed jobs
  private static String DONE_BEFORE_SERIAL_TAIL = doneSubdirsBeforeSerialTail();
  private static String DONE_LEAF_FILES = DONE_BEFORE_SERIAL_TAIL + "/*";
  private static boolean aclsEnabled = false;

  static final String CONF_FILE_NAME_SUFFIX = "_conf.xml";

  // XXXXX debug mode -- set this to false for production
  private static boolean DEBUG_MODE;
  private static final int SERIAL_NUMBER_DIRECTORY_DIGITS = 6;
  private static int SERIAL_NUMBER_LOW_DIGITS;

  private static String SERIAL_NUMBER_FORMAT;

  private static final Set<Path> existingDoneSubdirs = new HashSet<Path>();

  private static final SortedMap<Integer, String> idToDateString
    = new TreeMap<Integer, String>();

  /**
   * A filter for conf files
   */  
  private static final PathFilter CONF_FILTER = new PathFilter() {
    public boolean accept(Path path) {
      return path.getName().endsWith(CONF_FILE_NAME_SUFFIX);
    }
  };

  private static final Map<JobID, MovedFileInfo> jobHistoryFileMap =
    Collections.<JobID,MovedFileInfo>synchronizedMap(
        new LinkedHashMap<JobID, MovedFileInfo>());

  private static final SortedMap<Long, String>jobToDirectoryMap
    = new TreeMap<Long, String>();

  // JobHistory filename regex
  public static final Pattern JOBHISTORY_FILENAME_REGEX = 
    Pattern.compile("(" + JobID.JOBID_REGEX + ")_.+");
  // JobHistory conf-filename regex
  public static final Pattern CONF_FILENAME_REGEX =
    Pattern.compile("(" + JobID.JOBID_REGEX + ")_conf.xml");
  
  private static class MovedFileInfo {
    private final String historyFile;
    private final long timestamp;
    public MovedFileInfo(String historyFile, long timestamp) {
      this.historyFile = historyFile;
      this.timestamp = timestamp;
    }
  }

  /**
   * Given the job id, return the history file path from the cache
   */
  public static String getHistoryFilePath(JobID jobId) {
    MovedFileInfo info = jobHistoryFileMap.get(jobId);
    if (info == null) {
      return null;
    }
    return info.historyFile;
  }

  /**
   * A class that manages all the files related to a job. For now 
   *   - writers : list of open files
   *   - job history filename
   *   - job conf filename
   */
  private static class JobHistoryFilesManager {
    // a private (virtual) folder for all the files related to a running job
    private static class FilesHolder {
      ArrayList<PrintWriter> writers = new ArrayList<PrintWriter>();
      Path historyFilename; // path of job history file
      Path confFilename; // path of job's conf
    }
   
    private ThreadPoolExecutor executor = null;
    private final Configuration conf;
    private final JobTracker jobTracker;

   // cache from job-key to files associated with it.
    private Map<JobID, FilesHolder> fileCache = 
      new ConcurrentHashMap<JobID, FilesHolder>();

    JobHistoryFilesManager(Configuration conf, JobTracker jobTracker)
        throws IOException {
      this.conf = conf;
      this.jobTracker = jobTracker;
    }


    void start() {
      executor = new ThreadPoolExecutor(1, 3, 1, 
          TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>());
    }

    private FilesHolder getFileHolder(JobID id) {
      FilesHolder holder = fileCache.get(id);
     if (holder == null) {
         holder = new FilesHolder();
         fileCache.put(id, holder);
      }
      return holder;
    }

    void addWriter(JobID id, PrintWriter writer) {
      FilesHolder holder = getFileHolder(id);
      holder.writers.add(writer);
    }

    void setHistoryFile(JobID id, Path file) {
      FilesHolder holder = getFileHolder(id);
      holder.historyFilename = file;
    }

    void setConfFile(JobID id, Path file) {
      FilesHolder holder = getFileHolder(id);
      holder.confFilename = file;
    }

    ArrayList<PrintWriter> getWriters(JobID id) {
      FilesHolder holder = fileCache.get(id);
      return holder == null ? null : holder.writers;
    }

    Path getHistoryFile(JobID id) {
      FilesHolder holder = fileCache.get(id);
      return holder == null ? null : holder.historyFilename;
    }

    Path getConfFileWriters(JobID id) {
      FilesHolder holder = fileCache.get(id);
      return holder == null ? null : holder.confFilename;
    }

    void purgeJob(JobID id) {
      fileCache.remove(id);
    }

    void moveToDone(final JobID id) {
      final List<Path> paths = new ArrayList<Path>();
      final Path historyFile = fileManager.getHistoryFile(id);
      if (historyFile == null) {
        LOG.info("No file for job-history with " + id + " found in cache!");
      } else {
        paths.add(historyFile);
      }

      final Path confPath = fileManager.getConfFileWriters(id);
      if (confPath == null) {
        LOG.info("No file for jobconf with " + id + " found in cache!");
      } else {
        paths.add(confPath);
      }

      executor.execute(new Runnable() {

        public void run() {
          long millisecondTime = System.currentTimeMillis();

          Path resultDir = canonicalHistoryLogPath(id, millisecondTime);

          //move the files to DONE canonical subfolder
          try {
            for (Path path : paths) {
              //check if path exists, in case of retries it may not exist
              if (LOGDIR_FS.exists(path)) {
                maybeMakeSubdirectory(id, millisecondTime);

                LOG.info("Moving " + path.toString() + " to " + 
                    resultDir.toString()); 
                DONEDIR_FS.moveFromLocalFile(path, resultDir);
                DONEDIR_FS.setPermission(new Path(resultDir, path.getName()), 
                    new FsPermission(HISTORY_FILE_PERMISSION));
              }
            }
          } catch (Throwable e) {
            LOG.error("Unable to move history file to DONE canonical subfolder.", e);
          }
          String historyFileDonePath = null;
          if (historyFile != null) {
            historyFileDonePath = new Path(resultDir, 
                historyFile.getName()).toString();
          }

          jobHistoryFileMap.put(id, new MovedFileInfo(historyFileDonePath,
                                                      millisecondTime));
          jobTracker.historyFileCopied(id, historyFileDonePath);
          
          //purge the job from the cache
          fileManager.purgeJob(id);
        }

      });
    }

    void removeWriter(JobID jobId, PrintWriter writer) {
      fileManager.getWriters(jobId).remove(writer);
    }
  }

  // several methods for manipulating the subdirectories of the DONE
  // directory 

  private static int jobSerialNumber(JobID id) {
    return id.getId();
  }

  private static String serialNumberDirectoryComponent(JobID id) {
    return String.format(SERIAL_NUMBER_FORMAT,
                         Integer.valueOf(jobSerialNumber(id)))
              .substring(0, SERIAL_NUMBER_DIRECTORY_DIGITS);
  }

  // directory components may contain internal slashes, but do NOT
  // contain slashes at either end.

  private static String timestampDirectoryComponent(JobID id, long millisecondTime) {
    int serialNumber = jobSerialNumber(id);
    Integer boxedSerialNumber = serialNumber;

    // don't want to do this inside the lock
    Calendar timestamp = Calendar.getInstance();
    timestamp.setTimeInMillis(millisecondTime);

    synchronized (idToDateString) {
      String dateString = idToDateString.get(boxedSerialNumber);

      if (dateString == null) {

        dateString = String.format
          ("%04d/%02d/%02d",
           timestamp.get(Calendar.YEAR),
           // months are 0-based in Calendar, but people will expect January
           // to be month #1.
           timestamp.get(DEBUG_MODE ? Calendar.HOUR : Calendar.MONTH) + 1,
           timestamp.get(DEBUG_MODE ? Calendar.MINUTE : Calendar.DAY_OF_MONTH));

        dateString = dateString.intern();

        idToDateString.put(boxedSerialNumber, dateString);

        if (idToDateString.size() > MAXIMUM_DATESTRING_COUNT) {
          idToDateString.remove(idToDateString.firstKey());
        }
      }

      return dateString;
    }
  }

  // returns false iff the directory already existed
  private static boolean maybeMakeSubdirectory(JobID id, long millisecondTime)
          throws IOException {
    Path dir = canonicalHistoryLogPath(id, millisecondTime);

    synchronized (existingDoneSubdirs) {
      if (existingDoneSubdirs.contains(dir)) {
        if (DEBUG_MODE && !DONEDIR_FS.exists(dir)) {
          System.err.println("JobHistory.maybeMakeSubdirectory -- We believed "
                             + dir + " already existed, but it didn't.");
        }
          
        return true;
      }

      if (!DONEDIR_FS.exists(dir)) {
        LOG.info("Creating DONE subfolder at "+ dir);

        if (!FileSystem.mkdirs(DONEDIR_FS, dir,
                               new FsPermission(HISTORY_DIR_PERMISSION))) {
          throw new IOException("Mkdirs failed to create " + dir.toString());
        }

        existingDoneSubdirs.add(dir);

        return false;
      } else {
        if (DEBUG_MODE) {
          System.err.println("JobHistory.maybeMakeSubdirectory -- We believed "
                             + dir + " didn't already exist, but it did.");
        }

        return false;
      }
    }
  }

  private static Path canonicalHistoryLogPath(JobID id, long millisecondTime) {
    return new Path(DONE, historyLogSubdirectory(id, millisecondTime));
  }

  private static String historyLogSubdirectory(JobID id, long millisecondTime) {
    String result
      = (DONE_DIRECTORY_FORMAT_DIRNAME
         + "/" + jobtrackerDirectoryComponent(id));

    String serialNumberDirectory = serialNumberDirectoryComponent(id);

    result = (result
              + "/" + timestampDirectoryComponent(id, millisecondTime)
              + "/" + serialNumberDirectory
              + "/");

    return result;
  }

  private static String jobtrackerDirectoryComponent(JobID id) {
    return JOBTRACKER_UNIQUE_STRING;
  }

  private static String doneSubdirsBeforeSerialTail() {
    // job tracker ID
    String result
      = ("/" + DONE_DIRECTORY_FORMAT_DIRNAME
         + "/*");   // job tracker instance ID

    // date
    result = result + "/*/*/*";  // YYYY/MM/DD ;

    return result;
  }

  /**
   * Record types are identifiers for each line of log in history files. 
   * A record type appears as the first token in a single line of log. 
   */
  public static enum RecordTypes {
    Jobtracker, Job, Task, MapAttempt, ReduceAttempt, Meta
  }

  /**
   * Job history files contain key="value" pairs, where keys belong to this enum. 
   * It acts as a global namespace for all keys. 
   */
  public static enum Keys { 
    JOBTRACKERID,
    START_TIME, FINISH_TIME, JOBID, JOBNAME, USER, JOBCONF, SUBMIT_TIME, 
    LAUNCH_TIME, TOTAL_MAPS, TOTAL_REDUCES, FAILED_MAPS, FAILED_REDUCES, 
    FINISHED_MAPS, FINISHED_REDUCES, JOB_STATUS, TASKID, HOSTNAME, TASK_TYPE, 
    ERROR, TASK_ATTEMPT_ID, TASK_STATUS, COPY_PHASE, SORT_PHASE, REDUCE_PHASE, 
    SHUFFLE_FINISHED, SORT_FINISHED, COUNTERS, SPLITS, JOB_PRIORITY, HTTP_PORT, 
    TRACKER_NAME, STATE_STRING, VERSION, MAP_COUNTERS, REDUCE_COUNTERS,
    VIEW_JOB, MODIFY_JOB, JOB_QUEUE, FAIL_REASON
  }

  /**
   * This enum contains some of the values commonly used by history log events. 
   * since values in history can only be strings - Values.name() is used in 
   * most places in history file. 
   */
  public static enum Values {
    SUCCESS, FAILED, KILLED, MAP, REDUCE, CLEANUP, RUNNING, PREP, SETUP
  }

  /**
   * Initialize JobHistory files. 
   * @param conf Jobconf of the job tracker.
   * @param hostname jobtracker's hostname
   * @param jobTrackerStartTime jobtracker's start time
   */
  public static void init(JobTracker jobTracker, JobConf conf,
             String hostname, long jobTrackerStartTime) throws IOException {
    initLogDir(conf);
    DEBUG_MODE = conf.getBoolean(JOBHISTORY_DEBUG_MODE, false);
    SERIAL_NUMBER_LOW_DIGITS = DEBUG_MODE ? 1 : 3;
    SERIAL_NUMBER_FORMAT = ("%0"
       + (SERIAL_NUMBER_DIRECTORY_DIGITS + SERIAL_NUMBER_LOW_DIGITS)
       + "d");
    JOBTRACKER_UNIQUE_STRING = hostname + "_" + 
                                  String.valueOf(jobTrackerStartTime) + "_";
    jobtrackerHostname = hostname;
    Path logDir = new Path(LOG_DIR);
    if (!LOGDIR_FS.exists(logDir)){
      if (!LOGDIR_FS.mkdirs(logDir, new FsPermission(HISTORY_DIR_PERMISSION))) {
        throw new IOException("Mkdirs failed to create " + logDir.toString());
      }
    } else { // directory exists
      checkDirectoryPermissions(LOGDIR_FS, logDir, "hadoop.job.history.location");
    }
    conf.set("hadoop.job.history.location", LOG_DIR);
    // set the job history block size (default is 3MB)
    jobHistoryBlockSize = 
      conf.getLong("mapred.jobtracker.job.history.block.size", 
                   3 * 1024 * 1024);
    jtConf = conf;

    // queue and job level security is enabled on the mapreduce cluster or not
    aclsEnabled = conf.getBoolean(JobConf.MR_ACLS_ENABLED, false);

    // initialize the file manager
    fileManager = new JobHistoryFilesManager(conf, jobTracker);
  }

  private static void initLogDir(JobConf conf) throws IOException {
    LOG_DIR = conf.get("hadoop.job.history.location" ,
      "file:///" + new File(
      System.getProperty("hadoop.log.dir")).getAbsolutePath()
      + File.separator + "history");
    Path logDir = new Path(LOG_DIR);
    LOGDIR_FS = logDir.getFileSystem(conf);
  }

  static void initDone(JobConf conf, FileSystem fs) throws IOException {
    initDone(conf, fs, true);
  }

  static void initDone(JobConf conf, FileSystem fs,
                                     boolean setup)
      throws IOException {
    //if completed job history location is set, use that
    String doneLocation = conf.
                     get("mapred.job.tracker.history.completed.location");
    if (doneLocation != null) {
      DONE = fs.makeQualified(new Path(doneLocation));
      DONEDIR_FS = fs;
    } else {
      if (!setup) {
        initLogDir(conf);
      }
      DONE = new Path(LOG_DIR, "done");
      DONEDIR_FS = LOGDIR_FS;
    }
    Path versionSubdir = new Path(DONE, DONE_DIRECTORY_FORMAT_DIRNAME);
    //If not already present create the done folder with appropriate 
    //permission
    if (!DONEDIR_FS.exists(DONE)) {
      LOG.info("Creating DONE folder at "+ DONE);
      if (!DONEDIR_FS.mkdirs(DONE, 
          new FsPermission(HISTORY_DIR_PERMISSION))) {
        throw new IOException("Mkdirs failed to create " + DONE.toString());
      }

      if (!DONEDIR_FS.exists(versionSubdir)) {
        if (!DONEDIR_FS.mkdirs(versionSubdir,
                               new FsPermission(HISTORY_DIR_PERMISSION))) {
          throw new IOException("Mkdirs failed to create " + versionSubdir);
        }
      }
    } else { // directory exists. Checks version subdirectory permissions as
      // well.
      checkDirectoryPermissions(DONEDIR_FS, DONE,
          "mapred.job.tracker.history.completed.location");
      if (DONEDIR_FS.exists(versionSubdir))
        checkDirectoryPermissions(DONEDIR_FS, versionSubdir,
            "mapred.job.tracker.history.completed.location-versionsubdir");
    }

    if (!setup) {
      return;
    }

    fileManager.start();
  }

  /**
   * @param FileSystem
   * @param Path
   * @param configKey 
   * @throws IOException
   * @throws DiskErrorException
   */
  static void checkDirectoryPermissions(FileSystem fs, Path path,
      String configKey) throws IOException, DiskErrorException {
    FileStatus stat = fs.getFileStatus(path);
    FsPermission actual = stat.getPermission();
    if (!stat.isDir())
      throw new DiskErrorException(configKey + " - not a directory: "
          + path.toString());
    FsAction user = actual.getUserAction();
    if (!user.implies(FsAction.READ))
      throw new DiskErrorException("bad " + configKey
          + "- directory is not readable: " + path.toString());
    if (!user.implies(FsAction.WRITE))
      throw new DiskErrorException("bad " + configKey
          + "- directory is not writable " + path.toString());
  }

  /**
   * Manages job-history's meta information such as version etc.
   * Helps in logging version information to the job-history and recover
   * version information from the history. 
   */
  static class MetaInfoManager implements Listener {
    private long version = 0L;
    private KeyValuePair pairs = new KeyValuePair();
    
    // Extract the version of the history that was used to write the history
    public MetaInfoManager(String line) throws IOException {
      if (null != line) {
        // Parse the line
        parseLine(line, this, false);
      }
    }
    
    // Get the line delimiter
    char getLineDelim() {
      if (version == 0) {
        return '"';
      } else {
        return LINE_DELIMITER_CHAR;
      }
    }
    
    // Checks if the values are escaped or not
    boolean isValueEscaped() {
      // Note that the values are not escaped in version 0
      return version != 0;
    }
    
    public void handle(RecordTypes recType, Map<Keys, String> values) 
    throws IOException {
      // Check if the record is of type META
      if (RecordTypes.Meta == recType) {
        pairs.handle(values);
        version = pairs.getLong(Keys.VERSION); // defaults to 0
      }
    }
    
    /**
     * Logs history meta-info to the history file. This needs to be called once
     * per history file. 
     * @param jobId job id, assigned by jobtracker. 
     */
    static void logMetaInfo(ArrayList<PrintWriter> writers){
      if (null != writers){
         JobHistory.log(writers, RecordTypes.Meta, 
             new Keys[] {Keys.VERSION},
             new String[] {String.valueOf(VERSION)}); 
      }
    }
  }
  
  /** Escapes the string especially for {@link JobHistory}
   */
  static String escapeString(String data) {
    return StringUtils.escapeString(data, StringUtils.ESCAPE_CHAR, 
                                    charsToEscape);
  }
  
  /**
   * Parses history file and invokes Listener.handle() for 
   * each line of history. It can be used for looking through history
   * files for specific items without having to keep whole history in memory. 
   * @param path path to history file
   * @param l Listener for history events 
   * @param fs FileSystem where history file is present
   * @throws IOException
   */
  public static void parseHistoryFromFS(String path, Listener l, FileSystem fs)
  throws IOException{
    FSDataInputStream in = fs.open(new Path(path));
    BufferedReader reader = new BufferedReader(new InputStreamReader (in));
    try {
      String line = null; 
      StringBuffer buf = new StringBuffer(); 
      
      // Read the meta-info line. Note that this might a jobinfo line for files
      // written with older format
      line = reader.readLine();
      
      // Check if the file is empty
      if (line == null) {
        return;
      }
      
      // Get the information required for further processing
      MetaInfoManager mgr = new MetaInfoManager(line);
      boolean isEscaped = mgr.isValueEscaped();
      String lineDelim = String.valueOf(mgr.getLineDelim());  
      String escapedLineDelim = 
        StringUtils.escapeString(lineDelim, StringUtils.ESCAPE_CHAR, 
                                 mgr.getLineDelim());
      
      do {
        buf.append(line); 
        if (!line.trim().endsWith(lineDelim) 
            || line.trim().endsWith(escapedLineDelim)) {
          buf.append("\n");
          continue; 
        }
        parseLine(buf.toString(), l, isEscaped);
        buf = new StringBuffer(); 
      } while ((line = reader.readLine())!= null);
    } finally {
      try { reader.close(); } catch (IOException ex) {}
    }
  }

  /**
   * Parse a single line of history. 
   * @param line
   * @param l
   * @throws IOException
   */
  private static void parseLine(String line, Listener l, boolean isEscaped) 
  throws IOException{
    // extract the record type 
    int idx = line.indexOf(' '); 
    String recType = line.substring(0, idx);
    String data = line.substring(idx+1, line.length());
    
    Matcher matcher = pattern.matcher(data); 
    Map<Keys,String> parseBuffer = new HashMap<Keys, String>();

    while(matcher.find()){
      String tuple = matcher.group(0);
      String []parts = StringUtils.split(tuple, StringUtils.ESCAPE_CHAR, '=');
      String value = parts[1].substring(1, parts[1].length() -1);
      if (isEscaped) {
        value = StringUtils.unEscapeString(value, StringUtils.ESCAPE_CHAR,
                                           charsToEscape);
      }
      parseBuffer.put(Keys.valueOf(parts[0]), value);
    }

    l.handle(RecordTypes.valueOf(recType), parseBuffer); 
    
    parseBuffer.clear(); 
  }
  
  
  /**
   * Log a raw record type with keys and values. This is method is generally not used directly. 
   * @param recordType type of log event
   * @param key key
   * @param value value
   */
  
  static void log(PrintWriter out, RecordTypes recordType, Keys key, 
                  String value){
    value = escapeString(value);
    out.println(recordType.name() + DELIMITER + key + "=\"" + value + "\""
                + DELIMITER + LINE_DELIMITER_CHAR); 
  }
  
  /**
   * Log a number of keys and values with record. the array length of keys and values
   * should be same. 
   * @param recordType type of log event
   * @param keys type of log event
   * @param values type of log event
   */

  /**
   * Log a number of keys and values with record. the array length of keys and values
   * should be same. 
   * @param recordType type of log event
   * @param keys type of log event
   * @param values type of log event
   */

  static void log(ArrayList<PrintWriter> writers, RecordTypes recordType, 
                  Keys[] keys, String[] values) {
    log(writers, recordType, keys, values, null);
  }
  
  /**
   * Log a number of keys and values with record. the array length of keys and values
   * should be same. 
   * @param recordType type of log event
   * @param keys type of log event
   * @param values type of log event
   * @param JobID jobid of the job  
   */

  static void log(ArrayList<PrintWriter> writers, RecordTypes recordType, 
                  Keys[] keys, String[] values, JobID id) {

    // First up calculate the length of buffer, so that we are performant
    // enough.
    int length = recordType.name().length() + keys.length * 4 + 2;
    for (int i = 0; i < keys.length; i++) { 
      values[i] = escapeString(values[i]);
      length += values[i].length() + keys[i].toString().length();
    }

    // We have the length of the buffer, now construct it.
    StringBuilder builder = new StringBuilder(length);
    builder.append(recordType.name());
    builder.append(DELIMITER); 
    for(int i =0; i< keys.length; i++){
      builder.append(keys[i]);
      builder.append("=\"");
      builder.append(values[i]);
      builder.append("\"");
      builder.append(DELIMITER); 
    }
    builder.append(LINE_DELIMITER_CHAR);
    
    for (Iterator<PrintWriter> iter = writers.iterator(); iter.hasNext();) {
      PrintWriter out = iter.next();
      out.println(builder.toString());
      if (out.checkError() && id != null) {
        LOG.info("Logging failed for job " + id + "removing PrintWriter from FileManager");
        iter.remove();
      }
    }
  }
  
  /**
   * Get the history location
   */
  static Path getJobHistoryLocation() {
    return new Path(LOG_DIR);
  } 
  
  /**
   * Get the history location for completed jobs
   */
  static Path getCompletedJobHistoryLocation() {
    return DONE;
  }

  static int serialNumberDirectoryDigits() {
    return SERIAL_NUMBER_DIRECTORY_DIGITS;
  }

  static int serialNumberTotalDigits() {
    return serialNumberDirectoryDigits() + SERIAL_NUMBER_LOW_DIGITS;
  }

  /**
   * Get the 
   */
  
  /**
   * Base class contais utility stuff to manage types key value pairs with enums. 
   */
  static class KeyValuePair{
    private Map<Keys, String> values = new HashMap<Keys, String>(); 

    /**
     * Get 'String' value for given key. Most of the places use Strings as 
     * values so the default get' method returns 'String'.  This method never returns 
     * null to ease on GUIs. if no value is found it returns empty string ""
     * @param k 
     * @return if null it returns empty string - "" 
     */
    public String get(Keys k){
      String s = values.get(k); 
      return s == null ? "" : s; 
    }
    /**
     * Convert value from history to int and return. 
     * if no value is found it returns 0.
     * @param k key 
     */
    public int getInt(Keys k){
      String s = values.get(k); 
      if (null != s){
        return Integer.parseInt(s);
      }
      return 0; 
    }
    /**
     * Convert value from history to int and return. 
     * if no value is found it returns 0.
     * @param k
     */
    public long getLong(Keys k){
      String s = values.get(k); 
      if (null != s){
        return Long.parseLong(s);
      }
      return 0; 
    }
    /**
     * Set value for the key. 
     * @param k
     * @param s
     */
    public void set(Keys k, String s){
      values.put(k, s); 
    }
    /**
     * Adds all values in the Map argument to its own values. 
     * @param m
     */
    public void set(Map<Keys, String> m){
      values.putAll(m);
    }
    /**
     * Reads values back from the history, input is same Map as passed to Listener by parseHistory().  
     * @param values
     */
    public synchronized void handle(Map<Keys, String> values){
      set(values); 
    }
    /**
     * Returns Map containing all key-values. 
     */
    public Map<Keys, String> getValues(){
      return values; 
    }
  }

  // hasMismatches is just used to return a second value if you want
  // one.  I would have used MutableBoxedBoolean if such had been provided.
  static Path[] filteredStat2Paths
          (FileStatus[] stats, boolean dirs, AtomicBoolean hasMismatches) {
    int resultCount = 0;

    if (hasMismatches == null) {
      hasMismatches = new AtomicBoolean(false);
    }

    for (int i = 0; i < stats.length; ++i) {
      if (stats[i].isDir() == dirs) {
        stats[resultCount++] = stats[i];
      } else {
        hasMismatches.set(true);
      }
    }

    Path[] paddedResult = FileUtil.stat2Paths(stats);

    Path[] result = new Path[resultCount];

    System.arraycopy(paddedResult, 0, result, 0, resultCount);

    return result;
  }

  static FileStatus[] localGlobber
        (FileSystem fs, Path root, String tail) 
      throws IOException {
    return localGlobber(fs, root, tail, null);
  }

  static FileStatus[] localGlobber
        (FileSystem fs, Path root, String tail, PathFilter filter) 
      throws IOException {
    return localGlobber(fs, root, tail, filter, null);
  }
  
  private static FileStatus[] nullToEmpty(FileStatus[] result) {
    return result == null ? new FileStatus[0] : result;
  }
      
  private static FileStatus[] listFilteredStatus
        (FileSystem fs, Path root, PathFilter filter)
     throws IOException {
    return filter == null ? fs.listStatus(root) : fs.listStatus(root, filter);
  }

  // hasMismatches is just used to return a second value if you want
  // one.  I would have used MutableBoxedBoolean if such had been provided.
  static FileStatus[] localGlobber
    (FileSystem fs, Path root, String tail, PathFilter filter,
     AtomicBoolean hasFlatFiles)
      throws IOException {
    if (tail.equals("")) {
      return nullToEmpty(listFilteredStatus(fs, root, filter));
    }

    if (tail.startsWith("/*")) {
      Path[] subdirs = filteredStat2Paths(nullToEmpty(fs.listStatus(root)),
                                          true, hasFlatFiles);

      FileStatus[][] subsubdirs = new FileStatus[subdirs.length][];

      int subsubdirCount = 0;

      if (subsubdirs.length == 0) {
        return new FileStatus[0];
      }

      String newTail = tail.substring(2);

      for (int i = 0; i < subdirs.length; ++i) {
        subsubdirs[i] = localGlobber(fs, subdirs[i], newTail, filter, null);
        subsubdirCount += subsubdirs[i].length;
      }

      FileStatus[] result = new FileStatus[subsubdirCount];

      int segmentStart = 0;

      for (int i = 0; i < subsubdirs.length; ++i) {
        System.arraycopy(subsubdirs[i], 0, result, segmentStart, subsubdirs[i].length);
        segmentStart += subsubdirs[i].length;
      }

      return result;
    }

    if (tail.startsWith("/")) {
      int split = tail.indexOf('/', 1);

      if (split < 0) {
        return nullToEmpty
          (listFilteredStatus(fs, new Path(root, tail.substring(1)), filter));
      } else {
        String thisSegment = tail.substring(1, split);
        String newTail = tail.substring(split);
        return localGlobber
          (fs, new Path(root, thisSegment), newTail, filter, hasFlatFiles);
      }
    }

    IOException e = new IOException("localGlobber: bad tail");

    throw e;
  }

  static Path confPathFromLogFilePath(Path logFile) {
    String jobId = jobIdNameFromLogFileName(logFile.getName());
      
    Path logDir = logFile.getParent();

    return new Path(logDir, jobId + CONF_FILE_NAME_SUFFIX);
  }

  static String jobIdNameFromLogFileName(String logFileName) {
    String[] jobDetails = logFileName.split("_");
    return jobDetails[0] + "_" + jobDetails[1] + "_" + jobDetails[2];
  }

  static String userNameFromLogFileName(String logFileName) {
    String[] jobDetails = logFileName.split("_");
    return jobDetails[3];
  }

  static String jobNameFromLogFileName(String logFileName) {
    String[] jobDetails = logFileName.split("_");
    return jobDetails[4];
  }


  // This code will be inefficient if the subject contains dozens of underscores
  static String escapeUnderscores(String escapee) {
    return replaceStringInstances(escapee, "_", UNDERSCORE_ESCAPE);
  }

  static String nonOccursString(String logFileName) {
    int adHocIndex = 0;

    String unfoundString = "q" + adHocIndex;

    while (logFileName.contains(unfoundString)) {
      unfoundString = "q" + ++adHocIndex;
    }

    return unfoundString + "q";
  }

  // I tolerate this code because I expect a low number of
  // occurrences in a relatively short string
  static String replaceStringInstances
      (String logFileName, String old, String replacement) {
    int index = logFileName.indexOf(old);

    while (index > 0) {
      logFileName = (logFileName.substring(0, index)
                     + replacement
                     + replaceStringInstances
                         (logFileName.substring(index + old.length()),
                          old, replacement));

      index = logFileName.indexOf(old);
    }

    return logFileName;
  }      

  
  /**
   * Helper class for logging or reading back events related to job start, finish or failure. 
   */
  public static class JobInfo extends KeyValuePair{
    
    private Map<String, Task> allTasks = new TreeMap<String, Task>();
    private Map<JobACL, AccessControlList> jobACLs =
        new HashMap<JobACL, AccessControlList>();
    private String queueName = null;// queue to which this job was submitted to
    
    /** Create new JobInfo */
    public JobInfo(String jobId){ 
      set(Keys.JOBID, jobId);  
    }

    /**
     * Returns all map and reduce tasks <taskid-Task>. 
     */
    public Map<String, Task> getAllTasks() { return allTasks; }

    /**
     * Get the job acls.
     * 
     * @return a {@link Map} from {@link JobACL} to {@link AccessControlList}
     */
    public Map<JobACL, AccessControlList> getJobACLs() {
      return jobACLs;
    }

    @Override
    public synchronized void handle(Map<Keys, String> values) {
      if (values.containsKey(Keys.SUBMIT_TIME)) {// job submission
        // construct the job ACLs
        String viewJobACL = values.get(Keys.VIEW_JOB);
        String modifyJobACL = values.get(Keys.MODIFY_JOB);
        if (viewJobACL != null) {
          jobACLs.put(JobACL.VIEW_JOB, new AccessControlList(viewJobACL));
        }
        if (modifyJobACL != null) {
          jobACLs.put(JobACL.MODIFY_JOB, new AccessControlList(modifyJobACL));
        }
        // get the job queue name
        queueName = values.get(Keys.JOB_QUEUE);
      }
      super.handle(values);
    }

    String getJobQueue() {
      return queueName;
    }

    /**
     * Get the path of the locally stored job file
     * @param jobId id of the job
     * @return the path of the job file on the local file system 
     */
    public static String getLocalJobFilePath(JobID jobId){
      return System.getProperty("hadoop.log.dir") + File.separator +
               jobId + CONF_FILE_NAME_SUFFIX;
    }
                      
    
    /**
     * Helper function to encode the URL of the path of the job-history
     * log file. 
     * 
     * @param logFile path of the job-history file
     * @return URL encoded path
     * @throws IOException
     */
    public static String encodeJobHistoryFilePath(String logFile)
    throws IOException {
      Path rawPath = new Path(logFile);
      String encodedFileName = null;
      try {
        encodedFileName = URLEncoder.encode(rawPath.getName(), "UTF-8");
      } catch (UnsupportedEncodingException uee) {
        IOException ioe = new IOException();
        ioe.initCause(uee);
        ioe.setStackTrace(uee.getStackTrace());
        throw ioe;
      }
      
      Path encodedPath = new Path(rawPath.getParent(), encodedFileName);
      return encodedPath.toString();
    }
    
    /**
     * Helper function to encode the URL of the filename of the job-history 
     * log file.
     * 
     * @param logFileName file name of the job-history file
     * @return URL encoded filename
     * @throws IOException
     */
    public static String encodeJobHistoryFileName(String logFileName)
    throws IOException {
      String replacementUnderscoreEscape = null;

      if (logFileName.contains(UNDERSCORE_ESCAPE)) {
        replacementUnderscoreEscape = nonOccursString(logFileName);

        logFileName = replaceStringInstances
          (logFileName, UNDERSCORE_ESCAPE, replacementUnderscoreEscape);
      }

      String encodedFileName = null;
      try {
        encodedFileName = URLEncoder.encode(logFileName, "UTF-8");
      } catch (UnsupportedEncodingException uee) {
        IOException ioe = new IOException();
        ioe.initCause(uee);
        ioe.setStackTrace(uee.getStackTrace());
        throw ioe;
      }
      
      if (replacementUnderscoreEscape != null) {
        encodedFileName = replaceStringInstances
          (encodedFileName, replacementUnderscoreEscape, UNDERSCORE_ESCAPE);
      }

      return encodedFileName;
    }
    
    /**
     * Helper function to decode the URL of the filename of the job-history 
     * log file.
     * 
     * @param logFileName file name of the job-history file
     * @return URL decoded filename
     * @throws IOException
     */
    public static String decodeJobHistoryFileName(String logFileName)
    throws IOException {
      String decodedFileName = null;
      try {
        decodedFileName = URLDecoder.decode(logFileName, "UTF-8");
      } catch (UnsupportedEncodingException uee) {
        IOException ioe = new IOException();
        ioe.initCause(uee);
        ioe.setStackTrace(uee.getStackTrace());
        throw ioe;
      }
      return decodedFileName;
    }
    
    /**
     * Get the job name from the job conf
     */
    static String getJobName(JobConf jobConf) {
      String jobName = jobConf.getJobName();
      if (jobName == null || jobName.length() == 0) {
        jobName = "NA";
      }
      return jobName;
    }
    
    /**
     * Get the user name from the job conf
     */
    public static String getUserName(JobConf jobConf) {
      String user = jobConf.getUser();
      if (user == null || user.length() == 0) {
        user = "NA";
      }
      return user;
    }
    
    /**
     * Get the job history file path given the history filename
     */
    public static Path getJobHistoryLogLocation(String logFileName)
    {
      return LOG_DIR == null ? null : new Path(LOG_DIR, logFileName);
    }

    /**
     * Get the user job history file path
     */
    public static Path getJobHistoryLogLocationForUser(String logFileName, 
                                                       JobConf jobConf) {
      // find user log directory 
      Path userLogFile = null;
      Path outputPath = FileOutputFormat.getOutputPath(jobConf);
      String userLogDir = jobConf.get("hadoop.job.history.user.location",
                                      outputPath == null 
                                      ? null 
                                      : outputPath.toString());
      if ("none".equals(userLogDir)) {
        userLogDir = null;
      }
      if (userLogDir != null) {
        userLogDir = userLogDir + Path.SEPARATOR + "_logs" + Path.SEPARATOR 
                     + "history";
        userLogFile = new Path(userLogDir, logFileName);
      }
      return userLogFile;
    }

    /**
     * Generates the job history filename for a new job
     */
    private static String getNewJobHistoryFileName(JobConf jobConf, JobID id, long submitTime) {
      return
        id.toString() + "_"
        + submitTime + "_"
        + escapeUnderscores(getUserName(jobConf)) + "_" 
        + escapeUnderscores(trimJobName(getJobName(jobConf)));
    }
    
    /**
     * Trims the job-name if required
     */
    private static String trimJobName(String jobName) {
      if (jobName.length() > JOB_NAME_TRIM_LENGTH) {
        jobName = jobName.substring(0, JOB_NAME_TRIM_LENGTH);
      }
      return jobName;
    }
    
    private static String escapeRegexChars( String string ) {
      return "\\Q"+string.replaceAll("\\\\E", "\\\\E\\\\\\\\E\\\\Q")+"\\E";
    }

    /**
     * Recover the job history filename from the history folder. 
     * Uses the following pattern
     *    $jt-hostname_[0-9]*_$job-id_$user_$job-name*
     * @param jobConf the job conf
     * @param id job id
     */
    public static synchronized String getJobHistoryFileName(JobConf jobConf, 
                                                            JobID id) 
    throws IOException {                    
      return getJobHistoryFileName(jobConf, id, new Path(LOG_DIR), LOGDIR_FS);
    }

    // Returns that portion of the pathname that sits under the DONE directory
    static synchronized String getDoneJobHistoryFileName(JobConf jobConf, 
        JobID id) throws IOException {
      if (DONE == null) {
        return null;
      }
      return getJobHistoryFileName(jobConf, id, DONE, DONEDIR_FS);
    }
    
    /**
     * @param dir The directory where to search.
     */
    private static synchronized String getJobHistoryFileName(JobConf jobConf, 
                                          JobID id, Path dir, FileSystem fs) 
    throws IOException {
      String user =  getUserName(jobConf);
      String jobName = trimJobName(getJobName(jobConf));
      if (LOG_DIR == null) {
        return null;
      }

      // Make the pattern matching the job's history file

      final String regexp
        = id.toString() + "_" + DIGITS + "_" + user + "_"
             + escapeRegexChars(jobName) + "+";
      
      final Pattern historyFilePattern = Pattern.compile(regexp);

      // a path filter that matches 4 parts of the filenames namely
      //  - jt-hostname
      //  - job-id
      //  - username
      //  - jobname
      PathFilter filter = new PathFilter() {
        public boolean accept(Path path) {
          String unescapedFileName = path.getName();
          String fileName = null;
          try {
            fileName = decodeJobHistoryFileName(unescapedFileName);
          } catch (IOException ioe) {
            LOG.info("Error while decoding history file " + fileName + "."
                     + " Ignoring file.", ioe);
            return false;
          }

          return historyFilePattern.matcher(fileName).find();
        }
      };

      FileStatus[] statuses = null;

      if (dir == DONE) {
        final String snDirectoryComponent
          = serialNumberDirectoryComponent(id);

        final String scanTail
          = (DONE_BEFORE_SERIAL_TAIL
             + "/" + serialNumberDirectoryComponent(id));

        if (DEBUG_MODE) {
          System.err.println("JobHistory.getJobHistoryFileName DONE dir: scanning " + scanTail);

          (new IOException("debug exception")).printStackTrace(System.err);
        }

        statuses = localGlobber(fs, DONE, scanTail, filter);
      } else {
        statuses = fs.listStatus(dir, filter);
      }
      
      String filename = null;
      if (statuses == null || statuses.length == 0) {
        if (DEBUG_MODE) {
          System.err.println("Nothing to recover for job " + id);
        }
        LOG.info("Nothing to recover for job " + id);
      } else {
        // return filename considering that fact the name can be a 
        // secondary filename like filename.recover
        filename = getPrimaryFilename(statuses[0].getPath().getName(), jobName);
        if (dir == DONE) {
          Path parent = statuses[0].getPath().getParent();
          String parentPathName = parent.toString();
          String donePathName = DONE.toString();
          filename = (parentPathName.substring(donePathName.length() + Path.SEPARATOR.length())
                      + Path.SEPARATOR + filename);
        }
        
        LOG.info("Recovered job history filename for job " + id + " is " 
                 + filename);
      }
      return filename;
    }
    
    // removes all extra extensions from a filename and returns the core/primary
    // filename
    private static String getPrimaryFilename(String filename, String jobName) 
    throws IOException{
      filename = decodeJobHistoryFileName(filename);
      // Remove the '.recover' suffix if it exists
      if (filename.endsWith(jobName + SECONDARY_FILE_SUFFIX)) {
        int newLength = filename.length() - SECONDARY_FILE_SUFFIX.length();
        filename = filename.substring(0, newLength);
      }
      return encodeJobHistoryFileName(filename);
    }    

    /** Since there was a restart, there should be a master file and 
     * a recovery file. Once the recovery is complete, the master should be 
     * deleted as an indication that the recovery file should be treated as the 
     * master upon completion or next restart.
     * @param fileName the history filename that needs checkpointing
     * @param conf Job conf
     * @throws IOException
     */
    static synchronized void checkpointRecovery(String fileName, JobConf conf) 
    throws IOException {
      Path logPath = JobHistory.JobInfo.getJobHistoryLogLocation(fileName);
      if (logPath != null) {
        LOG.info("Deleting job history file " + logPath.getName());
        LOGDIR_FS.delete(logPath, false);
      }
      // do the same for the user file too
      logPath = JobHistory.JobInfo.getJobHistoryLogLocationForUser(fileName, 
                                                                   conf);
      if (logPath != null) {
        FileSystem fs = logPath.getFileSystem(conf);
        fs.delete(logPath, false);
      }
    }
    
    static String getSecondaryJobHistoryFile(String filename) 
    throws IOException {
      return encodeJobHistoryFileName(
          decodeJobHistoryFileName(filename) + SECONDARY_FILE_SUFFIX);
    }
    
    /** Selects one of the two files generated as a part of recovery. 
     * The thumb rule is that always select the oldest file. 
     * This call makes sure that only one file is left in the end. 
     * @param conf job conf
     * @param logFilePath Path of the log file
     * @throws IOException 
     */
    public synchronized static Path recoverJobHistoryFile(JobConf conf, 
                                                          Path logFilePath) 
    throws IOException {
      Path ret;
      String logFileName = logFilePath.getName();
      String tmpFilename = getSecondaryJobHistoryFile(logFileName);
      Path logDir = logFilePath.getParent();
      Path tmpFilePath = new Path(logDir, tmpFilename);
      if (LOGDIR_FS.exists(logFilePath)) {
        LOG.info(logFileName + " exists!");
        if (LOGDIR_FS.exists(tmpFilePath)) {
          LOG.info("Deleting " + tmpFilename 
                   + "  and using " + logFileName + " for recovery.");
          LOGDIR_FS.delete(tmpFilePath, false);
        }
        ret = tmpFilePath;
      } else {
        LOG.info(logFileName + " doesnt exist! Using " 
                 + tmpFilename + " for recovery.");
        if (LOGDIR_FS.exists(tmpFilePath)) {
          LOG.info("Renaming " + tmpFilename + " to " + logFileName);
          LOGDIR_FS.rename(tmpFilePath, logFilePath);
          ret = tmpFilePath;
        } else {
          ret = logFilePath;
        }
      }

      // do the same for the user files too
      logFilePath = getJobHistoryLogLocationForUser(logFileName, conf);
      if (logFilePath != null) {
        FileSystem fs = logFilePath.getFileSystem(conf);
        logDir = logFilePath.getParent();
        tmpFilePath = new Path(logDir, tmpFilename);
        if (fs.exists(logFilePath)) {
          LOG.info(logFileName + " exists!");
          if (fs.exists(tmpFilePath)) {
            LOG.info("Deleting " + tmpFilename + "  and making " + logFileName 
                     + " as the master history file for user.");
            fs.delete(tmpFilePath, false);
          }
        } else {
          LOG.info(logFileName + " doesnt exist! Using " 
                   + tmpFilename + " as the master history file for user.");
          if (fs.exists(tmpFilePath)) {
            LOG.info("Renaming " + tmpFilename + " to " + logFileName 
                     + " in user directory");
            fs.rename(tmpFilePath, logFilePath);
          }
        }
      }
      
      return ret;
    }

    /** Finalize the recovery and make one file in the end. 
     * This invloves renaming the recover file to the master file.
     * Note that this api should be invoked only if recovery is involved.
     * @param id Job id  
     * @param conf the job conf
     * @throws IOException
     */
    static synchronized void finalizeRecovery(JobID id, JobConf conf)
    throws IOException {
       Path tmpLogPath = fileManager.getHistoryFile(id);
       if (tmpLogPath == null) {
         if (LOG.isDebugEnabled()) {
           LOG.debug("No file for job with " + id + " found in cache!");
         }
         return;
       }
       String tmpLogFileName = tmpLogPath.getName();
       
       // get the primary filename from the cached filename
       String masterLogFileName = 
         getPrimaryFilename(tmpLogFileName, getJobName(conf));
       Path masterLogPath = new Path(tmpLogPath.getParent(), masterLogFileName);
       
       // rename the tmp file to the master file. Note that this should be 
       // done only when the file is closed and handles are released.
       LOG.info("Renaming " + tmpLogFileName + " to " + masterLogFileName);
       LOGDIR_FS.rename(tmpLogPath, masterLogPath);
       // update the cache
       fileManager.setHistoryFile(id, masterLogPath);

      // do the same for the user file too
      masterLogPath = 
        JobHistory.JobInfo.getJobHistoryLogLocationForUser(masterLogFileName,
                                                           conf);
      tmpLogPath = 
        JobHistory.JobInfo.getJobHistoryLogLocationForUser(tmpLogFileName, 
                                                           conf);
      if (masterLogPath != null) {
        FileSystem fs = masterLogPath.getFileSystem(conf);
        if (fs.exists(tmpLogPath)) {
          LOG.info("Renaming " + tmpLogFileName + " to " + masterLogFileName
                   + " in user directory");
          fs.rename(tmpLogPath, masterLogPath);
        }
      }
    }

    /**
     * Deletes job data from the local disk.
     * For now just deletes the localized copy of job conf
     */
    static void cleanupJob(JobID id) {
      String localJobFilePath =  JobInfo.getLocalJobFilePath(id);
      File f = new File (localJobFilePath);
      LOG.info("Deleting localized job conf at " + f);
      if (!f.delete()) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Failed to delete file " + f);
        }
      }
    }

    /**
     * Delete job conf from the history folder.
     */
    static void deleteConfFiles() throws IOException {
      LOG.info("Cleaning up config files from the job history folder");
      FileSystem fs = new Path(LOG_DIR).getFileSystem(jtConf);
      FileStatus[] status = fs.listStatus(new Path(LOG_DIR), CONF_FILTER);
      for (FileStatus s : status) {
        LOG.info("Deleting conf file " + s.getPath());
        fs.delete(s.getPath(), false);
      }
    }

    /**
     * Move the completed job into the completed folder.
     * This assumes that the jobhistory file is closed and all operations on the
     * jobhistory file is complete.
     * This *should* be the last call to jobhistory for a given job.
     */
    static void markCompleted(JobID id) throws IOException {
      fileManager.moveToDone(id);
    }

     /**
     * Log job submitted event to history. Creates a new file in history 
     * for the job. if history file creation fails, it disables history 
     * for all other events. 
     * @param jobId job id assigned by job tracker.
     * @param jobConf job conf of the job
     * @param jobConfPath path to job conf xml file in HDFS.
     * @param submitTime time when job tracker received the job
     * @throws IOException
     * @deprecated Use 
     *     {@link #logSubmitted(JobID, JobConf, String, long, boolean)} instead.
     */
    @Deprecated
    public static void logSubmitted(JobID jobId, JobConf jobConf, 
                                    String jobConfPath, long submitTime) 
    throws IOException {
      logSubmitted(jobId, jobConf, jobConfPath, submitTime, true);
    }
    
    public static void logSubmitted(JobID jobId, JobConf jobConf, 
                                    String jobConfPath, long submitTime, 
                                    boolean restarted) 
    throws IOException {
      FileSystem fs = null;
      String userLogDir = null;
      String jobUniqueString = jobId.toString();

      // Get the username and job name to be used in the actual log filename;
      // sanity check them too        
      String jobName = getJobName(jobConf);
      String user = getUserName(jobConf);
      
      // get the history filename
      String logFileName = null;
      if (restarted) {
        logFileName = getJobHistoryFileName(jobConf, jobId);
        if (logFileName == null) {
          logFileName =
            encodeJobHistoryFileName(getNewJobHistoryFileName
                                     (jobConf, jobId, submitTime));
        } else {
          String parts[] = logFileName.split("_");
          //TODO this is a hack :(
          // jobtracker-hostname_jobtracker-identifier_
          String jtUniqueString = parts[0] + "_" + parts[1] + "_";
          jobUniqueString = jobId.toString();
        }
      } else {
        logFileName = 
          encodeJobHistoryFileName(getNewJobHistoryFileName
                                   (jobConf, jobId, submitTime));
      }

      // setup the history log file for this job
      Path logFile = getJobHistoryLogLocation(logFileName);
      
      // find user log directory
      Path userLogFile = 
        getJobHistoryLogLocationForUser(logFileName, jobConf);
      PrintWriter writer = null;
      try{
        FSDataOutputStream out = null;
        if (LOG_DIR != null) {
          // create output stream for logging in hadoop.job.history.location
          if (restarted) {
            logFile = recoverJobHistoryFile(jobConf, logFile);
            logFileName = logFile.getName();
          }
          
          int defaultBufferSize = 
            LOGDIR_FS.getConf().getInt("io.file.buffer.size", 4096);
          out = LOGDIR_FS.create(logFile, 
                          new FsPermission(HISTORY_FILE_PERMISSION),
                          true, 
                          defaultBufferSize, 
                          LOGDIR_FS.getDefaultReplication(), 
                          jobHistoryBlockSize, null);
          writer = new PrintWriter(out);
          fileManager.addWriter(jobId, writer);

          // cache it ...
          fileManager.setHistoryFile(jobId, logFile);
        }
        if (userLogFile != null) {
          // Get the actual filename as recoverJobHistoryFile() might return
          // a different filename
          userLogDir = userLogFile.getParent().toString();
          userLogFile = new Path(userLogDir, logFileName);
          
          // create output stream for logging 
          // in hadoop.job.history.user.location
          fs = userLogFile.getFileSystem(jobConf);
 
          out = fs.create(userLogFile, true, 4096);
          writer = new PrintWriter(out);
          fileManager.addWriter(jobId, writer);
        }
        
        ArrayList<PrintWriter> writers = fileManager.getWriters(jobId);
        // Log the history meta info
        JobHistory.MetaInfoManager.logMetaInfo(writers);

        String viewJobACL = "*";
        String modifyJobACL = "*";
        if (aclsEnabled) {
          viewJobACL = jobConf.get(JobACL.VIEW_JOB.getAclName(), " ");
          modifyJobACL = jobConf.get(JobACL.MODIFY_JOB.getAclName(), " ");
        }
        //add to writer as well 
        JobHistory.log(writers, RecordTypes.Job,
                       new Keys[]{Keys.JOBID, Keys.JOBNAME, Keys.USER,
                                  Keys.SUBMIT_TIME, Keys.JOBCONF,
                                  Keys.VIEW_JOB, Keys.MODIFY_JOB,
                                  Keys.JOB_QUEUE}, 
                       new String[]{jobId.toString(), jobName, user, 
                                    String.valueOf(submitTime) , jobConfPath,
                                    viewJobACL, modifyJobACL,
                                    jobConf.getQueueName()}, jobId
                      ); 
           
      }catch(IOException e){
        LOG.error("Failed creating job history log file for job " + jobId, e);
        if (writer != null) {
          fileManager.removeWriter(jobId, writer);
        }
      }
      // Always store job conf on local file system 
      String localJobFilePath =  JobInfo.getLocalJobFilePath(jobId); 
      File localJobFile = new File(localJobFilePath);
      FileOutputStream jobOut = null;
      try {
        jobOut = new FileOutputStream(localJobFile);
        jobConf.writeXml(jobOut);
        if (LOG.isDebugEnabled()) {
          LOG.debug("Job conf for " + jobId + " stored at " 
                    + localJobFile.getAbsolutePath());
        }
      } catch (IOException ioe) {
        LOG.error("Failed to store job conf on the local filesystem ", ioe);
      } finally {
        if (jobOut != null) {
          try {
            jobOut.close();
          } catch (IOException ie) {
            LOG.info("Failed to close the job configuration file " 
                       + StringUtils.stringifyException(ie));
          }
        }
      }

      /* Storing the job conf on the log dir */
      Path jobFilePath = null;
      if (LOG_DIR != null) {
        jobFilePath = new Path(LOG_DIR + File.separator + 
                               jobUniqueString + CONF_FILE_NAME_SUFFIX);
        fileManager.setConfFile(jobId, jobFilePath);
      }
      Path userJobFilePath = null;
      if (userLogDir != null) {
        userJobFilePath = new Path(userLogDir + File.separator +
                                   jobUniqueString + CONF_FILE_NAME_SUFFIX);
      }
      FSDataOutputStream jobFileOut = null;
      try {
        if (LOG_DIR != null) {
          int defaultBufferSize = 
              LOGDIR_FS.getConf().getInt("io.file.buffer.size", 4096);
          if (!LOGDIR_FS.exists(jobFilePath)) {
            jobFileOut = LOGDIR_FS.create(jobFilePath, 
                                   new FsPermission(HISTORY_FILE_PERMISSION),
                                   true, 
                                   defaultBufferSize, 
                                   LOGDIR_FS.getDefaultReplication(), 
                                   LOGDIR_FS.getDefaultBlockSize(), null);
            jobConf.writeXml(jobFileOut);
            jobFileOut.close();
          }
        } 
        if (userLogDir != null) {
          fs = new Path(userLogDir).getFileSystem(jobConf);
          jobFileOut = fs.create(userJobFilePath);
          jobConf.writeXml(jobFileOut);
        }
        if (LOG.isDebugEnabled()) {
          LOG.debug("Job conf for " + jobId + " stored at " 
                    + jobFilePath + "and" + userJobFilePath );
        }
      } catch (IOException ioe) {
        LOG.error("Failed to store job conf in the log dir", ioe);
      } finally {
        if (jobFileOut != null) {
          try {
            jobFileOut.close();
          } catch (IOException ie) {
            LOG.info("Failed to close the job configuration file " 
                     + StringUtils.stringifyException(ie));
          }
        }
      } 
    }
    /**
     * Logs launch time of job. 
     * 
     * @param jobId job id, assigned by jobtracker. 
     * @param startTime start time of job. 
     * @param totalMaps total maps assigned by jobtracker. 
     * @param totalReduces total reduces. 
     */
    public static void logInited(JobID jobId, long startTime, 
                                 int totalMaps, int totalReduces) {
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobId); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Job, 
            new Keys[] {Keys.JOBID, Keys.LAUNCH_TIME, Keys.TOTAL_MAPS, 
                        Keys.TOTAL_REDUCES, Keys.JOB_STATUS},
            new String[] {jobId.toString(), String.valueOf(startTime), 
                          String.valueOf(totalMaps), 
                          String.valueOf(totalReduces), 
                          Values.PREP.name()}, jobId); 
      }
    }
    
   /**
     * Logs the job as RUNNING. 
     *
     * @param jobId job id, assigned by jobtracker. 
     * @param startTime start time of job. 
     * @param totalMaps total maps assigned by jobtracker. 
     * @param totalReduces total reduces. 
     * @deprecated Use {@link #logInited(JobID, long, int, int)} and 
     * {@link #logStarted(JobID)}
     */
    @Deprecated
    public static void logStarted(JobID jobId, long startTime, 
                                  int totalMaps, int totalReduces) {
      logStarted(jobId);
    }
    
    /**
     * Logs job as running 
     * @param jobId job id, assigned by jobtracker. 
     */
    public static void logStarted(JobID jobId){
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobId); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Job, 
            new Keys[] {Keys.JOBID, Keys.JOB_STATUS},
            new String[] {jobId.toString(),  
                          Values.RUNNING.name()}, jobId); 
      }
    }
    
    /**
     * Log job finished. closes the job file in history. 
     * @param jobId job id, assigned by jobtracker. 
     * @param finishTime finish time of job in ms. 
     * @param finishedMaps no of maps successfully finished. 
     * @param finishedReduces no of reduces finished sucessfully. 
     * @param failedMaps no of failed map tasks. 
     * @param failedReduces no of failed reduce tasks. 
     * @param counters the counters from the job
     */ 
    public static void logFinished(JobID jobId, long finishTime, 
                                   int finishedMaps, int finishedReduces,
                                   int failedMaps, int failedReduces,
                                   Counters mapCounters,
                                   Counters reduceCounters,
                                   Counters counters) {
        // close job file for this job
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobId); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Job,          
                       new Keys[] {Keys.JOBID, Keys.FINISH_TIME, 
                                   Keys.JOB_STATUS, Keys.FINISHED_MAPS, 
                                   Keys.FINISHED_REDUCES,
                                   Keys.FAILED_MAPS, Keys.FAILED_REDUCES,
                                   Keys.MAP_COUNTERS, Keys.REDUCE_COUNTERS,
                                   Keys.COUNTERS},
                       new String[] {jobId.toString(),  Long.toString(finishTime), 
                                     Values.SUCCESS.name(), 
                                     String.valueOf(finishedMaps), 
                                     String.valueOf(finishedReduces),
                                     String.valueOf(failedMaps), 
                                     String.valueOf(failedReduces),
                                     mapCounters.makeEscapedCompactString(),
                                     reduceCounters.makeEscapedCompactString(),
                                     counters.makeEscapedCompactString()}, jobId);
        for (PrintWriter out : writer) {
          out.close();
        }
      }
      Thread historyCleaner  = new Thread(new HistoryCleaner());
      historyCleaner.start(); 
    }
    /**
     * Logs job failed event. Closes the job history log file. 
     * @param jobid job id
     * @param timestamp time when job failure was detected in ms.  
     * @param finishedMaps no finished map tasks. 
     * @param finishedReduces no of finished reduce tasks. 
     */
    public static void logFailed(JobID jobid, long timestamp, int finishedMaps, int finishedReduces, String failReason){
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobid); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Job,
                       new Keys[] {Keys.JOBID, Keys.FINISH_TIME, Keys.JOB_STATUS, Keys.FINISHED_MAPS, Keys.FINISHED_REDUCES, Keys.FAIL_REASON },
                       new String[] {jobid.toString(),  String.valueOf(timestamp), Values.FAILED.name(), String.valueOf(finishedMaps), 
                                     String.valueOf(finishedReduces), failReason}, jobid); 
        for (PrintWriter out : writer) {
          out.close();
        }
      }
    }
    /**
     * Logs job killed event. Closes the job history log file.
     * 
     * @param jobid
     *          job id
     * @param timestamp
     *          time when job killed was issued in ms.
     * @param finishedMaps
     *          no finished map tasks.
     * @param finishedReduces
     *          no of finished reduce tasks.
     */
    public static void logKilled(JobID jobid, long timestamp, int finishedMaps,
        int finishedReduces) {
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobid);

      if (null != writer) {
        JobHistory.log(writer, RecordTypes.Job, new Keys[] { Keys.JOBID,
            Keys.FINISH_TIME, Keys.JOB_STATUS, Keys.FINISHED_MAPS,
            Keys.FINISHED_REDUCES }, new String[] { jobid.toString(),
            String.valueOf(timestamp), Values.KILLED.name(),
            String.valueOf(finishedMaps), String.valueOf(finishedReduces) }, jobid);
        for (PrintWriter out : writer) {
          out.close();
        }
      }
    }
    /**
     * Log job's priority. 
     * @param jobid job id
     * @param priority Jobs priority 
     */
    public static void logJobPriority(JobID jobid, JobPriority priority){
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobid); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Job,
                       new Keys[] {Keys.JOBID, Keys.JOB_PRIORITY},
                       new String[] {jobid.toString(), priority.toString()}, jobid);
      }
    }
    /**
     * Log job's submit-time/launch-time 
     * @param jobid job id
     * @param submitTime job's submit time
     * @param launchTime job's launch time
     * @param restartCount number of times the job got restarted
     * @deprecated Use {@link #logJobInfo(JobID, long, long)} instead.
     */
    @Deprecated
    public static void logJobInfo(JobID jobid, long submitTime, long launchTime,
                                  int restartCount){
      logJobInfo(jobid, submitTime, launchTime);
    }

    public static void logJobInfo(JobID jobid, long submitTime, long launchTime)
    {
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobid); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Job,
                       new Keys[] {Keys.JOBID, Keys.SUBMIT_TIME, 
                                   Keys.LAUNCH_TIME},
                       new String[] {jobid.toString(), 
                                     String.valueOf(submitTime), 
                                     String.valueOf(launchTime)}, jobid);
      }
    }
  }
  
  /**
   * Helper class for logging or reading back events related to Task's start, finish or failure. 
   * All events logged by this class are logged in a separate file per job in 
   * job tracker history. These events map to TIPs in jobtracker. 
   */
  public static class Task extends KeyValuePair{
    private Map <String, TaskAttempt> taskAttempts = new TreeMap<String, TaskAttempt>(); 

    /**
     * Log start time of task (TIP).
     * @param taskId task id
     * @param taskType MAP or REDUCE
     * @param startTime startTime of tip. 
     */
    public static void logStarted(TaskID taskId, String taskType, 
                                  long startTime, String splitLocations) {
      JobID id = taskId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Task, 
                       new Keys[]{Keys.TASKID, Keys.TASK_TYPE ,
                                  Keys.START_TIME, Keys.SPLITS}, 
                       new String[]{taskId.toString(), taskType,
                                    String.valueOf(startTime),
                                    splitLocations}, id);
      }
    }
    /**
     * Log finish time of task. 
     * @param taskId task id
     * @param taskType MAP or REDUCE
     * @param finishTime finish timeof task in ms
     */
    public static void logFinished(TaskID taskId, String taskType, 
                                   long finishTime, Counters counters){
      JobID id = taskId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id);

      if (null != writer){
         JobHistory.log(writer, RecordTypes.Task, 
                        new Keys[]{Keys.TASKID, Keys.TASK_TYPE, 
                                   Keys.TASK_STATUS, Keys.FINISH_TIME,
                                   Keys.COUNTERS}, 
                        new String[]{ taskId.toString(), taskType, Values.SUCCESS.name(), 
                                      String.valueOf(finishTime),
                                      counters.makeEscapedCompactString()}, id);
      }
    }

    /**
     * Update the finish time of task. 
     * @param taskId task id
     * @param finishTime finish time of task in ms
     */
    public static void logUpdates(TaskID taskId, long finishTime){
      JobID id = taskId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Task, 
                       new Keys[]{Keys.TASKID, Keys.FINISH_TIME}, 
                       new String[]{ taskId.toString(), 
                                     String.valueOf(finishTime)}, id);
      }
    }

    /**
     * Log job failed event.
     * @param taskId task id
     * @param taskType MAP or REDUCE.
     * @param time timestamp when job failed detected. 
     * @param error error message for failure. 
     */
    public static void logFailed(TaskID taskId, String taskType, long time, String error){
      logFailed(taskId, taskType, time, error, null);
    }
    
    /**
     * @param failedDueToAttempt The attempt that caused the failure, if any
     */
    public static void logFailed(TaskID taskId, String taskType, long time,
                                 String error, 
                                 TaskAttemptID failedDueToAttempt){
      JobID id = taskId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id); 

      if (null != writer){
        String failedAttempt = failedDueToAttempt == null
                               ? ""
                               : failedDueToAttempt.toString();
        JobHistory.log(writer, RecordTypes.Task, 
                       new Keys[]{Keys.TASKID, Keys.TASK_TYPE, 
                                  Keys.TASK_STATUS, Keys.FINISH_TIME, 
                                  Keys.ERROR, Keys.TASK_ATTEMPT_ID}, 
                       new String[]{ taskId.toString(),  taskType, 
                                    Values.FAILED.name(), 
                                    String.valueOf(time) , error, 
                                    failedAttempt}, id);
      }
    }
    /**
     * Returns all task attempts for this task. <task attempt id - TaskAttempt>
     */
    public Map<String, TaskAttempt> getTaskAttempts(){
      return this.taskAttempts;
    }
  }

  /**
   * Base class for Map and Reduce TaskAttempts. 
   */
  public static class TaskAttempt extends Task{} 

  /**
   * Helper class for logging or reading back events related to start, finish or failure of 
   * a Map Attempt on a node.
   */
  public static class MapAttempt extends TaskAttempt{
    /**
     * Log start time of this map task attempt. 
     * @param taskAttemptId task attempt id
     * @param startTime start time of task attempt as reported by task tracker. 
     * @param hostName host name of the task attempt. 
     * @deprecated Use 
     *             {@link #logStarted(TaskAttemptID, long, String, int, String)}
     */
    @Deprecated
    public static void logStarted(TaskAttemptID taskAttemptId, long startTime, String hostName){
      logStarted(taskAttemptId, startTime, hostName, -1, Values.MAP.name());
    }
    
    /**
     * Log start time of this map task attempt.
     *  
     * @param taskAttemptId task attempt id
     * @param startTime start time of task attempt as reported by task tracker. 
     * @param trackerName name of the tracker executing the task attempt.
     * @param httpPort http port of the task tracker executing the task attempt
     * @param taskType Whether the attempt is cleanup or setup or map 
     */
    public static void logStarted(TaskAttemptID taskAttemptId, long startTime,
                                  String trackerName, int httpPort, 
                                  String taskType) {
      JobID id = taskAttemptId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.MapAttempt, 
                       new Keys[]{ Keys.TASK_TYPE, Keys.TASKID, 
                                   Keys.TASK_ATTEMPT_ID, Keys.START_TIME, 
                                   Keys.TRACKER_NAME, Keys.HTTP_PORT},
                       new String[]{taskType,
                                    taskAttemptId.getTaskID().toString(), 
                                    taskAttemptId.toString(), 
                                    String.valueOf(startTime), trackerName,
                                    httpPort == -1 ? "" : 
                                      String.valueOf(httpPort)}, id); 
      }
    }
    
    /**
     * Log finish time of map task attempt. 
     * @param taskAttemptId task attempt id 
     * @param finishTime finish time
     * @param hostName host name 
     * @deprecated Use 
     * {@link #logFinished(TaskAttemptID, long, String, String, String, Counters)}
     */
    @Deprecated
    public static void logFinished(TaskAttemptID taskAttemptId, long finishTime, 
                                   String hostName){
      logFinished(taskAttemptId, finishTime, hostName, Values.MAP.name(), "", 
                  new Counters());
    }

    /**
     * Log finish time of map task attempt. 
     * 
     * @param taskAttemptId task attempt id 
     * @param finishTime finish time
     * @param hostName host name 
     * @param taskType Whether the attempt is cleanup or setup or map 
     * @param stateString state string of the task attempt
     * @param counter counters of the task attempt
     */
    public static void logFinished(TaskAttemptID taskAttemptId, 
                                   long finishTime, 
                                   String hostName,
                                   String taskType,
                                   String stateString, 
                                   Counters counter) {
      JobID id = taskAttemptId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.MapAttempt, 
                       new Keys[]{ Keys.TASK_TYPE, Keys.TASKID, 
                                   Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS, 
                                   Keys.FINISH_TIME, Keys.HOSTNAME, 
                                   Keys.STATE_STRING, Keys.COUNTERS},
                       new String[]{taskType, 
                                    taskAttemptId.getTaskID().toString(),
                                    taskAttemptId.toString(), 
                                    Values.SUCCESS.name(),  
                                    String.valueOf(finishTime), hostName, 
                                    stateString, 
                                    counter.makeEscapedCompactString()}, id); 
      }
    }

    /**
     * Log task attempt failed event.  
     * @param taskAttemptId task attempt id
     * @param timestamp timestamp
     * @param hostName hostname of this task attempt.
     * @param error error message if any for this task attempt.
     * @deprecated Use
     * {@link #logFailed(TaskAttemptID, long, String, String, String)} 
     */
    @Deprecated
    public static void logFailed(TaskAttemptID taskAttemptId, 
                                 long timestamp, String hostName, 
                                 String error) {
      logFailed(taskAttemptId, timestamp, hostName, error, Values.MAP.name());
    }

    /**
     * Log task attempt failed event. 
     *  
     * @param taskAttemptId task attempt id
     * @param timestamp timestamp
     * @param hostName hostname of this task attempt.
     * @param error error message if any for this task attempt. 
     * @param taskType Whether the attempt is cleanup or setup or map 
     */
    public static void logFailed(TaskAttemptID taskAttemptId, 
                                 long timestamp, String hostName, 
                                 String error, String taskType) {
      JobID id = taskAttemptId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.MapAttempt, 
                       new Keys[]{Keys.TASK_TYPE, Keys.TASKID, 
                                  Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS, 
                                  Keys.FINISH_TIME, Keys.HOSTNAME, Keys.ERROR},
                       new String[]{ taskType, 
                                     taskAttemptId.getTaskID().toString(),
                                     taskAttemptId.toString(), 
                                     Values.FAILED.name(),
                                     String.valueOf(timestamp), 
                                     hostName, error}, id); 
      }
    }
    
    /**
     * Log task attempt killed event.  
     * @param taskAttemptId task attempt id
     * @param timestamp timestamp
     * @param hostName hostname of this task attempt.
     * @param error error message if any for this task attempt. 
     * @deprecated Use 
     * {@link #logKilled(TaskAttemptID, long, String, String, String)}
     */
    @Deprecated
    public static void logKilled(TaskAttemptID taskAttemptId, 
                                 long timestamp, String hostName, String error){
      logKilled(taskAttemptId, timestamp, hostName, error, Values.MAP.name());
    } 
    
    /**
     * Log task attempt killed event.  
     * 
     * @param taskAttemptId task attempt id
     * @param timestamp timestamp
     * @param hostName hostname of this task attempt.
     * @param error error message if any for this task attempt. 
     * @param taskType Whether the attempt is cleanup or setup or map 
     */
    public static void logKilled(TaskAttemptID taskAttemptId, 
                                 long timestamp, String hostName,
                                 String error, String taskType) {
      JobID id = taskAttemptId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.MapAttempt, 
                       new Keys[]{Keys.TASK_TYPE, Keys.TASKID,
                                  Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS, 
                                  Keys.FINISH_TIME, Keys.HOSTNAME,
                                  Keys.ERROR},
                       new String[]{ taskType, 
                                     taskAttemptId.getTaskID().toString(), 
                                     taskAttemptId.toString(),
                                     Values.KILLED.name(),
                                     String.valueOf(timestamp), 
                                     hostName, error}, id); 
      }
    } 
  }
  /**
   * Helper class for logging or reading back events related to start, finish or failure of 
   * a Map Attempt on a node.
   */
  public static class ReduceAttempt extends TaskAttempt{
    /**
     * Log start time of  Reduce task attempt. 
     * @param taskAttemptId task attempt id
     * @param startTime start time
     * @param hostName host name 
     * @deprecated Use 
     * {@link #logStarted(TaskAttemptID, long, String, int, String)}
     */
    @Deprecated
    public static void logStarted(TaskAttemptID taskAttemptId, 
                                  long startTime, String hostName){
      logStarted(taskAttemptId, startTime, hostName, -1, Values.REDUCE.name());
    }
    
    /**
     * Log start time of  Reduce task attempt. 
     * 
     * @param taskAttemptId task attempt id
     * @param startTime start time
     * @param trackerName tracker name 
     * @param httpPort the http port of the tracker executing the task attempt
     * @param taskType Whether the attempt is cleanup or setup or reduce 
     */
    public static void logStarted(TaskAttemptID taskAttemptId, 
                                  long startTime, String trackerName, 
                                  int httpPort, 
                                  String taskType) {
              JobID id = taskAttemptId.getJobID();
	      ArrayList<PrintWriter> writer = fileManager.getWriters(id); 

              if (null != writer){
		  JobHistory.log(writer, RecordTypes.ReduceAttempt, 
				 new Keys[]{  Keys.TASK_TYPE, Keys.TASKID, 
					      Keys.TASK_ATTEMPT_ID, Keys.START_TIME,
					      Keys.TRACKER_NAME, Keys.HTTP_PORT},
				 new String[]{taskType,
					      taskAttemptId.getTaskID().toString(), 
					      taskAttemptId.toString(), 
					      String.valueOf(startTime), trackerName,
					      httpPort == -1 ? "" : 
						String.valueOf(httpPort)}, id); 
              }
	    }
	    
	    /**
	     * Log finished event of this task. 
	     * @param taskAttemptId task attempt id
	     * @param shuffleFinished shuffle finish time
	     * @param sortFinished sort finish time
	     * @param finishTime finish time of task
	     * @param hostName host name where task attempt executed
	     * @deprecated Use 
	     * {@link #logFinished(TaskAttemptID, long, long, long, String, String, String, Counters)}
	     */
	    @Deprecated
	    public static void logFinished(TaskAttemptID taskAttemptId, long shuffleFinished, 
					   long sortFinished, long finishTime, 
					   String hostName){
	      logFinished(taskAttemptId, shuffleFinished, sortFinished, 
			  finishTime, hostName, Values.REDUCE.name(),
			  "", new Counters());
	    }
	    
	    /**
	     * Log finished event of this task. 
	     * 
	     * @param taskAttemptId task attempt id
	     * @param shuffleFinished shuffle finish time
	     * @param sortFinished sort finish time
	     * @param finishTime finish time of task
	     * @param hostName host name where task attempt executed
	     * @param taskType Whether the attempt is cleanup or setup or reduce 
	     * @param stateString the state string of the attempt
	     * @param counter counters of the attempt
	     */
	    public static void logFinished(TaskAttemptID taskAttemptId, 
					   long shuffleFinished, 
					   long sortFinished, long finishTime, 
					   String hostName, String taskType,
					   String stateString, Counters counter) {
        JobID id = taskAttemptId.getJobID();
        ArrayList<PrintWriter> writer = fileManager.getWriters(id); 

        if (null != writer){
          JobHistory.log(writer, RecordTypes.ReduceAttempt, 
                         new Keys[]{ Keys.TASK_TYPE, Keys.TASKID, 
                                     Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS, 
                                     Keys.SHUFFLE_FINISHED, Keys.SORT_FINISHED,
                                     Keys.FINISH_TIME, Keys.HOSTNAME, 
                                     Keys.STATE_STRING, Keys.COUNTERS},
                         new String[]{taskType,
                                      taskAttemptId.getTaskID().toString(), 
                                      taskAttemptId.toString(), 
                                      Values.SUCCESS.name(), 
                                      String.valueOf(shuffleFinished), 
                                      String.valueOf(sortFinished),
                                      String.valueOf(finishTime), hostName,
                                      stateString, 
                                      counter.makeEscapedCompactString()}, id); 
        }
    }
    
    /**
     * Log failed reduce task attempt. 
     * @param taskAttemptId task attempt id
     * @param timestamp time stamp when task failed
     * @param hostName host name of the task attempt.  
     * @param error error message of the task.
     * @deprecated Use 
     * {@link #logFailed(TaskAttemptID, long, String, String, String)} 
     */
    @Deprecated
    public static void logFailed(TaskAttemptID taskAttemptId, long timestamp, 
                                 String hostName, String error){
      logFailed(taskAttemptId, timestamp, hostName, error, Values.REDUCE.name());
    }
    
    /**
     * Log failed reduce task attempt.
     *  
     * @param taskAttemptId task attempt id
     * @param timestamp time stamp when task failed
     * @param hostName host name of the task attempt.  
     * @param error error message of the task. 
     * @param taskType Whether the attempt is cleanup or setup or reduce 
     */
    public static void logFailed(TaskAttemptID taskAttemptId, long timestamp, 
                                 String hostName, String error, 
                                 String taskType) {
      JobID id = taskAttemptId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.ReduceAttempt, 
                       new Keys[]{  Keys.TASK_TYPE, Keys.TASKID, 
                                    Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS, 
                                    Keys.FINISH_TIME, Keys.HOSTNAME,
                                    Keys.ERROR },
                       new String[]{ taskType, 
                                     taskAttemptId.getTaskID().toString(), 
                                     taskAttemptId.toString(), 
                                     Values.FAILED.name(), 
                                     String.valueOf(timestamp), hostName, error }, id); 
      }
    }
    
    /**
     * Log killed reduce task attempt. 
     * @param taskAttemptId task attempt id
     * @param timestamp time stamp when task failed
     * @param hostName host name of the task attempt.  
     * @param error error message of the task.
     * @deprecated Use 
     * {@link #logKilled(TaskAttemptID, long, String, String, String)} 
     */
    @Deprecated
    public static void logKilled(TaskAttemptID taskAttemptId, long timestamp, 
                                 String hostName, String error) {
      logKilled(taskAttemptId, timestamp, hostName, error, Values.REDUCE.name());
    }
    
    /**
     * Log killed reduce task attempt. 
     * 
     * @param taskAttemptId task attempt id
     * @param timestamp time stamp when task failed
     * @param hostName host name of the task attempt.  
     * @param error error message of the task. 
     * @param taskType Whether the attempt is cleanup or setup or reduce 
    */
    public static void logKilled(TaskAttemptID taskAttemptId, long timestamp, 
                                 String hostName, String error, 
                                 String taskType) {
      JobID id = taskAttemptId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id); 

      if (null != writer){
        JobHistory.log(writer, RecordTypes.ReduceAttempt, 
                       new Keys[]{  Keys.TASK_TYPE, Keys.TASKID, 
                                    Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS, 
                                    Keys.FINISH_TIME, Keys.HOSTNAME, 
                                    Keys.ERROR },
                       new String[]{ taskType,
                                     taskAttemptId.getTaskID().toString(), 
                                     taskAttemptId.toString(), 
                                     Values.KILLED.name(), 
                                     String.valueOf(timestamp), 
                                     hostName, error }, id); 
      }
    }
  }

  /**
   * Callback interface for reading back log events from JobHistory. This interface 
   * should be implemented and passed to JobHistory.parseHistory() 
   *
   */
  public static interface Listener{
    /**
     * Callback method for history parser. 
     * @param recType type of record, which is the first entry in the line. 
     * @param values a map of key-value pairs as thry appear in history.
     * @throws IOException
     */
    public void handle(RecordTypes recType, Map<Keys, String> values) throws IOException; 
  }

  static long directoryTime(String year, String seg2, String seg3) {
    // set to current time.  In debug mode, this is where the month
    // and day get set.
    Calendar result = Calendar.getInstance();
    // canonicalize by filling in unset fields
    result.setTimeInMillis(System.currentTimeMillis());

    result.set(Calendar.YEAR, Integer.parseInt(year));

    // months are 0-based in Calendar, but people will expect January
    // to be month #1 .  Therefore the number is bumped before we make the 
    // directory name and must be debumped to seek the time.
    result.set(DEBUG_MODE ? Calendar.HOUR : Calendar.MONTH,
               Integer.parseInt(seg2) - 1);

    result.set(DEBUG_MODE ? Calendar.MINUTE : Calendar.DAY_OF_MONTH,
               Integer.parseInt(seg3));

    return result.getTimeInMillis();
  }
  
  /**
   * Delete history files older than one month. Update master index and remove all 
   * jobs older than one month. Also if a job tracker has no jobs in last one month
   * remove reference to the job tracker. 
   *
   */
  public static class HistoryCleaner implements Runnable {
    static final long ONE_DAY_IN_MS = 24 * 60 * 60 * 1000L;
    static final long DIRECTORY_LIFE_IN_MS
      = DEBUG_MODE ? 20 * 60 * 1000L : 30 * ONE_DAY_IN_MS;
    static final long RUN_INTERVAL
      = DEBUG_MODE ? 10L * 60L * 1000L : ONE_DAY_IN_MS;
    private long now; 
    private static final AtomicBoolean isRunning = new AtomicBoolean(false); 
    private static long lastRan = 0; 

    private static Pattern parseDirectory
      = Pattern.compile(".+/([0-9]+)/([0-9]+)/([0-9]+)/[0-9]+/?");

    /**
     * Cleans up history data. 
     */
    public void run() {
      if (isRunning.getAndSet(true)) {
        return; 
      }
      now = System.currentTimeMillis();
      // clean history only once a day at max
      if (lastRan != 0 && (now - lastRan) < RUN_INTERVAL) {
        isRunning.set(false);
        return; 
      }
      lastRan = now;

      Set<String> deletedPathnames = new HashSet<String>();

      // XXXXX debug code
      boolean printedOneDeletee = false;
      boolean printedOneMovedFile = false;

      try {
        Path[] datedDirectories
          = FileUtil.stat2Paths(localGlobber(DONEDIR_FS, DONE,
                                             DONE_BEFORE_SERIAL_TAIL, null));
        // find directories older than 30 days
        for (int i = 0; i < datedDirectories.length; ++i) {
          String thisDir = datedDirectories[i].toString();
          Matcher pathMatcher = parseDirectory.matcher(thisDir);

          if (pathMatcher.matches()) {
            long dirTime = directoryTime(pathMatcher.group(1),
                                         pathMatcher.group(2),
                                         pathMatcher.group(3));

            if (DEBUG_MODE) {
              System.err.println("HistoryCleaner.run just parsed " + thisDir
                                 + " as year/month/day = " + pathMatcher.group(1)
                                 + "/" + pathMatcher.group(2) + "/"
                                 + pathMatcher.group(3));
            }

            if (dirTime < now - DIRECTORY_LIFE_IN_MS) {

              if (DEBUG_MODE) {
                Calendar then = Calendar.getInstance();
                then.setTimeInMillis(dirTime);
                Calendar nnow = Calendar.getInstance();
                nnow.setTimeInMillis(now);
                
                System.err.println("HistoryCleaner.run directory: " + thisDir
                                   + " because its time is " + then
                                   + " but it's now " + nnow);
                System.err.println("then = " + dirTime);
                System.err.println("now  = " + now);
              }

              // remove every file in the directory and save the name
              // so we can remove it from jobHistoryFileMap
              Path[] deletees
                = FileUtil.stat2Paths(localGlobber(DONEDIR_FS,
                                                   datedDirectories[i],
                                                   "/*/*", // sn + individual files
                                                   null));

              for (int j = 0; j < deletees.length; ++j) {

                if (DEBUG_MODE && !printedOneDeletee) {
                  System.err.println("HistoryCleaner.run deletee: " + deletees[j].toString());
                  printedOneDeletee = true;
                }

                DONEDIR_FS.delete(deletees[j]);
                deletedPathnames.add(deletees[j].toString());
              }
              synchronized (existingDoneSubdirs) {
                if (!existingDoneSubdirs.contains(datedDirectories[i]))
                  {
                    LOG.warn("JobHistory: existingDoneSubdirs doesn't contain "
                             + datedDirectories[i] + ", but should.");
                  }
                DONEDIR_FS.delete(datedDirectories[i], true);
                existingDoneSubdirs.remove(datedDirectories[i]);
              }
            }
          }
        }

        //walking over the map to purge entries from jobHistoryFileMap
        synchronized (jobHistoryFileMap) {
          Iterator<Entry<JobID, MovedFileInfo>> it =
            jobHistoryFileMap.entrySet().iterator();
          while (it.hasNext()) {
            MovedFileInfo info = it.next().getValue();

            if (DEBUG_MODE && !printedOneMovedFile) {
              System.err.println("HistoryCleaner.run a moved file: " + info.historyFile);
              printedOneMovedFile = true;
            }            

            if (deletedPathnames.contains(info.historyFile)) {
              it.remove();
            }
          }
        }
      } catch (IOException ie) {
        LOG.info("Error cleaning up history directory" + 
                 StringUtils.stringifyException(ie));
      } finally {
          isRunning.set(false);
      }
    }
    
    static long getLastRan() {
      return lastRan;
    }
  }

  /**
   * Return the TaskLogsUrl of a particular TaskAttempt
   * 
   * @param attempt
   * @return the taskLogsUrl. null if http-port or tracker-name or
   *         task-attempt-id are unavailable.
   */
  public static String getTaskLogsUrl(JobHistory.TaskAttempt attempt) {
    if (attempt.get(Keys.HTTP_PORT).equals("")
        || attempt.get(Keys.TRACKER_NAME).equals("")
        || attempt.get(Keys.TASK_ATTEMPT_ID).equals("")) {
      return null;
    }

    String taskTrackerName =
      JobInProgress.convertTrackerNameToHostName(
        attempt.get(Keys.TRACKER_NAME));
    return TaskLogServlet.getTaskLogUrl(taskTrackerName, attempt
        .get(Keys.HTTP_PORT), attempt.get(Keys.TASK_ATTEMPT_ID));
  }
}