mapreduce/src/test/mapred/org/apache/hadoop/mapred/HadoopTestCase.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.mapred;

import junit.framework.TestCase;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.File;
import java.io.IOException;

/**
 * Abstract Test case class to run MR in local or cluster mode and in local FS
 * or DFS.
 *
 * The Hadoop instance is started and stopped on each test method.
 *
 * If using DFS the filesystem is reformated at each start (test method).
 *
 * Job Configurations should be created using a configuration returned by the
 * 'createJobConf()' method.
 */
public abstract class HadoopTestCase extends TestCase {
  public static final int LOCAL_MR = 1;
  public static final int CLUSTER_MR = 2;
  public static final int LOCAL_FS = 4;
  public static final int DFS_FS = 8;

  private boolean localMR;
  private boolean localFS;

  private int taskTrackers;
  private int dataNodes;

  /**
   * Creates a testcase for local or cluster MR using DFS.
   *
   * The DFS will be formatted regardless if there was one or not before in the
   * given location.
   *
   * @param mrMode indicates if the MR should be local (LOCAL_MR) or cluster
   * (CLUSTER_MR)
   * @param fsMode indicates if the FS should be local (LOCAL_FS) or DFS (DFS_FS)
   *
   * local FS when using relative PATHs)
   *
   * @param taskTrackers number of task trackers to start when using cluster
   *
   * @param dataNodes number of data nodes to start when using DFS
   *
   * @throws IOException thrown if the base directory cannot be set.
   */
  public HadoopTestCase(int mrMode, int fsMode, int taskTrackers, int dataNodes)
    throws IOException {
    if (mrMode != LOCAL_MR && mrMode != CLUSTER_MR) {
      throw new IllegalArgumentException(
                                         "Invalid MapRed mode, must be LOCAL_MR or CLUSTER_MR");
    }
    if (fsMode != LOCAL_FS && fsMode != DFS_FS) {
      throw new IllegalArgumentException(
                                         "Invalid FileSystem mode, must be LOCAL_FS or DFS_FS");
    }
    if (taskTrackers < 1) {
      throw new IllegalArgumentException(
                                         "Invalid taskTrackers value, must be greater than 0");
    }
    if (dataNodes < 1) {
      throw new IllegalArgumentException(
                                         "Invalid dataNodes value, must be greater than 0");
    }
    localMR = (mrMode == LOCAL_MR);
    localFS = (fsMode == LOCAL_FS);
    /*
      JobConf conf = new JobConf();
      fsRoot = conf.get("hadoop.tmp.dir");

      if (fsRoot == null) {
      throw new IllegalArgumentException(
      "hadoop.tmp.dir is not defined");
      }

      fsRoot = fsRoot.replace(' ', '+') + "/fs";

      File file = new File(fsRoot);
      if (!file.exists()) {
      if (!file.mkdirs()) {
      throw new RuntimeException("Could not create FS base path: " + file);
      }
      }
    */
    this.taskTrackers = taskTrackers;
    this.dataNodes = dataNodes;
  }

  /**
   * Indicates if the MR is running in local or cluster mode.
   *
   * @return returns TRUE if the MR is running locally, FALSE if running in
   * cluster mode.
   */
  public boolean isLocalMR() {
    return localMR;
  }

  /**
   * Indicates if the filesystem is local or DFS.
   *
   * @return returns TRUE if the filesystem is local, FALSE if it is DFS.
   */
  public boolean isLocalFS() {
    return localFS;
  }


  private MiniDFSCluster dfsCluster = null;
  private MiniMRCluster mrCluster = null;
  private FileSystem fileSystem = null;

  /**
   * Creates Hadoop instance based on constructor configuration before
   * a test case is run.
   *
   * @throws Exception
   */
  protected void setUp() throws Exception {
    super.setUp();
    if (localFS) {
      fileSystem = FileSystem.getLocal(new JobConf());
    }
    else {
      dfsCluster = new MiniDFSCluster(new JobConf(), dataNodes, true, null);
      fileSystem = dfsCluster.getFileSystem();
    }
    if (localMR) {
    }
    else {
      //noinspection deprecation
      mrCluster = new MiniMRCluster(taskTrackers, fileSystem.getUri().toString(), 1);
    }
  }

  /**
   * Destroys Hadoop instance based on constructor configuration after
   * a test case is run.
   *
   * @throws Exception
   */
  protected void tearDown() throws Exception {
    try {
      if (mrCluster != null) {
        mrCluster.shutdown();
      }
    }
    catch (Exception ex) {
      System.out.println(ex);
    }
    try {
      if (dfsCluster != null) {
        dfsCluster.shutdown();
      }
    }
    catch (Exception ex) {
      System.out.println(ex);
    }
    super.tearDown();
  }

  /**
   * Returns the Filesystem in use.
   *
   * TestCases should use this Filesystem as it
   * is properly configured with the workingDir for relative PATHs.
   *
   * @return the filesystem used by Hadoop.
   */
  protected FileSystem getFileSystem() {
    return fileSystem;
  }

  /**
   * Returns a job configuration preconfigured to run against the Hadoop
   * managed by the testcase.
   * @return configuration that works on the testcase Hadoop instance
   */
  protected JobConf createJobConf() {
    return (localMR) ? new JobConf() : mrCluster.createJobConf();
  }

}