branch-2.0.4-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReporter.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.mapred;

import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

import static org.junit.Assert.*;

/**
 * Tests the old mapred APIs with {@link Reporter#getProgress()}.
 */
public class TestReporter {
  private static final Path rootTempDir =
    new Path(System.getProperty("test.build.data", "/tmp"));
  private static final Path testRootTempDir = 
    new Path(rootTempDir, "TestReporter");
  
  private static FileSystem fs = null;

  @BeforeClass
  public static void setup() throws Exception {
    fs = FileSystem.getLocal(new Configuration());
    fs.delete(testRootTempDir, true);
    fs.mkdirs(testRootTempDir);
  }

  @AfterClass
  public static void cleanup() throws Exception {
    fs.delete(testRootTempDir, true);
  }
  
  // an input with 4 lines
  private static final String INPUT = "Hi\nHi\nHi\nHi\n";
  private static final int INPUT_LINES = INPUT.split("\n").length;
  
  @SuppressWarnings("deprecation")
  static class ProgressTesterMapper extends MapReduceBase 
  implements Mapper<LongWritable, Text, Text, Text> {
    private float progressRange = 0;
    private int numRecords = 0;
    private Reporter reporter = null;
    
    @Override
    public void configure(JobConf job) {
      super.configure(job);
      // set the progress range accordingly
      if (job.getNumReduceTasks() == 0) {
        progressRange = 1f;
      } else {
        progressRange = 0.667f;
      }
    }
    
    @Override
    public void map(LongWritable key, Text value, 
                    OutputCollector<Text, Text> output, Reporter reporter) 
    throws IOException {
      this.reporter = reporter;
      
      // calculate the actual map progress
      float mapProgress = ((float)++numRecords)/INPUT_LINES;
      // calculate the attempt progress based on the progress range
      float attemptProgress = progressRange * mapProgress;
      assertEquals("Invalid progress in map", 
                   attemptProgress, reporter.getProgress(), 0f);
      output.collect(new Text(value.toString() + numRecords), value);
    }
    
    @Override
    public void close() throws IOException {
      super.close();
      assertEquals("Invalid progress in map cleanup", 
                   progressRange, reporter.getProgress(), 0f);
    }
  }

  static class StatusLimitMapper extends
      org.apache.hadoop.mapreduce.Mapper<LongWritable, Text, Text, Text> {

    @Override
    public void map(LongWritable key, Text value, Context context)
        throws IOException {
      StringBuilder sb = new StringBuilder(512);
      for (int i = 0; i < 1000; i++) {
        sb.append("a");
      }
      context.setStatus(sb.toString());
      int progressStatusLength = context.getConfiguration().getInt(
          MRConfig.PROGRESS_STATUS_LEN_LIMIT_KEY,
          MRConfig.PROGRESS_STATUS_LEN_LIMIT_DEFAULT);

      if (context.getStatus().length() > progressStatusLength) {
        throw new IOException("Status is not truncated");
      }
    }
  }

  /**
   * Test {@link Reporter}'s progress for a map-only job.
   * This will make sure that only the map phase decides the attempt's progress.
   */
  @SuppressWarnings("deprecation")
  @Test
  public void testReporterProgressForMapOnlyJob() throws IOException {
    Path test = new Path(testRootTempDir, "testReporterProgressForMapOnlyJob");
    
    JobConf conf = new JobConf();
    conf.setMapperClass(ProgressTesterMapper.class);
    conf.setMapOutputKeyClass(Text.class);
    // fail early
    conf.setMaxMapAttempts(1);
    conf.setMaxReduceAttempts(0);
    
    RunningJob job = 
      UtilsForTests.runJob(conf, new Path(test, "in"), new Path(test, "out"), 
                           1, 0, INPUT);
    job.waitForCompletion();
    
    assertTrue("Job failed", job.isSuccessful());
  }
  
  /**
   * A {@link Reducer} implementation that checks the progress on every call
   * to {@link Reducer#reduce(Object, Iterator, OutputCollector, Reporter)}.
   */
  @SuppressWarnings("deprecation")
  static class ProgressTestingReducer extends MapReduceBase 
  implements Reducer<Text, Text, Text, Text> {
    private int recordCount = 0;
    private Reporter reporter = null;
    // reduce task has a fixed split of progress amongst copy, shuffle and 
    // reduce phases.
    private final float REDUCE_PROGRESS_RANGE = 1.0f/3;
    private final float SHUFFLE_PROGRESS_RANGE = 1 - REDUCE_PROGRESS_RANGE;
    
    @Override
    public void configure(JobConf job) {
      super.configure(job);
    }
    
    @Override
    public void reduce(Text key, Iterator<Text> values,
        OutputCollector<Text, Text> output, Reporter reporter)
    throws IOException {
      float reducePhaseProgress = ((float)++recordCount)/INPUT_LINES;
      float weightedReducePhaseProgress = 
              reducePhaseProgress * REDUCE_PROGRESS_RANGE;
      assertEquals("Invalid progress in reduce", 
                   SHUFFLE_PROGRESS_RANGE + weightedReducePhaseProgress, 
                   reporter.getProgress(), 0.02f);
      this.reporter = reporter;
    }
    
    @Override
    public void close() throws IOException {
      super.close();
      assertEquals("Invalid progress in reduce cleanup", 
                   1.0f, reporter.getProgress(), 0f);
    }
  }
  
  /**
   * Test {@link Reporter}'s progress for map-reduce job.
   */
  @Test
  public void testReporterProgressForMRJob() throws IOException {
    Path test = new Path(testRootTempDir, "testReporterProgressForMRJob");
    
    JobConf conf = new JobConf();
    conf.setMapperClass(ProgressTesterMapper.class);
    conf.setReducerClass(ProgressTestingReducer.class);
    conf.setMapOutputKeyClass(Text.class);
    // fail early
    conf.setMaxMapAttempts(1);
    conf.setMaxReduceAttempts(1);

    RunningJob job = 
      UtilsForTests.runJob(conf, new Path(test, "in"), new Path(test, "out"), 
                           1, 1, INPUT);
    job.waitForCompletion();
    
    assertTrue("Job failed", job.isSuccessful());
  }

  @Test
  public void testStatusLimit() throws IOException, InterruptedException,
      ClassNotFoundException {
    Path test = new Path(testRootTempDir, "testStatusLimit");

    Configuration conf = new Configuration();
    Path inDir = new Path(test, "in");
    Path outDir = new Path(test, "out");
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(inDir)) {
      fs.delete(inDir, true);
    }
    fs.mkdirs(inDir);
    DataOutputStream file = fs.create(new Path(inDir, "part-" + 0));
    file.writeBytes("testStatusLimit");
    file.close();

    if (fs.exists(outDir)) {
      fs.delete(outDir, true);
    }

    Job job = Job.getInstance(conf, "testStatusLimit");

    job.setMapperClass(StatusLimitMapper.class);
    job.setNumReduceTasks(0);

    FileInputFormat.addInputPath(job, inDir);
    FileOutputFormat.setOutputPath(job, outDir);

    job.waitForCompletion(true);

    assertTrue("Job failed", job.isSuccessful());
  }

}