mapreduce/src/test/mapred/org/apache/hadoop/conf/TestJobConf.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.conf;

import org.junit.Assert;
import org.junit.Test;

import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.JobContext;

public class TestJobConf {

  @Test
  public void testProfileParamsDefaults() {
    JobConf configuration = new JobConf();

    Assert.assertNull(configuration.get(JobContext.TASK_PROFILE_PARAMS));

    String result = configuration.getProfileParams();

    Assert.assertNotNull(result);
    Assert.assertTrue(result.contains("file=%s"));
    Assert.assertTrue(result.startsWith("-agentlib:hprof"));
  }

  @Test
  public void testProfileParamsSetter() {
    JobConf configuration = new JobConf();

    configuration.setProfileParams("test");
    Assert.assertEquals("test", configuration.get(JobContext.TASK_PROFILE_PARAMS));
  }

  @Test
  public void testProfileParamsGetter() {
    JobConf configuration = new JobConf();

    configuration.set(JobContext.TASK_PROFILE_PARAMS, "test");
    Assert.assertEquals("test", configuration.getProfileParams());
  }

  /**
   * Testing mapred.task.maxvmem replacement with new values
   *
   */
  @Test
  public void testMemoryConfigForMapOrReduceTask(){
    JobConf configuration = new JobConf();
    configuration.set(JobContext.MAP_MEMORY_MB,String.valueOf(300));
    configuration.set(JobContext.REDUCE_MEMORY_MB,String.valueOf(300));
    Assert.assertEquals(configuration.getMemoryForMapTask(),300);
    Assert.assertEquals(configuration.getMemoryForReduceTask(),300);

    configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
    configuration.set(JobContext.MAP_MEMORY_MB,String.valueOf(300));
    configuration.set(JobContext.REDUCE_MEMORY_MB,String.valueOf(300));
    Assert.assertEquals(configuration.getMemoryForMapTask(),2);
    Assert.assertEquals(configuration.getMemoryForReduceTask(),2);

    configuration = new JobConf();
    configuration.set("mapred.task.maxvmem" , "-1");
    configuration.set(JobContext.MAP_MEMORY_MB,String.valueOf(300));
    configuration.set(JobContext.REDUCE_MEMORY_MB,String.valueOf(400));
    Assert.assertEquals(configuration.getMemoryForMapTask(), 300);
    Assert.assertEquals(configuration.getMemoryForReduceTask(), 400);

    configuration = new JobConf();
    configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
    configuration.set(JobContext.MAP_MEMORY_MB,"-1");
    configuration.set(JobContext.REDUCE_MEMORY_MB,"-1");
    Assert.assertEquals(configuration.getMemoryForMapTask(),2);
    Assert.assertEquals(configuration.getMemoryForReduceTask(),2);

    configuration = new JobConf();
    configuration.set("mapred.task.maxvmem" , String.valueOf(-1));
    configuration.set(JobContext.MAP_MEMORY_MB,"-1");
    configuration.set(JobContext.REDUCE_MEMORY_MB,"-1");
    Assert.assertEquals(configuration.getMemoryForMapTask(),-1);
    Assert.assertEquals(configuration.getMemoryForReduceTask(),-1);    

    configuration = new JobConf();
    configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
    configuration.set(JobContext.MAP_MEMORY_MB, "3");
    configuration.set(JobContext.REDUCE_MEMORY_MB, "3");
    Assert.assertEquals(configuration.getMemoryForMapTask(),2);
    Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
    
  }

  /**
   * Test that negative values for MAPRED_TASK_MAXVMEM_PROPERTY cause
   * new configuration keys' values to be used.
   */
  @Test
  public void testNegativeValueForTaskVmem() {
    JobConf configuration = new JobConf();
    
    configuration.set(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, "-3");
    configuration.set(JobContext.MAP_MEMORY_MB, "4");
    configuration.set(JobContext.REDUCE_MEMORY_MB, "5");
    Assert.assertEquals(4, configuration.getMemoryForMapTask());
    Assert.assertEquals(5, configuration.getMemoryForReduceTask());
    
  }
  
  /**
   * Test that negative values for all memory configuration properties causes
   * APIs to disable memory limits
   */
  @Test
  public void testNegativeValuesForMemoryParams() {
    JobConf configuration = new JobConf();
    
    configuration.set(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, "-4");
    configuration.set(JobContext.MAP_MEMORY_MB, "-5");
    configuration.set(JobContext.REDUCE_MEMORY_MB, "-6");
    
    Assert.assertEquals(JobConf.DISABLED_MEMORY_LIMIT,
                        configuration.getMemoryForMapTask());
    Assert.assertEquals(JobConf.DISABLED_MEMORY_LIMIT,
                        configuration.getMemoryForReduceTask());
    Assert.assertEquals(JobConf.DISABLED_MEMORY_LIMIT,
                        configuration.getMaxVirtualMemoryForTask());
  }
  
  /**
   *   Test deprecated accessor and mutator method for mapred.task.maxvmem
   */
  @Test
  public void testMaxVirtualMemoryForTask() {
    JobConf configuration = new JobConf();

    //get test case
    configuration.set(JobContext.MAP_MEMORY_MB, String.valueOf(300));
    configuration.set(JobContext.REDUCE_MEMORY_MB, String.valueOf(-1));
    Assert.assertEquals(
      configuration.getMaxVirtualMemoryForTask(), 300 * 1024 * 1024);

    configuration = new JobConf();
    configuration.set(JobContext.MAP_MEMORY_MB, String.valueOf(-1));
    configuration.set(JobContext.REDUCE_MEMORY_MB, String.valueOf(200));
    Assert.assertEquals(
      configuration.getMaxVirtualMemoryForTask(), 200 * 1024 * 1024);

    configuration = new JobConf();
    configuration.set(JobContext.MAP_MEMORY_MB, String.valueOf(-1));
    configuration.set(JobContext.REDUCE_MEMORY_MB, String.valueOf(-1));
    configuration.set("mapred.task.maxvmem", String.valueOf(1 * 1024 * 1024));
    Assert.assertEquals(
      configuration.getMaxVirtualMemoryForTask(), 1 * 1024 * 1024);

    configuration = new JobConf();
    configuration.set("mapred.task.maxvmem", String.valueOf(1 * 1024 * 1024));
    Assert.assertEquals(
      configuration.getMaxVirtualMemoryForTask(), 1 * 1024 * 1024);

    //set test case

    configuration = new JobConf();
    configuration.setMaxVirtualMemoryForTask(2 * 1024 * 1024);
    Assert.assertEquals(configuration.getMemoryForMapTask(), 2);
    Assert.assertEquals(configuration.getMemoryForReduceTask(), 2);

    configuration = new JobConf();   
    configuration.set(JobContext.MAP_MEMORY_MB, String.valueOf(300));
    configuration.set(JobContext.REDUCE_MEMORY_MB, String.valueOf(400));
    configuration.setMaxVirtualMemoryForTask(2 * 1024 * 1024);
    Assert.assertEquals(configuration.getMemoryForMapTask(), 2);
    Assert.assertEquals(configuration.getMemoryForReduceTask(), 2);
    
    
  }
}