| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.mapred.gridmix; |
| |
| import org.junit.Test; |
| import static org.junit.Assert.*; |
| |
| import java.io.IOException; |
| |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.mapred.JobConf; |
| import org.apache.hadoop.mapred.gridmix.DebugJobProducer.MockJob; |
| import org.apache.hadoop.mapred.gridmix.TestHighRamJob.DummyGridmixJob; |
| import org.apache.hadoop.mapred.gridmix.TestResourceUsageEmulators.FakeProgressive; |
| import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.TotalHeapUsageEmulatorPlugin; |
| import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.TotalHeapUsageEmulatorPlugin.DefaultHeapUsageEmulator; |
| import org.apache.hadoop.mapreduce.Job; |
| import org.apache.hadoop.mapreduce.MRJobConfig; |
| import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; |
| import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; |
| |
| /** |
| * Test Gridmix memory emulation. |
| */ |
| public class TestGridmixMemoryEmulation { |
| /** |
| * This is a dummy class that fakes heap usage. |
| */ |
| private static class FakeHeapUsageEmulatorCore |
| extends DefaultHeapUsageEmulator { |
| private int numCalls = 0; |
| |
| @Override |
| public void load(long sizeInMB) { |
| ++numCalls; |
| super.load(sizeInMB); |
| } |
| |
| // Get the total number of times load() was invoked |
| int getNumCalls() { |
| return numCalls; |
| } |
| |
| // Get the total number of 1mb objects stored within |
| long getHeapUsageInMB() { |
| return heapSpace.size(); |
| } |
| |
| @Override |
| public void reset() { |
| // no op to stop emulate() from resetting |
| } |
| |
| /** |
| * For re-testing purpose. |
| */ |
| void resetFake() { |
| numCalls = 0; |
| super.reset(); |
| } |
| } |
| |
| /** |
| * This is a dummy class that fakes the heap usage emulator plugin. |
| */ |
| private static class FakeHeapUsageEmulatorPlugin |
| extends TotalHeapUsageEmulatorPlugin { |
| private FakeHeapUsageEmulatorCore core; |
| |
| public FakeHeapUsageEmulatorPlugin(FakeHeapUsageEmulatorCore core) { |
| super(core); |
| this.core = core; |
| } |
| |
| @Override |
| protected long getMaxHeapUsageInMB() { |
| return Long.MAX_VALUE / ONE_MB; |
| } |
| |
| @Override |
| protected long getTotalHeapUsageInMB() { |
| return core.getHeapUsageInMB(); |
| } |
| } |
| |
| /** |
| * Test {@link TotalHeapUsageEmulatorPlugin}'s core heap usage emulation |
| * engine. |
| */ |
| @Test |
| public void testHeapUsageEmulator() throws IOException { |
| FakeHeapUsageEmulatorCore heapEmulator = new FakeHeapUsageEmulatorCore(); |
| |
| long testSizeInMB = 10; // 10 mb |
| long previousHeap = heapEmulator.getHeapUsageInMB(); |
| heapEmulator.load(testSizeInMB); |
| long currentHeap = heapEmulator.getHeapUsageInMB(); |
| |
| // check if the heap has increased by expected value |
| assertEquals("Default heap emulator failed to load 10mb", |
| previousHeap + testSizeInMB, currentHeap); |
| |
| // test reset |
| heapEmulator.resetFake(); |
| assertEquals("Default heap emulator failed to reset", |
| 0, heapEmulator.getHeapUsageInMB()); |
| } |
| |
| /** |
| * Test {@link TotalHeapUsageEmulatorPlugin}. |
| */ |
| @Test |
| public void testTotalHeapUsageEmulatorPlugin() throws Exception { |
| Configuration conf = new Configuration(); |
| // set the dummy resource calculator for testing |
| ResourceCalculatorPlugin monitor = new DummyResourceCalculatorPlugin(); |
| long maxHeapUsage = 1024 * TotalHeapUsageEmulatorPlugin.ONE_MB; // 1GB |
| conf.setLong(DummyResourceCalculatorPlugin.MAXPMEM_TESTING_PROPERTY, |
| maxHeapUsage); |
| monitor.setConf(conf); |
| |
| // no buffer to be reserved |
| conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F); |
| // only 1 call to be made per cycle |
| conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F); |
| long targetHeapUsageInMB = 200; // 200mb |
| |
| // fake progress indicator |
| FakeProgressive fakeProgress = new FakeProgressive(); |
| |
| // fake heap usage generator |
| FakeHeapUsageEmulatorCore fakeCore = new FakeHeapUsageEmulatorCore(); |
| |
| // a heap usage emulator with fake core |
| FakeHeapUsageEmulatorPlugin heapPlugin = |
| new FakeHeapUsageEmulatorPlugin(fakeCore); |
| |
| // test with invalid or missing resource usage value |
| ResourceUsageMetrics invalidUsage = |
| TestResourceUsageEmulators.createMetrics(0); |
| heapPlugin.initialize(conf, invalidUsage, null, null); |
| |
| // test if disabled heap emulation plugin's emulate() call is a no-operation |
| // this will test if the emulation plugin is disabled or not |
| int numCallsPre = fakeCore.getNumCalls(); |
| long heapUsagePre = fakeCore.getHeapUsageInMB(); |
| heapPlugin.emulate(); |
| int numCallsPost = fakeCore.getNumCalls(); |
| long heapUsagePost = fakeCore.getHeapUsageInMB(); |
| |
| // test if no calls are made heap usage emulator core |
| assertEquals("Disabled heap usage emulation plugin works!", |
| numCallsPre, numCallsPost); |
| // test if no calls are made heap usage emulator core |
| assertEquals("Disabled heap usage emulation plugin works!", |
| heapUsagePre, heapUsagePost); |
| |
| // test with get progress |
| float progress = heapPlugin.getProgress(); |
| assertEquals("Invalid progress of disabled cumulative heap usage emulation " |
| + "plugin!", 1.0f, progress, 0f); |
| |
| // test with wrong/invalid configuration |
| Boolean failed = null; |
| invalidUsage = |
| TestResourceUsageEmulators.createMetrics(maxHeapUsage |
| + TotalHeapUsageEmulatorPlugin.ONE_MB); |
| try { |
| heapPlugin.initialize(conf, invalidUsage, monitor, null); |
| failed = false; |
| } catch (Exception e) { |
| failed = true; |
| } |
| assertNotNull("Fail case failure!", failed); |
| assertTrue("Expected failure!", failed); |
| |
| // test with valid resource usage value |
| ResourceUsageMetrics metrics = |
| TestResourceUsageEmulators.createMetrics(targetHeapUsageInMB |
| * TotalHeapUsageEmulatorPlugin.ONE_MB); |
| |
| // test with default emulation interval |
| // in every interval, the emulator will add 100% of the expected usage |
| // (since gridmix.emulators.resource-usage.heap.load-ratio=1) |
| // so at 10%, emulator will add 10% (difference), at 20% it will add 10% ... |
| // So to emulate 200MB, it will add |
| // 20mb + 20mb + 20mb + 20mb + .. = 200mb |
| testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, |
| 10); |
| |
| // test with custom value for emulation interval of 20% |
| conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL, |
| 0.2F); |
| // 40mb + 40mb + 40mb + 40mb + 40mb = 200mb |
| testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, 5); |
| |
| // test with custom value of free heap ratio and load ratio = 1 |
| conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F); |
| conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0.5F); |
| // 40mb + 0mb + 80mb + 0mb + 0mb = 120mb |
| testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 120, 2); |
| |
| // test with custom value of heap load ratio and min free heap ratio = 0 |
| conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 0.5F); |
| conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F); |
| // 20mb (call#1) + 20mb (call#1) + 20mb (call#2) + 20mb (call#2) +.. = 200mb |
| testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, |
| 10); |
| |
| // test with custom value of free heap ratio = 0.3 and load ratio = 0.5 |
| conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0.25F); |
| conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 0.5F); |
| // 20mb (call#1) + 20mb (call#1) + 30mb (call#2) + 0mb (call#2) |
| // + 30mb (call#3) + 0mb (call#3) + 35mb (call#4) + 0mb (call#4) |
| // + 37mb (call#5) + 0mb (call#5) = 162mb |
| testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 162, 6); |
| |
| // test if emulation interval boundary is respected |
| fakeProgress = new FakeProgressive(); // initialize |
| conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F); |
| conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F); |
| conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL, |
| 0.25F); |
| heapPlugin.initialize(conf, metrics, monitor, fakeProgress); |
| fakeCore.resetFake(); |
| // take a snapshot after the initialization |
| long initHeapUsage = fakeCore.getHeapUsageInMB(); |
| long initNumCallsUsage = fakeCore.getNumCalls(); |
| // test with 0 progress |
| testEmulationBoundary(0F, fakeCore, fakeProgress, heapPlugin, initHeapUsage, |
| initNumCallsUsage, "[no-op, 0 progress]"); |
| // test with 24% progress |
| testEmulationBoundary(0.24F, fakeCore, fakeProgress, heapPlugin, |
| initHeapUsage, initNumCallsUsage, |
| "[no-op, 24% progress]"); |
| // test with 25% progress |
| testEmulationBoundary(0.25F, fakeCore, fakeProgress, heapPlugin, |
| targetHeapUsageInMB / 4, 1, "[op, 25% progress]"); |
| // test with 80% progress |
| testEmulationBoundary(0.80F, fakeCore, fakeProgress, heapPlugin, |
| (targetHeapUsageInMB * 4) / 5, 2, "[op, 80% progress]"); |
| |
| // now test if the final call with 100% progress ramps up the heap usage |
| testEmulationBoundary(1F, fakeCore, fakeProgress, heapPlugin, |
| targetHeapUsageInMB, 3, "[op, 100% progress]"); |
| } |
| |
| // test whether the heap usage emulator achieves the desired target using |
| // desired calls to the underling core engine. |
| private static void testEmulationAccuracy(Configuration conf, |
| FakeHeapUsageEmulatorCore fakeCore, |
| ResourceCalculatorPlugin monitor, |
| ResourceUsageMetrics metrics, |
| TotalHeapUsageEmulatorPlugin heapPlugin, |
| long expectedTotalHeapUsageInMB, |
| long expectedTotalNumCalls) |
| throws Exception { |
| FakeProgressive fakeProgress = new FakeProgressive(); |
| fakeCore.resetFake(); |
| heapPlugin.initialize(conf, metrics, monitor, fakeProgress); |
| int numLoops = 0; |
| while (fakeProgress.getProgress() < 1) { |
| ++numLoops; |
| float progress = numLoops / 100.0F; |
| fakeProgress.setProgress(progress); |
| heapPlugin.emulate(); |
| } |
| |
| // test if the resource plugin shows the expected usage |
| assertEquals("Cumulative heap usage emulator plugin failed (total usage)!", |
| expectedTotalHeapUsageInMB, fakeCore.getHeapUsageInMB(), 1L); |
| // test if the resource plugin shows the expected num calls |
| assertEquals("Cumulative heap usage emulator plugin failed (num calls)!", |
| expectedTotalNumCalls, fakeCore.getNumCalls(), 0L); |
| } |
| |
| // tests if the heap usage emulation plugin emulates only at the expected |
| // progress gaps |
| private static void testEmulationBoundary(float progress, |
| FakeHeapUsageEmulatorCore fakeCore, FakeProgressive fakeProgress, |
| TotalHeapUsageEmulatorPlugin heapPlugin, long expectedTotalHeapUsageInMB, |
| long expectedTotalNumCalls, String info) throws Exception { |
| fakeProgress.setProgress(progress); |
| heapPlugin.emulate(); |
| // test heap usage |
| assertEquals("Emulation interval test for heap usage failed " + info + "!", |
| expectedTotalHeapUsageInMB, fakeCore.getHeapUsageInMB(), 0L); |
| // test num calls |
| assertEquals("Emulation interval test for heap usage failed " + info + "!", |
| expectedTotalNumCalls, fakeCore.getNumCalls(), 0L); |
| } |
| |
| /** |
| * Test the specified task java heap options. |
| */ |
| @SuppressWarnings("deprecation") |
| private void testJavaHeapOptions(String mapOptions, |
| String reduceOptions, String taskOptions, String defaultMapOptions, |
| String defaultReduceOptions, String defaultTaskOptions, |
| String expectedMapOptions, String expectedReduceOptions, |
| String expectedTaskOptions) throws Exception { |
| Configuration simulatedConf = new Configuration(); |
| // reset the configuration parameters |
| simulatedConf.unset(MRJobConfig.MAP_JAVA_OPTS); |
| simulatedConf.unset(MRJobConfig.REDUCE_JAVA_OPTS); |
| simulatedConf.unset(JobConf.MAPRED_TASK_JAVA_OPTS); |
| |
| // set the default map task options |
| if (defaultMapOptions != null) { |
| simulatedConf.set(MRJobConfig.MAP_JAVA_OPTS, defaultMapOptions); |
| } |
| // set the default reduce task options |
| if (defaultReduceOptions != null) { |
| simulatedConf.set(MRJobConfig.REDUCE_JAVA_OPTS, defaultReduceOptions); |
| } |
| // set the default task options |
| if (defaultTaskOptions != null) { |
| simulatedConf.set(JobConf.MAPRED_TASK_JAVA_OPTS, defaultTaskOptions); |
| } |
| |
| Configuration originalConf = new Configuration(); |
| // reset the configuration parameters |
| originalConf.unset(MRJobConfig.MAP_JAVA_OPTS); |
| originalConf.unset(MRJobConfig.REDUCE_JAVA_OPTS); |
| originalConf.unset(JobConf.MAPRED_TASK_JAVA_OPTS); |
| |
| // set the map task options |
| if (mapOptions != null) { |
| originalConf.set(MRJobConfig.MAP_JAVA_OPTS, mapOptions); |
| } |
| // set the reduce task options |
| if (reduceOptions != null) { |
| originalConf.set(MRJobConfig.REDUCE_JAVA_OPTS, reduceOptions); |
| } |
| // set the task options |
| if (taskOptions != null) { |
| originalConf.set(JobConf.MAPRED_TASK_JAVA_OPTS, taskOptions); |
| } |
| |
| // configure the task jvm's heap options |
| GridmixJob.configureTaskJVMOptions(originalConf, simulatedConf); |
| |
| assertEquals("Map heap options mismatch!", expectedMapOptions, |
| simulatedConf.get(MRJobConfig.MAP_JAVA_OPTS)); |
| assertEquals("Reduce heap options mismatch!", expectedReduceOptions, |
| simulatedConf.get(MRJobConfig.REDUCE_JAVA_OPTS)); |
| assertEquals("Task heap options mismatch!", expectedTaskOptions, |
| simulatedConf.get(JobConf.MAPRED_TASK_JAVA_OPTS)); |
| } |
| |
| /** |
| * Test task-level java heap options configuration in {@link GridmixJob}. |
| */ |
| @Test |
| public void testJavaHeapOptions() throws Exception { |
| // test missing opts |
| testJavaHeapOptions(null, null, null, null, null, null, null, null, |
| null); |
| |
| // test original heap opts and missing default opts |
| testJavaHeapOptions("-Xms10m", "-Xms20m", "-Xms30m", null, null, null, |
| null, null, null); |
| |
| // test missing opts with default opts |
| testJavaHeapOptions(null, null, null, "-Xms10m", "-Xms20m", "-Xms30m", |
| "-Xms10m", "-Xms20m", "-Xms30m"); |
| |
| // test empty option |
| testJavaHeapOptions("", "", "", null, null, null, null, null, null); |
| |
| // test empty default option and no original heap options |
| testJavaHeapOptions(null, null, null, "", "", "", "", "", ""); |
| |
| // test empty opts and default opts |
| testJavaHeapOptions("", "", "", "-Xmx10m -Xms1m", "-Xmx50m -Xms2m", |
| "-Xms2m -Xmx100m", "-Xmx10m -Xms1m", "-Xmx50m -Xms2m", |
| "-Xms2m -Xmx100m"); |
| |
| // test custom heap opts with no default opts |
| testJavaHeapOptions("-Xmx10m", "-Xmx20m", "-Xmx30m", null, null, null, |
| "-Xmx10m", "-Xmx20m", "-Xmx30m"); |
| |
| // test heap opts with default opts (multiple value) |
| testJavaHeapOptions("-Xms5m -Xmx200m", "-Xms15m -Xmx300m", |
| "-Xms25m -Xmx50m", "-XXabc", "-XXxyz", "-XXdef", |
| "-XXabc -Xmx200m", "-XXxyz -Xmx300m", "-XXdef -Xmx50m"); |
| |
| // test heap opts with default opts (duplication of -Xmx) |
| testJavaHeapOptions("-Xms5m -Xmx200m", "-Xms15m -Xmx300m", |
| "-Xms25m -Xmx50m", "-XXabc -Xmx500m", "-XXxyz -Xmx600m", |
| "-XXdef -Xmx700m", "-XXabc -Xmx200m", "-XXxyz -Xmx300m", |
| "-XXdef -Xmx50m"); |
| |
| // test heap opts with default opts (single value) |
| testJavaHeapOptions("-Xmx10m", "-Xmx20m", "-Xmx50m", "-Xms2m", |
| "-Xms3m", "-Xms5m", "-Xms2m -Xmx10m", "-Xms3m -Xmx20m", |
| "-Xms5m -Xmx50m"); |
| |
| // test heap opts with default opts (duplication of -Xmx) |
| testJavaHeapOptions("-Xmx10m", "-Xmx20m", "-Xmx50m", "-Xmx2m", |
| "-Xmx3m", "-Xmx5m", "-Xmx10m", "-Xmx20m", "-Xmx50m"); |
| } |
| |
| /** |
| * Test disabled task heap options configuration in {@link GridmixJob}. |
| */ |
| @Test |
| @SuppressWarnings("deprecation") |
| public void testJavaHeapOptionsDisabled() throws Exception { |
| Configuration gridmixConf = new Configuration(); |
| gridmixConf.setBoolean(GridmixJob.GRIDMIX_TASK_JVM_OPTIONS_ENABLE, false); |
| |
| // set the default values of simulated job |
| gridmixConf.set(MRJobConfig.MAP_JAVA_OPTS, "-Xmx1m"); |
| gridmixConf.set(MRJobConfig.REDUCE_JAVA_OPTS, "-Xmx2m"); |
| gridmixConf.set(JobConf.MAPRED_TASK_JAVA_OPTS, "-Xmx3m"); |
| |
| // set the default map and reduce task options for original job |
| final JobConf originalConf = new JobConf(); |
| originalConf.set(MRJobConfig.MAP_JAVA_OPTS, "-Xmx10m"); |
| originalConf.set(MRJobConfig.REDUCE_JAVA_OPTS, "-Xmx20m"); |
| originalConf.set(JobConf.MAPRED_TASK_JAVA_OPTS, "-Xmx30m"); |
| |
| // define a mock job |
| MockJob story = new MockJob(originalConf) { |
| public JobConf getJobConf() { |
| return originalConf; |
| } |
| }; |
| |
| GridmixJob job = new DummyGridmixJob(gridmixConf, story); |
| Job simulatedJob = job.getJob(); |
| Configuration simulatedConf = simulatedJob.getConfiguration(); |
| |
| assertEquals("Map heap options works when disabled!", "-Xmx1m", |
| simulatedConf.get(MRJobConfig.MAP_JAVA_OPTS)); |
| assertEquals("Reduce heap options works when disabled!", "-Xmx2m", |
| simulatedConf.get(MRJobConfig.REDUCE_JAVA_OPTS)); |
| assertEquals("Task heap options works when disabled!", "-Xmx3m", |
| simulatedConf.get(JobConf.MAPRED_TASK_JAVA_OPTS)); |
| } |
| } |