| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.mapred.gridmix.emulators.resourceusage; |
| |
| import java.io.IOException; |
| import java.util.Random; |
| |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.mapred.gridmix.Progressive; |
| import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; |
| import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; |
| |
| /** |
| * <p>A {@link ResourceUsageEmulatorPlugin} that emulates the cumulative CPU |
| * usage by performing certain CPU intensive operations. Performing such CPU |
| * intensive operations essentially uses up some CPU. Every |
| * {@link ResourceUsageEmulatorPlugin} is configured with a feedback module i.e |
| * a {@link ResourceCalculatorPlugin}, to monitor the resource usage.</p> |
| * |
| * <p>{@link CumulativeCpuUsageEmulatorPlugin} emulates the CPU usage in steps. |
| * The frequency of emulation can be configured via |
| * {@link #CPU_EMULATION_PROGRESS_INTERVAL}. |
| * CPU usage values are matched via emulation only on the interval boundaries. |
| * </p> |
| * |
| * {@link CumulativeCpuUsageEmulatorPlugin} is a wrapper program for managing |
| * the CPU usage emulation feature. It internally uses an emulation algorithm |
| * (called as core and described using {@link CpuUsageEmulatorCore}) for |
| * performing the actual emulation. Multiple calls to this core engine should |
| * use up some amount of CPU.<br> |
| * |
| * <p>{@link CumulativeCpuUsageEmulatorPlugin} provides a calibration feature |
| * via {@link #initialize(Configuration, ResourceUsageMetrics, |
| * ResourceCalculatorPlugin, Progressive)} to calibrate |
| * the plugin and its core for the underlying hardware. As a result of |
| * calibration, every call to the emulation engine's core should roughly use up |
| * 1% of the total usage value to be emulated. This makes sure that the |
| * underlying hardware is profiled before use and that the plugin doesn't |
| * accidently overuse the CPU. With 1% as the unit emulation target value for |
| * the core engine, there will be roughly 100 calls to the engine resulting in |
| * roughly 100 calls to the feedback (resource usage monitor) module. |
| * Excessive usage of the feedback module is discouraged as |
| * it might result into excess CPU usage resulting into no real CPU emulation. |
| * </p> |
| */ |
| public class CumulativeCpuUsageEmulatorPlugin |
| implements ResourceUsageEmulatorPlugin { |
| protected CpuUsageEmulatorCore emulatorCore; |
| private ResourceCalculatorPlugin monitor; |
| private Progressive progress; |
| private boolean enabled = true; |
| private float emulationInterval; // emulation interval |
| private long targetCpuUsage = 0; |
| private float lastSeenProgress = 0; |
| private long lastSeenCpuUsage = 0; |
| |
| // Configuration parameters |
| public static final String CPU_EMULATION_PROGRESS_INTERVAL = |
| "gridmix.emulators.resource-usage.cpu.emulation-interval"; |
| private static final float DEFAULT_EMULATION_FREQUENCY = 0.1F; // 10 times |
| |
| /** |
| * This is the core CPU usage emulation algorithm. This is the core engine |
| * which actually performs some CPU intensive operations to consume some |
| * amount of CPU. Multiple calls of {@link #compute()} should help the |
| * plugin emulate the desired level of CPU usage. This core engine can be |
| * calibrated using the {@link #calibrate(ResourceCalculatorPlugin, long)} |
| * API to suit the underlying hardware better. It also can be used to optimize |
| * the emulation cycle. |
| */ |
| public interface CpuUsageEmulatorCore { |
| /** |
| * Performs some computation to use up some CPU. |
| */ |
| public void compute(); |
| |
| /** |
| * Allows the core to calibrate itself. |
| */ |
| public void calibrate(ResourceCalculatorPlugin monitor, |
| long totalCpuUsage); |
| } |
| |
| /** |
| * This is the core engine to emulate the CPU usage. The only responsibility |
| * of this class is to perform certain math intensive operations to make sure |
| * that some desired value of CPU is used. |
| */ |
| public static class DefaultCpuUsageEmulator implements CpuUsageEmulatorCore { |
| // number of times to loop for performing the basic unit computation |
| private int numIterations; |
| private final Random random; |
| |
| /** |
| * This is to fool the JVM and make it think that we need the value |
| * stored in the unit computation i.e {@link #compute()}. This will prevent |
| * the JVM from optimizing the code. |
| */ |
| protected double returnValue; |
| |
| /** |
| * Initialized the {@link DefaultCpuUsageEmulator} with default values. |
| * Note that the {@link DefaultCpuUsageEmulator} should be calibrated |
| * (see {@link #calibrate(ResourceCalculatorPlugin, long)}) when initialized |
| * using this constructor. |
| */ |
| public DefaultCpuUsageEmulator() { |
| this(-1); |
| } |
| |
| DefaultCpuUsageEmulator(int numIterations) { |
| this.numIterations = numIterations; |
| random = new Random(); |
| } |
| |
| /** |
| * This will consume some desired level of CPU. This API will try to use up |
| * 'X' percent of the target cumulative CPU usage. Currently X is set to |
| * 10%. |
| */ |
| public void compute() { |
| for (int i = 0; i < numIterations; ++i) { |
| performUnitComputation(); |
| } |
| } |
| |
| // Perform unit computation. The complete CPU emulation will be based on |
| // multiple invocations to this unit computation module. |
| protected void performUnitComputation() { |
| //TODO can this be configurable too. Users/emulators should be able to |
| // pick and choose what MATH operations to run. |
| // Example : |
| // BASIC : ADD, SUB, MUL, DIV |
| // ADV : SQRT, SIN, COSIN.. |
| // COMPO : (BASIC/ADV)* |
| // Also define input generator. For now we can use the random number |
| // generator. Later this can be changed to accept multiple sources. |
| |
| int randomData = random.nextInt(); |
| int randomDataCube = randomData * randomData * randomData; |
| double randomDataCubeRoot = Math.cbrt(randomData); |
| returnValue = Math.log(Math.tan(randomDataCubeRoot |
| * Math.exp(randomDataCube)) |
| * Math.sqrt(randomData)); |
| } |
| |
| /** |
| * This will calibrate the algorithm such that a single invocation of |
| * {@link #compute()} emulates roughly 1% of the total desired resource |
| * usage value. |
| */ |
| public void calibrate(ResourceCalculatorPlugin monitor, |
| long totalCpuUsage) { |
| long initTime = monitor.getCumulativeCpuTime(); |
| |
| long defaultLoopSize = 0; |
| long finalTime = initTime; |
| |
| //TODO Make this configurable |
| while (finalTime - initTime < 100) { // 100 ms |
| ++defaultLoopSize; |
| performUnitComputation(); //perform unit computation |
| finalTime = monitor.getCumulativeCpuTime(); |
| } |
| |
| long referenceRuntime = finalTime - initTime; |
| |
| // time for one loop = (final-time - init-time) / total-loops |
| float timePerLoop = ((float)referenceRuntime) / defaultLoopSize; |
| |
| // compute the 1% of the total CPU usage desired |
| //TODO Make this configurable |
| long onePercent = totalCpuUsage / 100; |
| |
| // num-iterations for 1% = (total-desired-usage / 100) / time-for-one-loop |
| numIterations = Math.max(1, (int)((float)onePercent/timePerLoop)); |
| |
| System.out.println("Calibration done. Basic computation runtime : " |
| + timePerLoop + " milliseconds. Optimal number of iterations (1%): " |
| + numIterations); |
| } |
| } |
| |
| public CumulativeCpuUsageEmulatorPlugin() { |
| this(new DefaultCpuUsageEmulator()); |
| } |
| |
| /** |
| * For testing. |
| */ |
| public CumulativeCpuUsageEmulatorPlugin(CpuUsageEmulatorCore core) { |
| emulatorCore = core; |
| } |
| |
| // Note that this weighing function uses only the current progress. In future, |
| // this might depend on progress, emulation-interval and expected target. |
| private float getWeightForProgressInterval(float progress) { |
| // we want some kind of exponential growth function that gives less weight |
| // on lower progress boundaries but high (exact emulation) near progress |
| // value of 1. |
| // so here is how the current growth function looks like |
| // progress weight |
| // 0.1 0.0001 |
| // 0.2 0.0016 |
| // 0.3 0.0081 |
| // 0.4 0.0256 |
| // 0.5 0.0625 |
| // 0.6 0.1296 |
| // 0.7 0.2401 |
| // 0.8 0.4096 |
| // 0.9 0.6561 |
| // 1.0 1.000 |
| |
| return progress * progress * progress * progress; |
| } |
| |
| private synchronized long getCurrentCPUUsage() { |
| return monitor.getCumulativeCpuTime(); |
| } |
| |
| @Override |
| public float getProgress() { |
| return enabled |
| ? Math.min(1f, ((float)getCurrentCPUUsage())/targetCpuUsage) |
| : 1.0f; |
| } |
| |
| @Override |
| //TODO Multi-threading for speedup? |
| public void emulate() throws IOException, InterruptedException { |
| if (enabled) { |
| float currentProgress = progress.getProgress(); |
| if (lastSeenProgress < currentProgress |
| && ((currentProgress - lastSeenProgress) >= emulationInterval |
| || currentProgress == 1)) { |
| // Estimate the final cpu usage |
| // |
| // Consider the following |
| // Cl/Cc/Cp : Last/Current/Projected Cpu usage |
| // Pl/Pc/Pp : Last/Current/Projected progress |
| // Then |
| // (Cp-Cc)/(Pp-Pc) = (Cc-Cl)/(Pc-Pl) |
| // Solving this for Cp, we get |
| // Cp = Cc + (1-Pc)*(Cc-Cl)/Pc-Pl) |
| // Note that (Cc-Cl)/(Pc-Pl) is termed as 'rate' in the following |
| // section |
| |
| long currentCpuUsage = getCurrentCPUUsage(); |
| // estimate the cpu usage rate |
| float rate = (currentCpuUsage - lastSeenCpuUsage) |
| / (currentProgress - lastSeenProgress); |
| long projectedUsage = |
| currentCpuUsage + (long)((1 - currentProgress) * rate); |
| |
| if (projectedUsage < targetCpuUsage) { |
| // determine the correction factor between the current usage and the |
| // expected usage and add some weight to the target |
| long currentWeighedTarget = |
| (long)(targetCpuUsage |
| * getWeightForProgressInterval(currentProgress)); |
| |
| while (getCurrentCPUUsage() < currentWeighedTarget) { |
| emulatorCore.compute(); |
| // sleep for 100ms |
| try { |
| Thread.sleep(100); |
| } catch (InterruptedException ie) { |
| String message = |
| "CumulativeCpuUsageEmulatorPlugin got interrupted. Exiting."; |
| throw new RuntimeException(message); |
| } |
| } |
| } |
| |
| // set the last seen progress |
| lastSeenProgress = progress.getProgress(); |
| // set the last seen usage |
| lastSeenCpuUsage = getCurrentCPUUsage(); |
| } |
| } |
| } |
| |
| @Override |
| public void initialize(Configuration conf, ResourceUsageMetrics metrics, |
| ResourceCalculatorPlugin monitor, |
| Progressive progress) { |
| this.monitor = monitor; |
| this.progress = progress; |
| |
| // get the target CPU usage |
| targetCpuUsage = metrics.getCumulativeCpuUsage(); |
| if (targetCpuUsage <= 0 ) { |
| enabled = false; |
| return; |
| } else { |
| enabled = true; |
| } |
| |
| emulationInterval = conf.getFloat(CPU_EMULATION_PROGRESS_INTERVAL, |
| DEFAULT_EMULATION_FREQUENCY); |
| |
| // calibrate the core cpu-usage utility |
| emulatorCore.calibrate(monitor, targetCpuUsage); |
| |
| // initialize the states |
| lastSeenProgress = 0; |
| lastSeenCpuUsage = 0; |
| } |
| } |