SLIDER-1201 Slider should make resource normalization configurable for app-owners
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index 92890be..d6d8789 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -72,6 +72,14 @@
String YARN_CORES = "yarn.vcores";
/**
+ * If normalization is set to false, then if the resource (memory and/or
+ * vcore) requested by a role is higher than YARN limits, then the resource
+ * request is not normalized. If this causes failures at the YARN level then
+ * applications are expecting that to happen. Default value is false.
+ */
+ String YARN_RESOURCE_NORMALIZATION_ENABLED = "yarn.resource.normalization.enabled";
+
+ /**
* Number of disks per instance to ask YARN for
* {@value}
*/
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index e9f65ba..9638408 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -513,6 +513,26 @@
}
/**
+ * Extract the first line of a multi-line string. This is typically used to
+ * prune the stack trace appended to the end of exception messages returned by
+ * YARN in AMRMClientAsync callbacks.
+ *
+ * @param msg
+ * message string (most likely multi-lines)
+ * @return the first line of a multi-line string or the original string if it
+ * is a null, empty or single-line
+ */
+ public static String extractFirstLine(String msg) {
+ if (StringUtils.isNotBlank(msg)) {
+ int newlineIndex = msg.indexOf(System.lineSeparator());
+ if (newlineIndex != -1) {
+ msg = msg.substring(0, newlineIndex);
+ }
+ }
+ return msg;
+ }
+
+ /**
* Create a configuration with Slider-specific tuning.
* This is done rather than doing custom configs.
* @return the config
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 8232225..1337ec5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -27,6 +27,7 @@
import com.google.protobuf.BlockingService;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
@@ -2060,14 +2061,13 @@
@Override //AMRMClientAsync
public void onError(Throwable e) {
- //callback says it's time to finish
+ // callback says it's time to finish
LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e);
- signalAMComplete(new ActionStopSlider("stop",
- EXIT_EXCEPTION_THROWN,
+ signalAMComplete(new ActionStopSlider("stop", EXIT_EXCEPTION_THROWN,
FinalApplicationStatus.FAILED,
- "AMRMClientAsync.onError() received " + e));
+ SliderUtils.extractFirstLine(e.getLocalizedMessage())));
}
-
+
/* =================================================================== */
/* RMOperationHandlerActions */
/* =================================================================== */
@@ -2356,13 +2356,13 @@
/**
* Handle any exception in a thread. If the exception provides an exit
- * code, that is the one that will be used
+ * code, that is the one that will be used.
* @param thread thread throwing the exception
* @param exception exception
*/
public void onExceptionInThread(Thread thread, Throwable exception) {
log.error("Exception in {}: {}", thread.getName(), exception, exception);
-
+
// if there is a teardown in progress, ignore it
if (amCompletionFlag.get()) {
log.info("Ignoring exception: shutdown in progress");
@@ -2371,10 +2371,9 @@
if (exception instanceof ExitCodeProvider) {
exitCode = ((ExitCodeProvider) exception).getExitCode();
}
- signalAMComplete(new ActionStopSlider("stop",
- exitCode,
- FinalApplicationStatus.FAILED,
- exception.toString()));
+ signalAMComplete(
+ new ActionStopSlider("stop", exitCode, FinalApplicationStatus.FAILED,
+ SliderUtils.extractFirstLine(exception.getLocalizedMessage())));
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index c5fd38c..3db9388 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1385,9 +1385,10 @@
}
/**
- * Build up the resource requirements for this role from the
- * cluster specification, including substituing max allowed values
- * if the specification asked for it.
+ * Build up the resource requirements for this role from the cluster
+ * specification, including substituting max allowed values if the
+ * specification asked for it (except when
+ * {@link ResourceKeys#YARN_RESOURCE_NORMALIZATION_ENABLED} is set to false).
* @param role role
* @param capability capability to set up. A new one may be created
* during normalization
@@ -1409,6 +1410,12 @@
containerMaxMemory);
capability.setMemory(ram);
log.debug("Component {} has RAM={}, vCores ={}", name, ram, cores);
+ boolean normalize = resources.getComponentOptBool(group,
+ YARN_RESOURCE_NORMALIZATION_ENABLED, true);
+ if (!normalize) {
+ log.info("Resource normalization: disabled");
+ return Resources.createResource(ram, cores);
+ }
Resource normalized = recordFactory.normalize(capability, minResource,
maxResource);
if (!Resources.equals(normalized, capability)) {
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 0969824..7be01ad 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -18,10 +18,14 @@
package org.apache.slider.server.appmaster.model.history
+import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.hadoop.yarn.util.resource.Resources
+import org.apache.slider.api.ResourceKeys
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockPriority
import org.apache.slider.server.appmaster.model.mock.MockResource
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
@@ -36,6 +40,7 @@
import org.apache.slider.server.appmaster.state.RoleStatus
import org.junit.Test
+@Slf4j
class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
public static final String WORKERS_LABEL = "workers"
@@ -301,6 +306,53 @@
assert yarnRequest.nodes.size() == 2
}
+ @Test
+ public void testBuildResourceRequirements() throws Throwable {
+ // Store original values
+ def resources = appState.getResourcesSnapshot()
+ def origMem = resources.getComponentOpt(role0Status.group,
+ ResourceKeys.YARN_MEMORY, null)
+ def origVcores = resources.getComponentOpt(role0Status.group,
+ ResourceKeys.YARN_CORES, null)
+
+ // Resource values to be used for this test
+ def testMem = 32768
+ def testVcores = 2
+ resources.setComponentOpt(role0Status.group, ResourceKeys.YARN_MEMORY,
+ Integer.toString(testMem));
+ resources.setComponentOpt(role0Status.group, ResourceKeys.YARN_CORES,
+ Integer.toString(testVcores));
+
+ // Test normalization disabled
+ log.info("Test normalization: disabled")
+ resources.setComponentOpt(role0Status.group,
+ ResourceKeys.YARN_RESOURCE_NORMALIZATION_ENABLED, "false");
+ def requestedRes = new MockResource(testMem, testVcores)
+ def expectedRes = new MockResource(testMem, testVcores)
+ log.info("Resource requested: " + requestedRes)
+ def resFinal = appState.buildResourceRequirements(role0Status,
+ new MockResource())
+ log.info("Resource actual: " + resFinal)
+ assert Resources.equals(expectedRes, resFinal)
+
+ // Test normalization enabled
+ log.info("Test normalization: enabled")
+ resources.setComponentOpt(role0Status.group,
+ ResourceKeys.YARN_RESOURCE_NORMALIZATION_ENABLED, "true");
+ expectedRes = new MockResource(MockAppState.RM_MAX_RAM, testVcores)
+ log.info("Resource requested: " + requestedRes)
+ resFinal = appState.buildResourceRequirements(role0Status,
+ new MockResource())
+ log.info("Resource actual: " + resFinal)
+ assert Resources.equals(expectedRes, resFinal)
+
+ // revert resource configuration to original value
+ resources.setComponentOpt(role0Status.group, ResourceKeys.YARN_MEMORY,
+ origMem);
+ resources.setComponentOpt(role0Status.group, ResourceKeys.YARN_CORES,
+ origVcores);
+ }
+
/**
* Create a new request (always against host1)
* @param r role status
diff --git a/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java b/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
index 20e72c0..deca7a8 100644
--- a/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
+++ b/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
@@ -156,4 +156,21 @@
SliderUtils.write(testWriteFile, "test".getBytes("UTF-8"), true);
Assert.assertTrue(FileUtils.readFileToString(testWriteFile, "UTF-8").equals("test"));
}
+
+ @Test
+ public void testExtractFirstLine() {
+ String firstLine = "hello";
+ String msg = firstLine + "\n2nd line\n3rd line";
+ Assert.assertEquals("Should be first line only", firstLine,
+ SliderUtils.extractFirstLine(msg));
+ msg = "";
+ Assert.assertEquals("Should be empty", msg,
+ SliderUtils.extractFirstLine(msg));
+ msg = " ";
+ Assert.assertEquals("Should contain spaces only", msg,
+ SliderUtils.extractFirstLine(msg));
+ msg = null;
+ Assert.assertEquals("Should be null", msg,
+ SliderUtils.extractFirstLine(msg));
+ }
}