SLIDER-587: building test for dynamic role placement history
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index a69a60d..406086a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -574,20 +574,21 @@
* @return a new provider role
* @throws BadConfigException bad configuration
*/
+ @VisibleForTesting
public ProviderRole createDynamicProviderRole(String name,
MapOperations component) throws
BadConfigException {
String priOpt = component.getMandatoryOption(ResourceKeys.COMPONENT_PRIORITY);
- int pri = SliderUtils.parseAndValidate("value of " + name + " " +
+ int priority = SliderUtils.parseAndValidate("value of " + name + " " +
ResourceKeys.COMPONENT_PRIORITY,
priOpt, 0, 1, -1);
- log.info("Role {} assigned priority {}", name, pri);
+ log.info("Role {} assigned priority {}", name, priority);
String placementOpt = component.getOption(
ResourceKeys.COMPONENT_PLACEMENT_POLICY, "0");
int placement = SliderUtils.parseAndValidate("value of " + name + " " +
ResourceKeys.COMPONENT_PLACEMENT_POLICY,
placementOpt, 0, 0, -1);
- return new ProviderRole(name, pri, placement);
+ return new ProviderRole(name, priority, placement);
}
@@ -694,11 +695,13 @@
for (String name : roleNames) {
if (!roles.containsKey(name)) {
// this is a new value
+ MapOperations component = resources.getComponent(name);
log.info("Adding new role {}", name);
ProviderRole dynamicRole = createDynamicProviderRole(name,
- resources.getComponent(name));
+ component);
RoleStatus roleStatus = buildRole(dynamicRole);
roleStatus.setDesired(getDesiredInstanceCount(resources, name));
+ log.info("New role {}", roleStatus);
roleHistory.addNewProviderRole(dynamicRole);
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 2b35ad5..2b0ee18 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -430,9 +430,9 @@
/**
* Get the nodes for an ID -may be null
* @param id role ID
- * @return list
+ * @return potenially null list
*/
- private LinkedList<NodeInstance> getNodesForRoleId(int id) {
+ private List<NodeInstance> getNodesForRoleId(int id) {
return availableNodes.get(id);
}
@@ -481,7 +481,7 @@
List<NodeInstance> targets = getNodesForRoleId(roleKey);
int cnt = targets == null ? 0 : targets.size();
- log.debug("There're {} nodes to consider for {}", cnt, role.getName());
+ log.debug("There are {} node(s) to consider for {}", cnt, role.getName());
while (targets != null && !targets.isEmpty() && nodeInstance == null) {
NodeInstance head = targets.remove(0);
if (head.getActiveRoleInstances(roleKey) == 0) {
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index c6d8f4c..c8ddc6f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -51,8 +51,13 @@
* already been targeted for termination
*/
public boolean released;
+
+ /**
+ * Name of the role
+ */
public String role;
public int roleId;
+
/**
* state from {@link ClusterDescription}
*/
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
index 9902155..6368a3d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
@@ -29,7 +29,12 @@
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.state.RoleStatus
import org.junit.Test
/**
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
index 136e1ea..5ef639b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -22,12 +22,13 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.RoleInstance
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.junit.Test
@@ -58,7 +59,7 @@
@Override
void initApp() {
super.initApp()
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
def instance = factory.newInstanceDefinition(0,0,0)
@@ -90,5 +91,50 @@
appState.getRoleHistory().dump();
}
-
+
+
+ @Test
+ public void testDynamicRoleHistory() throws Throwable {
+
+ // snapshot and patch existing spec
+ def resources = ConfTreeOperations.fromInstance(
+ appState.resourcesSnapshot.confTree)
+
+ def name = "dynamic"
+ def dynamicComp = resources.getOrAddComponent(name)
+ int priority = 8
+ int placement = 3
+ dynamicComp.put(ResourceKeys.COMPONENT_PRIORITY, "8")
+ dynamicComp.put(ResourceKeys.COMPONENT_INSTANCES, "1")
+ dynamicComp.put(ResourceKeys.COMPONENT_PLACEMENT_POLICY, "3")
+
+ def component = resources.getComponent(name)
+ String priOpt = component.getMandatoryOption(
+ ResourceKeys.COMPONENT_PRIORITY);
+ int parsedPriority = SliderUtils.parseAndValidate(
+ "value of " + name + " " + ResourceKeys.COMPONENT_PRIORITY,
+ priOpt, 0, 1, -1);
+ assert priority == parsedPriority
+
+ def newRole = appState.createDynamicProviderRole(name, component)
+ assert newRole.id == priority
+
+ appState.updateResourceDefinitions(resources.confTree);
+ assert appState.roleMap[name] != null
+ def mappedRole = appState.roleMap[name]
+ assert mappedRole.id == priority
+
+ def priorityMap = appState.rolePriorityMap
+ assert priorityMap.size() == 4
+ def dynamicProviderRole
+ assert (dynamicProviderRole = priorityMap[priority]) != null
+ assert dynamicProviderRole.id == priority
+
+ // allocate the nodes
+ def allocations = createAndStartNodes()
+ assert allocations.size() == 1
+ RoleInstance ri = allocations[0]
+ assert ri.role == name
+ assert ri.roleId == priority
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
index 5c9dce9..8308a13 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -26,6 +26,7 @@
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
@@ -60,7 +61,7 @@
@Override
void initApp() {
super.initApp()
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
def instance = factory.newInstanceDefinition(0, 0, 0)
@@ -174,7 +175,7 @@
//now reset the app state
def historyWorkDir2 = new File("target/history" + testName + "-0002")
def historyPath2 = new Path(historyWorkDir2.toURI())
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
appState.buildInstance(
factory.newInstanceDefinition(0, 0, 0),
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
index e5ad4ae..ee5eead 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
@@ -31,7 +31,8 @@
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.operations.RMOperationHandler
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.RoleInstance
import org.junit.Test
import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
index c2783f3..b48a683 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
@@ -24,10 +24,14 @@
import org.apache.hadoop.yarn.api.records.Container
import org.apache.slider.api.StatusKeys
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.NodeMap
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.junit.Test
/**
@@ -67,7 +71,7 @@
NodeMap nodemap = appState.roleHistory.cloneNodemap()
// now destroy the app state
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
//and rebuild
appState.buildInstance(
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
index 17ebc31..6df8910 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
@@ -27,7 +27,9 @@
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.RoleHistoryUtils
+import org.apache.slider.server.appmaster.state.RoleInstance
import org.junit.Test
import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 6c0f571..fa54256 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -43,7 +43,7 @@
public static final int RM_MAX_RAM = 4096
public static final int RM_MAX_CORES = 64
MockFactory factory = new MockFactory()
- AppState appState
+ MockAppState appState
MockYarnEngine engine
protected HadoopFS fs
protected SliderFileSystem sliderFileSystem
@@ -86,7 +86,7 @@
historyWorkDir = new File("target/history", historyDirName)
historyPath = new Path(historyWorkDir.toURI())
fs.delete(historyPath, true)
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
appState.buildInstance(
buildInstanceDefinition(),
@@ -172,7 +172,7 @@
*/
public ContainerStatus containerStatus(ContainerId cid) {
ContainerStatus status = containerStatus(cid,
- LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN)
+ LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN)
return status
}
@@ -203,6 +203,20 @@
List<AppState.NodeCompletionResult> completionResults) {
List<ContainerId> released = []
List<RoleInstance> instances = createAndSubmitNodes(released)
+ processSubmissionOperations(instances, completionResults, released)
+ return instances
+ }
+
+ /**
+ * Process the start/stop operations from
+ * @param instances
+ * @param completionResults
+ * @param released
+ */
+ public void processSubmissionOperations(
+ List<RoleInstance> instances,
+ List<AppState.NodeCompletionResult> completionResults,
+ List<ContainerId> released) {
for (RoleInstance instance : instances) {
assert appState.onNodeManagerContainerStarted(instance.containerId)
}
@@ -212,7 +226,6 @@
"released",
0
)
- return instances
}
/**
@@ -256,6 +269,19 @@
public List<RoleInstance> createAndSubmitNodes(
List<ContainerId> released) {
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ return submitOperations(ops, released)
+ }
+
+ /**
+ * Process the RM operations and send <code>onContainersAllocated</code>
+ * events to the app state
+ * @param ops
+ * @param released
+ * @return
+ */
+ public List<RoleInstance> submitOperations(
+ List<AbstractRMOperation> ops,
+ List<ContainerId> released) {
List<Container> allocatedContainers = engine.execute(ops, released)
List<ContainerAssignment> assignments = [];
List<AbstractRMOperation> operations = []
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
index 25c957e..ad85b89 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
@@ -17,13 +17,27 @@
package org.apache.slider.server.appmaster.model.mock
+import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.state.AbstractRecordFactory
import org.apache.slider.server.appmaster.state.AppState
+/**
+ * Extended app state that makes more things public
+ */
class MockAppState extends AppState {
public MockAppState(AbstractRecordFactory recordFactory) {
super(recordFactory);
}
+ /**
+ * Instance with a mock record factory
+ */
+ public MockAppState() {
+ super(new MockRecordFactory());
+ }
+
+ public Map<String, ProviderRole> getRoleMap() {
+ return super.roleMap;
+ }
}
