YARN-10655. Limit queue creation depth relative to its first static parent. Contributed by Andras Gyori.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java
index e847737..898b075 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java
@@ -55,34 +55,43 @@
List<ApplicationPlacementContext> parentsToCreate = new ArrayList<>();
ApplicationPlacementContext queueCandidateContext = parentContext;
- CSQueue existingQueueCandidate = getQueue(
+ CSQueue firstExistingQueue = getQueue(
queueCandidateContext.getFullQueuePath());
- while (existingQueueCandidate == null) {
+ while (firstExistingQueue == null) {
parentsToCreate.add(queueCandidateContext);
queueCandidateContext = CSQueueUtils.extractQueuePath(
queueCandidateContext.getParentQueue());
- existingQueueCandidate = getQueue(
+ firstExistingQueue = getQueue(
queueCandidateContext.getFullQueuePath());
}
+ CSQueue firstExistingStaticQueue = firstExistingQueue;
+ // Include the LeafQueue in the distance
+ int firstStaticParentDistance = parentsToCreate.size() + 1;
+
+ while(isNonStaticParent(firstExistingStaticQueue)) {
+ queueCandidateContext = CSQueueUtils.extractQueuePath(
+ queueCandidateContext.getParentQueue());
+ firstExistingStaticQueue = getQueue(
+ queueCandidateContext.getFullQueuePath());
+ ++firstStaticParentDistance;
+ }
+
// Reverse the collection to to represent the hierarchy to be created
// from highest to lowest level
Collections.reverse(parentsToCreate);
- if (!(existingQueueCandidate instanceof ParentQueue)) {
+ if (!(firstExistingQueue instanceof ParentQueue)) {
throw new SchedulerDynamicEditException(
"Could not auto create hierarchy of "
+ queue.getFullQueuePath() + ". Queue "
- + existingQueueCandidate.getQueuePath() +
+ + firstExistingQueue.getQueuePath() +
" is not a ParentQueue."
);
}
- ParentQueue existingParentQueue = (ParentQueue) existingQueueCandidate;
+ ParentQueue existingParentQueue = (ParentQueue) firstExistingQueue;
int depthLimit = extractDepthLimit(existingParentQueue);
- // The number of levels to be created including the LeafQueue
- // (which is last)
- int levelsToCreate = parentsToCreate.size() + 1;
if (depthLimit == 0) {
throw new SchedulerDynamicEditException("Auto creation of queue " +
@@ -90,12 +99,12 @@
+ existingParentQueue.getQueuePath());
}
- if (levelsToCreate > depthLimit) {
+ if (firstStaticParentDistance > depthLimit) {
throw new SchedulerDynamicEditException(
"Could not auto create queue " + queue.getFullQueuePath()
- + ". In order to create the desired queue hierarchy, " +
- levelsToCreate + " levels of queues would need " +
- "to be created, which is above the limit.");
+ + ". The distance of the LeafQueue from the first static " +
+ "ParentQueue is" + firstStaticParentDistance + ", which is " +
+ "above the limit.");
}
for (ApplicationPlacementContext current : parentsToCreate) {
@@ -123,4 +132,9 @@
private CSQueue getQueue(String queue) {
return queue != null ? queueManager.getQueue(queue) : null;
}
+
+ private boolean isNonStaticParent(CSQueue queue) {
+ return (!(queue instanceof AbstractCSQueue)
+ || ((AbstractCSQueue) queue).isDynamicQueue());
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index 4facf94..2f83f1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -565,6 +565,26 @@
}
}
+ @Test
+ public void testAutoQueueCreationDepthLimitFromStaticParent()
+ throws Exception {
+ startScheduler();
+
+ // a is the first existing queue here and it is static, therefore
+ // the distance is 2
+ createQueue("root.a.a-auto.a1-auto");
+ Assert.assertNotNull(cs.getQueue("root.a.a-auto.a1-auto"));
+
+ try {
+ createQueue("root.a.a-auto.a2-auto.a3-auto");
+ Assert.fail("Queue creation should not succeed because the distance " +
+ "from the first static parent is above limit");
+ } catch (SchedulerDynamicEditException ignored) {
+
+ }
+
+ }
+
private LeafQueue createQueue(String queuePath) throws YarnException {
return autoQueueHandler.autoCreateQueue(
CSQueueUtils.extractQueuePath(queuePath));
