blob: eb2d402cd25dfb035f584236443a255796f131de [file] [log] [blame]
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
import java.util.HashSet;
import java.util.Set;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Before;
import org.junit.Test;
import com.google.common.collect.Sets;
public class TestQueueManager {
private FairSchedulerConfiguration conf;
private QueueManager queueManager;
private Set<FSQueue> notEmptyQueues;
private FairScheduler scheduler;
@Before
public void setUp() throws Exception {
conf = new FairSchedulerConfiguration();
scheduler = mock(FairScheduler.class);
AllocationConfiguration allocConf = new AllocationConfiguration(conf);
// Set up some queues to test default child max resource inheritance
allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test");
allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.test.childA");
allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test.childB");
when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
when(scheduler.getConf()).thenReturn(conf);
when(scheduler.getResourceCalculator()).thenReturn(
new DefaultResourceCalculator());
SystemClock clock = SystemClock.getInstance();
when(scheduler.getClock()).thenReturn(clock);
notEmptyQueues = new HashSet<>();
queueManager = new QueueManager(scheduler) {
@Override
public boolean isEmpty(FSQueue queue) {
return !notEmptyQueues.contains(queue);
}
};
FSQueueMetrics.forQueue("root", null, true, conf);
queueManager.initialize(conf);
}
@Test
public void testReloadTurnsLeafQueueIntoParent() throws Exception {
updateConfiguredLeafQueues(queueManager, "queue1");
// When no apps are running in the leaf queue, should be fine turning it
// into a parent.
updateConfiguredLeafQueues(queueManager, "queue1.queue2");
assertNull(queueManager.getLeafQueue("queue1", false));
assertNotNull(queueManager.getLeafQueue("queue1.queue2", false));
// When leaf queues are empty, should be ok deleting them and
// turning parent into a leaf.
updateConfiguredLeafQueues(queueManager, "queue1");
assertNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNotNull(queueManager.getLeafQueue("queue1", false));
// When apps exist in leaf queue, we shouldn't be able to create
// children under it, but things should work otherwise.
notEmptyQueues.add(queueManager.getLeafQueue("queue1", false));
updateConfiguredLeafQueues(queueManager, "queue1.queue2");
assertNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNotNull(queueManager.getLeafQueue("queue1", false));
// When apps exist in leaf queues under a parent queue, shouldn't be
// able to turn it into a leaf queue, but things should work otherwise.
notEmptyQueues.clear();
updateConfiguredLeafQueues(queueManager, "queue1.queue2");
notEmptyQueues.add(queueManager.getQueue("root.queue1"));
updateConfiguredLeafQueues(queueManager, "queue1");
assertNotNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNull(queueManager.getLeafQueue("queue1", false));
// Should never to be able to create a queue under the default queue
updateConfiguredLeafQueues(queueManager, "default.queue3");
assertNull(queueManager.getLeafQueue("default.queue3", false));
assertNotNull(queueManager.getLeafQueue("default", false));
}
@Test
public void testReloadTurnsLeafToParentWithNoLeaf() {
AllocationConfiguration allocConf = new AllocationConfiguration(conf);
// Create a leaf queue1
allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.queue1");
queueManager.updateAllocationConfiguration(allocConf);
assertNotNull(queueManager.getLeafQueue("root.queue1", false));
// Lets say later on admin makes queue1 a parent queue by
// specifying "type=parent" in the alloc xml and lets say apps running in
// queue1
notEmptyQueues.add(queueManager.getLeafQueue("root.queue1", false));
allocConf = new AllocationConfiguration(conf);
allocConf.configuredQueues.get(FSQueueType.PARENT)
.add("root.queue1");
// When allocs are reloaded queue1 shouldn't be converter to parent
queueManager.updateAllocationConfiguration(allocConf);
assertNotNull(queueManager.getLeafQueue("root.queue1", false));
assertNull(queueManager.getParentQueue("root.queue1", false));
// Now lets assume apps completed and there are no apps in queue1
notEmptyQueues.clear();
// We should see queue1 transform from leaf queue to parent queue.
queueManager.updateAllocationConfiguration(allocConf);
assertNull(queueManager.getLeafQueue("root.queue1", false));
assertNotNull(queueManager.getParentQueue("root.queue1", false));
// this parent should not have any children
assertTrue(queueManager.getParentQueue("root.queue1", false)
.getChildQueues().isEmpty());
}
@Test
public void testCheckQueueNodeName() {
assertFalse(queueManager.isQueueNameValid(""));
assertFalse(queueManager.isQueueNameValid(" "));
assertFalse(queueManager.isQueueNameValid(" a"));
assertFalse(queueManager.isQueueNameValid("a "));
assertFalse(queueManager.isQueueNameValid(" a "));
assertFalse(queueManager.isQueueNameValid("\u00a0"));
assertFalse(queueManager.isQueueNameValid("a\u00a0"));
assertFalse(queueManager.isQueueNameValid("\u00a0a\u00a0"));
assertTrue(queueManager.isQueueNameValid("a b"));
assertTrue(queueManager.isQueueNameValid("a"));
}
private void updateConfiguredLeafQueues(QueueManager queueMgr, String... confLeafQueues) {
AllocationConfiguration allocConf = new AllocationConfiguration(conf);
allocConf.configuredQueues.get(FSQueueType.LEAF).addAll(Sets.newHashSet(confLeafQueues));
queueMgr.updateAllocationConfiguration(allocConf);
}
/**
* Test simple leaf queue creation.
*/
@Test
public void testCreateLeafQueue() {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
queueManager.updateAllocationConfiguration(allocConf);
FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.LEAF);
assertNotNull("Leaf queue root.queue1 was not created",
queueManager.getLeafQueue("root.queue1", false));
assertEquals("createQueue() returned wrong queue",
"root.queue1", q1.getName());
}
/**
* Test creation of a leaf queue and its parent.
*/
@Test
public void testCreateLeafQueueAndParent() {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
queueManager.updateAllocationConfiguration(allocConf);
FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
FSQueueType.LEAF);
assertNotNull("Parent queue root.queue1 was not created",
queueManager.getParentQueue("root.queue1", false));
assertNotNull("Leaf queue root.queue1.queue2 was not created",
queueManager.getLeafQueue("root.queue1.queue2", false));
assertEquals("createQueue() returned wrong queue",
"root.queue1.queue2", q2.getName());
}
/**
* Test creation of leaf and parent child queues when the parent queue has
* child defaults set. In this test we rely on the root.test,
* root.test.childA and root.test.childB queues that are created in the
* {@link #setUp()} method.
*/
@Test
public void testCreateQueueWithChildDefaults() {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
queueManager.updateAllocationConfiguration(allocConf);
queueManager.getQueue("root.test").setMaxChildQueueResource(
new ConfigurableResource(Resources.createResource(8192, 256)));
FSQueue q1 = queueManager.createQueue("root.test.childC", FSQueueType.LEAF);
assertNotNull("Leaf queue root.test.childC was not created",
queueManager.getLeafQueue("root.test.childC", false));
assertEquals("createQueue() returned wrong queue",
"root.test.childC", q1.getName());
assertEquals("Max resources for root.queue1 were not inherited from "
+ "parent's max child resources", Resources.createResource(8192, 256),
q1.getMaxShare());
FSQueue q2 = queueManager.createQueue("root.test.childD",
FSQueueType.PARENT);
assertNotNull("Leaf queue root.test.childD was not created",
queueManager.getParentQueue("root.test.childD", false));
assertEquals("createQueue() returned wrong queue",
"root.test.childD", q2.getName());
assertEquals("Max resources for root.test.childD were not inherited "
+ "from parent's max child resources",
Resources.createResource(8192, 256),
q2.getMaxShare());
// Check that the childA and childB queues weren't impacted
// by the child defaults
assertNotNull("Leaf queue root.test.childA was not created during setup",
queueManager.getLeafQueue("root.test.childA", false));
assertEquals("Max resources for root.test.childA were inherited from "
+ "parent's max child resources", Resources.unbounded(),
queueManager.getLeafQueue("root.test.childA", false).getMaxShare());
assertNotNull("Leaf queue root.test.childB was not created during setup",
queueManager.getParentQueue("root.test.childB", false));
assertEquals("Max resources for root.test.childB were inherited from "
+ "parent's max child resources", Resources.unbounded(),
queueManager.getParentQueue("root.test.childB", false).getMaxShare());
}
/**
* Test creation of a leaf queue with no resource limits.
*/
@Test
public void testCreateLeafQueueWithDefaults() {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.LEAF);
assertNotNull("Leaf queue root.queue1 was not created",
queueManager.getLeafQueue("root.queue1", false));
assertEquals("createQueue() returned wrong queue",
"root.queue1", q1.getName());
// Min default is 0,0
assertEquals("Min resources were not set to default",
Resources.createResource(0, 0),
q1.getMinShare());
// Max default is unbounded
assertEquals("Max resources were not set to default", Resources.unbounded(),
q1.getMaxShare());
}
/**
* Test creation of a simple parent queue.
*/
@Test
public void testCreateParentQueue() {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
queueManager.updateAllocationConfiguration(allocConf);
FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.PARENT);
assertNotNull("Parent queue root.queue1 was not created",
queueManager.getParentQueue("root.queue1", false));
assertEquals("createQueue() returned wrong queue",
"root.queue1", q1.getName());
}
/**
* Test creation of a parent queue and its parent.
*/
@Test
public void testCreateParentQueueAndParent() {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
queueManager.updateAllocationConfiguration(allocConf);
FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
FSQueueType.PARENT);
assertNotNull("Parent queue root.queue1 was not created",
queueManager.getParentQueue("root.queue1", false));
assertNotNull("Leaf queue root.queue1.queue2 was not created",
queueManager.getParentQueue("root.queue1.queue2", false));
assertEquals("createQueue() returned wrong queue",
"root.queue1.queue2", q2.getName());
}
}