YARN-10168. FS-CS Converter: tool doesn't handle min/max resource conversion correctly. Contributed by Peter Bacsko
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java
index f4526cb..3a2d363 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java
@@ -55,6 +55,12 @@
public static final String MAX_CHILD_CAPACITY =
"maxChildCapacity.action";
+ public static final String MAX_RESOURCES =
+ "maxResources.action";
+
+ public static final String MIN_RESOURCES =
+ "minResources.action";
+
public static final String USER_MAX_RUNNING_APPS =
"userMaxRunningApps.action";
@@ -118,6 +124,8 @@
public void initPropertyActions() {
setActionForProperty(MAX_CAPACITY_PERCENTAGE);
setActionForProperty(MAX_CHILD_CAPACITY);
+ setActionForProperty(MAX_RESOURCES);
+ setActionForProperty(MIN_RESOURCES);
setActionForProperty(USER_MAX_RUNNING_APPS);
setActionForProperty(USER_MAX_APPS_DEFAULT);
setActionForProperty(DYNAMIC_MAX_ASSIGN);
@@ -138,6 +146,14 @@
handle(MAX_CHILD_CAPACITY, "<maxChildResources>", null);
}
+ public void handleMaxResources() {
+ handle(MAX_RESOURCES, "<maxResources>", null);
+ }
+
+ public void handleMinResources() {
+ handle(MIN_RESOURCES, "<minResources>", null);
+ }
+
public void handleChildQueueCount(String queue, int count) {
String value = properties.getProperty(MAX_CHILD_QUEUE_LIMIT);
if (value != null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java
index 2388970..8fd10f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java
@@ -27,7 +27,6 @@
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource;
@@ -36,7 +35,6 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
-import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
/**
@@ -51,6 +49,7 @@
private Configuration capacitySchedulerConfig;
private final boolean preemptionEnabled;
private final boolean sizeBasedWeight;
+ @SuppressWarnings("unused")
private final Resource clusterResource;
private final float queueMaxAMShareDefault;
private final boolean autoCreateChildQueues;
@@ -168,47 +167,13 @@
ConfigurableResource rawMaxShare = queue.getRawMaxShare();
final Resource maxResource = rawMaxShare.getResource();
- long memSize = 0;
- long vCores = 0;
- boolean defined = false;
-
- if (maxResource == null) {
- if (rawMaxShare.getPercentages() != null) {
- if (clusterResource == null) {
- String message = String.format(
- "<maxResources> defined in percentages for" +
- " queue %s, but cluster resource parameter is not" +
- " defined via CLI!", queueName);
-
- conversionOptions.handleConversionError(message);
- return;
- }
-
- ruleHandler.handleMaxCapacityPercentage(queueName);
-
- double[] percentages = rawMaxShare.getPercentages();
- int memIndex = ResourceUtils.getResourceTypeIndex().get("memory-mb");
- int vcoreIndex = ResourceUtils.getResourceTypeIndex().get("vcores");
-
- memSize = (long) (percentages[memIndex] *
- clusterResource.getMemorySize());
- vCores = (long) (percentages[vcoreIndex] *
- clusterResource.getVirtualCores());
- defined = true;
- } else {
- conversionOptions.handlePreconditionError(
- "Illegal ConfigurableResource object = " + rawMaxShare);
- }
- } else if (isNotUnboundedResource(maxResource)) {
- memSize = maxResource.getMemorySize();
- vCores = maxResource.getVirtualCores();
- defined = true;
+ if ((maxResource == null && rawMaxShare.getPercentages() != null)
+ || isNotUnboundedResource(maxResource)) {
+ ruleHandler.handleMaxResources();
}
- if (defined) {
- capacitySchedulerConfig.set(PREFIX + queueName + ".maximum-capacity",
- String.format("[memory=%d, vcores=%d]", memSize, vCores));
- }
+ capacitySchedulerConfig.set(PREFIX + queueName + ".maximum-capacity",
+ "100");
}
/**
@@ -330,7 +295,7 @@
List<FSQueue> children = queue.getChildQueues();
int totalWeight = getTotalWeight(children);
- Map<String, Capacity> capacities = getCapacities(totalWeight, children);
+ Map<String, BigDecimal> capacities = getCapacities(totalWeight, children);
capacities
.forEach((key, value) -> capacitySchedulerConfig.set(PREFIX + key +
".capacity", value.toString()));
@@ -353,23 +318,21 @@
}
}
- private Map<String, Capacity> getCapacities(int totalWeight,
+ private Map<String, BigDecimal> getCapacities(int totalWeight,
List<FSQueue> children) {
final BigDecimal hundred = new BigDecimal(100).setScale(3);
if (children.size() == 0) {
return new HashMap<>();
} else if (children.size() == 1) {
- Map<String, Capacity> capacity = new HashMap<>();
+ Map<String, BigDecimal> capacity = new HashMap<>();
String queueName = children.get(0).getName();
- capacity.put(queueName, Capacity.newCapacity(hundred));
+ capacity.put(queueName, hundred);
return capacity;
} else {
- Map<String, Capacity> capacities = new HashMap<>();
- Map<String, BigDecimal> bdCapacities = new HashMap<>();
+ Map<String, BigDecimal> capacities = new HashMap<>();
- MutableBoolean needVerifySum = new MutableBoolean(true);
children
.stream()
.forEach(queue -> {
@@ -381,48 +344,28 @@
.multiply(hundred)
.setScale(3);
- // <minResources> defined?
if (Resources.none().compareTo(queue.getMinShare()) != 0) {
- needVerifySum.setFalse();
-
- /* TODO: Needs discussion.
- *
- * Probably it's not entirely correct this way!
- * Eg. root.queue1 in FS translates to 33%
- * capacity, but minResources is defined as 1vcore,8GB
- * which is less than 33%.
- *
- * Therefore max(calculatedCapacity, minResource) is
- * more sound.
- */
- Resource minShare = queue.getMinShare();
- // TODO: in Phase-2, we have to deal with other resources as well
- String capacity = String.format("[memory=%d,vcores=%d]",
- minShare.getMemorySize(), minShare.getVirtualCores());
- capacities.put(queue.getName(), Capacity.newCapacity(capacity));
- } else {
- capacities.put(queue.getName(), Capacity.newCapacity(pct));
- bdCapacities.put(queue.getName(), pct);
+ ruleHandler.handleMinResources();
}
+
+ capacities.put(queue.getName(), pct);
});
- if (needVerifySum.isTrue()) {
- BigDecimal totalPct = new BigDecimal(0);
- for (Map.Entry<String, BigDecimal> entry : bdCapacities.entrySet()) {
- totalPct = totalPct.add(entry.getValue());
+ BigDecimal totalPct = new BigDecimal(0);
+ for (Map.Entry<String, BigDecimal> entry : capacities.entrySet()) {
+ totalPct = totalPct.add(entry.getValue());
+ }
+
+ // fix last value if total != 100.000
+ if (!totalPct.equals(hundred)) {
+ BigDecimal tmp = new BigDecimal(0);
+ for (int i = 0; i < children.size() - 2; i++) {
+ tmp = tmp.add(capacities.get(children.get(i).getQueueName()));
}
- // fix last value if total != 100.000
- if (!totalPct.equals(hundred)) {
- BigDecimal tmp = new BigDecimal(0);
- for (int i = 0; i < children.size() - 2; i++) {
- tmp = tmp.add(bdCapacities.get(children.get(i).getQueueName()));
- }
-
- String lastQueue = children.get(children.size() - 1).getName();
- BigDecimal corrected = hundred.subtract(tmp);
- capacities.put(lastQueue, Capacity.newCapacity(corrected));
- }
+ String lastQueue = children.get(children.size() - 1).getName();
+ BigDecimal corrected = hundred.subtract(tmp);
+ capacities.put(lastQueue, corrected);
}
return capacities;
@@ -445,38 +388,4 @@
private boolean isNotUnboundedResource(Resource res) {
return Resources.unbounded().compareTo(res) != 0;
}
-
- /*
- * Represents a queue capacity in either percentage
- * or in absolute resources
- */
- private static class Capacity {
- private BigDecimal percentage;
- private String absoluteResource;
-
- public static Capacity newCapacity(BigDecimal pct) {
- Capacity capacity = new Capacity();
- capacity.percentage = pct;
- capacity.absoluteResource = null;
-
- return capacity;
- }
-
- public static Capacity newCapacity(String absoluteResource) {
- Capacity capacity = new Capacity();
- capacity.percentage = null;
- capacity.absoluteResource = absoluteResource;
-
- return capacity;
- }
-
- @Override
- public String toString() {
- if (percentage != null) {
- return percentage.toString();
- } else {
- return absoluteResource;
- }
- }
- }
}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestConvertedConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestConvertedConfigValidator.java
index d6bc263..4feb4e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestConvertedConfigValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestConvertedConfigValidator.java
@@ -20,6 +20,8 @@
import java.io.File;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -36,9 +38,15 @@
@Before
public void setup() {
+ QueueMetrics.clearQueueMetrics();
validator = new ConvertedConfigValidator();
}
+ @After
+ public void after() {
+ QueueMetrics.clearQueueMetrics();
+ }
+
@Test
public void testValidationPassed() throws Exception {
validator.validateConvertedConfig(CONFIG_DIR_PASSES);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
index 5c04510..7b79225 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
@@ -301,19 +301,6 @@
}
@Test
- public void testConvertFSConfigPctModeUsedAndClusterResourceNotDefined()
- throws Exception {
- FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
- .build();
-
- expectedException.expect(ConversionException.class);
- expectedException.expectMessage("cluster resource parameter" +
- " is not defined via CLI");
-
- converter.convert(params);
- }
-
- @Test
public void testConvertFSConfigurationClusterResourceInvalid()
throws Exception {
FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java
index 4539e1b..b563e64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java
@@ -18,6 +18,8 @@
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.DYNAMIC_MAX_ASSIGN;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CAPACITY_PERCENTAGE;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_RESOURCES;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MIN_RESOURCES;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CHILD_CAPACITY;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CHILD_QUEUE_LIMIT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.QUEUE_AUTO_CREATE;
@@ -69,6 +71,8 @@
ruleHandler.handleDynamicMaxAssign();
ruleHandler.handleMaxCapacityPercentage("test");
ruleHandler.handleMaxChildCapacity();
+ ruleHandler.handleMinResources();
+ ruleHandler.handleMaxResources();
ruleHandler.handleQueueAutoCreate("test");
ruleHandler.handleReservationSystem();
ruleHandler.handleSpecifiedNotFirstRule();
@@ -81,6 +85,8 @@
Properties rules = new Properties();
rules.put(DYNAMIC_MAX_ASSIGN, WARNING);
rules.put(MAX_CAPACITY_PERCENTAGE, WARNING);
+ rules.put(MAX_RESOURCES, WARNING);
+ rules.put(MIN_RESOURCES, WARNING);
rules.put(MAX_CHILD_CAPACITY, WARNING);
rules.put(QUEUE_AUTO_CREATE, WARNING);
rules.put(RESERVATION_SYSTEM, WARNING);
@@ -95,6 +101,8 @@
ruleHandler.handleDynamicMaxAssign();
ruleHandler.handleMaxCapacityPercentage("test");
ruleHandler.handleMaxChildCapacity();
+ ruleHandler.handleMinResources();
+ ruleHandler.handleMaxResources();
ruleHandler.handleQueueAutoCreate("test");
ruleHandler.handleReservationSystem();
ruleHandler.handleSpecifiedNotFirstRule();
@@ -108,6 +116,8 @@
rules.put(DYNAMIC_MAX_ASSIGN, ABORT);
rules.put(MAX_CAPACITY_PERCENTAGE, ABORT);
rules.put(MAX_CHILD_CAPACITY, ABORT);
+ rules.put(MAX_RESOURCES, ABORT);
+ rules.put(MIN_RESOURCES, ABORT);
rules.put(QUEUE_AUTO_CREATE, ABORT);
rules.put(RESERVATION_SYSTEM, ABORT);
rules.put(SPECIFIED_NOT_FIRST, ABORT);
@@ -125,6 +135,8 @@
expectAbort(() -> ruleHandler.handleDynamicMaxAssign());
expectAbort(() -> ruleHandler.handleMaxCapacityPercentage("test"));
expectAbort(() -> ruleHandler.handleMaxChildCapacity());
+ expectAbort(() -> ruleHandler.handleMaxResources());
+ expectAbort(() -> ruleHandler.handleMinResources());
expectAbort(() -> ruleHandler.handleQueueAutoCreate("test"));
expectAbort(() -> ruleHandler.handleReservationSystem());
expectAbort(() -> ruleHandler.handleSpecifiedNotFirstRule());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
index 54e5056..580fbaa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
@@ -79,7 +79,7 @@
}
private FSQueueConverter converter;
- private Configuration config;
+ private Configuration yarnConfig;
private Configuration csConfig;
private FairScheduler fs;
private FSQueue rootQueue;
@@ -95,9 +95,10 @@
@Before
public void setup() {
- config = new Configuration(false);
- config.set(FairSchedulerConfiguration.ALLOCATION_FILE, FAIR_SCHEDULER_XML);
- config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
+ yarnConfig = new Configuration(false);
+ yarnConfig.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ FAIR_SCHEDULER_XML);
+ yarnConfig.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true);
csConfig = new Configuration(false);
dryRunResultHolder = new DryRunResultHolder();
conversionOptions =
@@ -123,7 +124,7 @@
FairScheduler fairScheduler = new FairScheduler();
fairScheduler.setRMContext(ctx);
- fairScheduler.init(config);
+ fairScheduler.init(yarnConfig);
return fairScheduler;
}
@@ -187,7 +188,7 @@
String absolutePath =
new File("src/test/resources/fair-scheduler-sameleafqueue.xml")
.getAbsolutePath();
- config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ yarnConfig.set(FairSchedulerConfiguration.ALLOCATION_FILE,
FILE_PREFIX + absolutePath);
fs.close();
fs = createFairScheduler();
@@ -317,19 +318,17 @@
converter.convertQueueHierarchy(rootQueue);
- assertEquals("root.users.joe maximum capacity", "[memory=8192, vcores=8]",
- csConfig.get(PREFIX + "root.users.joe.maximum-capacity"));
- assertEquals("root.admins.bob maximum capacity", "[memory=8192, vcores=2]",
- csConfig.get(PREFIX + "root.admins.bob.maximum-capacity"));
- assertEquals("root.admins.alice maximum capacity",
- "[memory=16384, vcores=4]",
- csConfig.get(PREFIX + "root.admins.alice.maximum-capacity"));
+ assertValueForQueues(ALL_QUEUES, ".maximum-capacity", csConfig, "100");
+ verify(ruleHandler, times(3)).handleMaxResources();
+ }
- Set<String> remaining = Sets.difference(ALL_QUEUES,
- Sets.newHashSet("root.users.joe",
- "root.admins.bob",
- "root.admins.alice"));
- assertNoValueForQueues(remaining, ".maximum-capacity", csConfig);
+ @Test
+ public void testQueueMinimumCapacity() {
+ converter = builder.build();
+
+ converter.convertQueueHierarchy(rootQueue);
+
+ verify(ruleHandler, times(2)).handleMinResources();
}
@Test
@@ -397,7 +396,7 @@
String absolutePath =
new File("src/test/resources/fair-scheduler-orderingpolicy.xml")
.getAbsolutePath();
- config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ yarnConfig.set(FairSchedulerConfiguration.ALLOCATION_FILE,
FILE_PREFIX + absolutePath);
fs.close();
fs = createFairScheduler();
@@ -434,7 +433,7 @@
String absolutePath =
new File("src/test/resources/fair-scheduler-orderingpolicy-mixed.xml")
.getAbsolutePath();
- config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ yarnConfig.set(FairSchedulerConfiguration.ALLOCATION_FILE,
FILE_PREFIX + absolutePath);
fs.close();
fs = createFairScheduler();
@@ -465,7 +464,8 @@
Mockito.doThrow(new UnsupportedPropertyException("maxCapacity"))
.when(ruleHandler).handleMaxChildCapacity();
- config.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true);
+ yarnConfig.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE,
+ true);
converter.convertQueueHierarchy(rootQueue);
}
@@ -475,7 +475,7 @@
String absolutePath =
new File("src/test/resources/fair-scheduler-sameleafqueue.xml")
.getAbsolutePath();
- config.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ yarnConfig.set(FairSchedulerConfiguration.ALLOCATION_FILE,
FILE_PREFIX + absolutePath);
fs.close();
fs = createFairScheduler();
@@ -492,23 +492,6 @@
error.contains("Leaf queues must be unique"));
}
- @Test
- public void testDryRunWithNoClusterResource() {
- builder.withClusterResource(null);
- prepareDryRunConverter();
-
- rootQueue = fs.getQueueManager().getRootQueue();
-
- converter.convertQueueHierarchy(rootQueue);
-
- assertEquals("Dry run errors", 1, dryRunResultHolder.getErrors().size());
- assertEquals("Dry run warnings", 0,
- dryRunResultHolder.getWarnings().size());
- String error = dryRunResultHolder.getErrors().iterator().next();
- assertTrue("Unexpected error message",
- error.contains("<maxResources> defined in percentages"));
- }
-
private void assertNoValueForQueues(Set<String> queues, String postfix,
Configuration config) {
for (String queue : queues) {
@@ -518,6 +501,15 @@
}
}
+ private void assertValueForQueues(Set<String> queues, String postfix,
+ Configuration config, String expectedValue) {
+ for (String queue : queues) {
+ String key = PREFIX + queue + postfix;
+ assertEquals("Key " + key + " has different value",
+ expectedValue, config.get(key));
+ }
+ }
+
private void assertTrueForQueues(Set<String> queues, String postfix,
Configuration config) {
for (String queue : queues) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml
index 67f9ed9..0f9abc3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml
@@ -31,6 +31,7 @@
<schedulingPolicy>drf</schedulingPolicy>
<queue name="john">
<weight>1.0</weight>
+ <minResources>memory-mb=4096, vcores=1</minResources>
<schedulingPolicy>drf</schedulingPolicy>
<aclSubmitApps>john </aclSubmitApps>
<aclAdministerApps>john </aclAdministerApps>
@@ -38,6 +39,7 @@
</queue>
<queue name="joe">
<maxResources>memory-mb=50.0%, vcores=50.0%</maxResources>
+ <minResources>memory-mb=4096, vcores=1</minResources>
<weight>3.0</weight>
<allowPreemptionFrom>false</allowPreemptionFrom>
<schedulingPolicy>drf</schedulingPolicy>
