This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new ca29768 YARN-10085. FS-CS converter: remove mixed ordering policy check. Contributed by Peter Bacsko ca29768 is described below commit ca297680352b7bf20ca6cd98ff81bb68e1358c5b Author: Szilard Nemeth <snem...@apache.org> AuthorDate: Tue Jan 28 15:22:12 2020 +0100 YARN-10085. FS-CS converter: remove mixed ordering policy check. Contributed by Peter Bacsko --- .../converter/FSConfigToCSConfigConverter.java | 61 +++++++++++++++----- .../converter/FSConfigToCSConfigRuleHandler.java | 12 ++++ .../scheduler/fair/converter/FSQueueConverter.java | 27 +++------ .../fair/converter/FSQueueConverterBuilder.java | 6 ++ .../fair/converter/FSYarnSiteConverter.java | 9 ++- .../fair/converter/QueuePlacementConverter.java | 3 - .../converter/TestFSConfigToCSConfigConverter.java | 67 +++++++++++++++------- .../TestFSConfigToCSConfigRuleHandler.java | 5 ++ .../fair/converter/TestFSQueueConverter.java | 20 +++++++ .../fair/converter/TestFSYarnSiteConverter.java | 33 +++++++++-- ...mixed.xml => fair-scheduler-onlyfairpolicy.xml} | 19 +++--- .../fair-scheduler-orderingpolicy-mixed.xml | 8 +-- 12 files changed, 195 insertions(+), 75 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java index a8fdb51..eae952d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -23,6 +23,7 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; @@ -39,9 +40,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationCo import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; -import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,6 +75,7 @@ public class FSConfigToCSConfigConverter { private boolean sizeBasedWeight = false; private boolean userAsDefaultQueue = false; private ConversionOptions conversionOptions; + private boolean drfUsed = false; private Configuration yarnSiteConfig; private Configuration capacitySchedulerConfig; @@ -198,7 +201,7 @@ public class FSConfigToCSConfigConverter { @VisibleForTesting void convert(Configuration conf) throws Exception { System.out.println(WARNING_TEXT); - + // initialize Fair Scheduler RMContext ctx = new RMContextImpl(); PlacementManager placementManager = new PlacementManager(); @@ -208,6 +211,8 @@ public class FSConfigToCSConfigConverter { fs.setRMContext(ctx); fs.init(conf); + drfUsed = isDrfUsed(fs); + AllocationConfiguration allocConf = fs.getAllocationConfiguration(); queueMaxAppsDefault = allocConf.getQueueMaxAppsDefault(); queueMaxAMShareDefault = allocConf.getQueueMaxAMShareDefault(); @@ -246,7 +251,7 @@ public class FSConfigToCSConfigConverter { private void convertYarnSiteXml(Configuration conf) { FSYarnSiteConverter siteConverter = new FSYarnSiteConverter(); - siteConverter.convertSiteProperties(conf, yarnSiteConfig); + siteConverter.convertSiteProperties(conf, yarnSiteConfig, drfUsed); autoCreateChildQueues = siteConverter.isAutoCreateChildQueues(); preemptionEnabled = siteConverter.isPreemptionEnabled(); @@ -271,6 +276,7 @@ public class FSConfigToCSConfigConverter { .withQueueMaxAMShareDefault(queueMaxAMShareDefault) .withQueueMaxAppsDefault(queueMaxAppsDefault) .withConversionOptions(conversionOptions) + .withDrfUsed(drfUsed) .build(); queueConverter.convertQueueHierarchy(rootQueue); @@ -287,18 +293,6 @@ public class FSConfigToCSConfigConverter { ruleHandler, userAsDefaultQueue); properties.forEach((k, v) -> capacitySchedulerConfig.set(k, v)); } - - // Validate ordering policy - if (queueConverter.isDrfPolicyUsedOnQueueLevel()) { - if (queueConverter.isFifoOrFairSharePolicyUsed()) { - throw new ConversionException( - "DRF ordering policy cannot be used together with fifo/fair"); - } else { - capacitySchedulerConfig.set( - CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, - DominantResourceCalculator.class.getCanonicalName()); - } - } } private void emitDefaultMaxApplications() { @@ -359,6 +353,38 @@ public class FSConfigToCSConfigConverter { } } + private boolean isDrfUsed(FairScheduler fs) { + FSQueue rootQueue = fs.getQueueManager().getRootQueue(); + AllocationConfiguration allocConf = fs.getAllocationConfiguration(); + + String defaultPolicy = allocConf.getDefaultSchedulingPolicy().getName(); + + if (DominantResourceFairnessPolicy.NAME.equals(defaultPolicy)) { + return true; + } else { + return isDrfUsedOnQueueLevel(rootQueue); + } + } + + private boolean isDrfUsedOnQueueLevel(FSQueue queue) { + String policy = queue.getPolicy().getName(); + boolean usesDrf = DominantResourceFairnessPolicy.NAME.equals(policy); + + if (usesDrf) { + return true; + } else { + List<FSQueue> children = queue.getChildQueues(); + + if (children != null) { + for (FSQueue child : children) { + usesDrf |= isDrfUsedOnQueueLevel(child); + } + } + + return usesDrf; + } + } + @VisibleForTesting Resource getClusterResource() { return clusterResource; @@ -373,4 +399,9 @@ public class FSConfigToCSConfigConverter { FSConfigToCSConfigRuleHandler getRuleHandler() { return ruleHandler; } + + @VisibleForTesting + Configuration getYarnSiteConfig() { + return yarnSiteConfig; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java index 2088334..a1db393 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java @@ -73,6 +73,9 @@ public class FSConfigToCSConfigRuleHandler { public static final String QUEUE_AUTO_CREATE = "queueAutoCreate.action"; + public static final String FAIR_AS_DRF = + "fairAsDrf.action"; + @VisibleForTesting enum RuleAction { WARNING, @@ -119,6 +122,7 @@ public class FSConfigToCSConfigRuleHandler { setActionForProperty(SPECIFIED_NOT_FIRST); setActionForProperty(RESERVATION_SYSTEM); setActionForProperty(QUEUE_AUTO_CREATE); + setActionForProperty(FAIR_AS_DRF); } public void handleMaxCapacityPercentage(String queueName) { @@ -181,6 +185,14 @@ public class FSConfigToCSConfigRuleHandler { placementRule)); } + public void handleFairAsDrf(String queueName) { + handle(FAIR_AS_DRF, + null, + format( + "Queue %s will use DRF policy instead of Fair", + queueName)); + } + private void handle(String actionName, String fsSetting, String message) { RuleAction action = actions.get(actionName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java index 1722302..44bdad4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java @@ -55,9 +55,7 @@ public class FSQueueConverter { private final float queueMaxAMShareDefault; private final boolean autoCreateChildQueues; private final int queueMaxAppsDefault; - - private boolean fifoOrFairSharePolicyUsed; - private boolean drfPolicyUsedOnQueueLevel; + private final boolean drfUsed; private ConversionOptions conversionOptions; @@ -72,6 +70,7 @@ public class FSQueueConverter { this.autoCreateChildQueues = builder.autoCreateChildQueues; this.queueMaxAppsDefault = builder.queueMaxAppsDefault; this.conversionOptions = builder.conversionOptions; + this.drfUsed = builder.drfUsed; } public void convertQueueHierarchy(FSQueue queue) { @@ -105,14 +104,6 @@ public class FSQueueConverter { } } - public boolean isFifoOrFairSharePolicyUsed() { - return fifoOrFairSharePolicyUsed; - } - - public boolean isDrfPolicyUsedOnQueueLevel() { - return drfPolicyUsedOnQueueLevel; - } - /** * Generates yarn.scheduler.capacity.<queue-name>.queues. * @param queueName @@ -306,20 +297,20 @@ public class FSQueueConverter { String policy = queue.getPolicy().getName(); switch (policy) { + case DominantResourceFairnessPolicy.NAME: + capacitySchedulerConfig.set(PREFIX + queueName + + ".ordering-policy", FairSharePolicy.NAME); + break; case FairSharePolicy.NAME: capacitySchedulerConfig.set(PREFIX + queueName + ".ordering-policy", FairSharePolicy.NAME); - fifoOrFairSharePolicyUsed = true; + if (drfUsed) { + ruleHandler.handleFairAsDrf(queueName); + } break; case FifoPolicy.NAME: capacitySchedulerConfig.set(PREFIX + queueName + ".ordering-policy", FifoPolicy.NAME); - fifoOrFairSharePolicyUsed = true; - break; - case DominantResourceFairnessPolicy.NAME: - // DRF is not supported on a queue level, - // it has to be global - drfPolicyUsedOnQueueLevel = true; break; default: String msg = String.format("Unexpected ordering policy " + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java index 615f1b9..dd5f05c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java @@ -32,6 +32,7 @@ public final class FSQueueConverterBuilder { float queueMaxAMShareDefault; int queueMaxAppsDefault; ConversionOptions conversionOptions; + boolean drfUsed; private FSQueueConverterBuilder() { } @@ -94,6 +95,11 @@ public final class FSQueueConverterBuilder { return this; } + public FSQueueConverterBuilder withDrfUsed(boolean drfUsed) { + this.drfUsed = drfUsed; + return this; + } + public FSQueueConverter build() { return new FSQueueConverter(this); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java index f6d023d..25db4d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; /** * Converts a Fair Scheduler site configuration to Capacity Scheduler @@ -37,7 +38,7 @@ public class FSYarnSiteConverter { @SuppressWarnings({"deprecation", "checkstyle:linelength"}) public void convertSiteProperties(Configuration conf, - Configuration yarnSiteConfig) { + Configuration yarnSiteConfig, boolean drfUsed) { yarnSiteConfig.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getCanonicalName()); @@ -139,6 +140,12 @@ public class FSYarnSiteConverter { FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE)) { userAsDefaultQueue = true; } + + if (drfUsed) { + yarnSiteConfig.set( + CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class.getCanonicalName()); + } } public boolean isPreemptionEnabled() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java index 4641dfc..8d3182f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java @@ -82,7 +82,6 @@ class QueuePlacementConverter { } mapping.append("u:" + USER + ":").append(defaultRule.defaultQueueName); } else if (rule instanceof SecondaryGroupExistingPlacementRule) { - // TODO: wait for YARN-9840 if (mapping.length() > 0) { mapping.append(";"); } @@ -107,10 +106,8 @@ class QueuePlacementConverter { mapping.append(";"); } if (pr instanceof PrimaryGroupPlacementRule) { - // TODO: wait for YARN-9841 mapping.append("u:" + USER + ":" + PRIMARY_GROUP + "." + USER); } else if (pr instanceof SecondaryGroupExistingPlacementRule) { - // TODO: wait for YARN-9865 mapping.append("u:" + USER + ":" + SECONDARY_GROUP + "." + USER); } else if (pr instanceof DefaultPlacementRule) { DefaultPlacementRule defaultRule = (DefaultPlacementRule) pr; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 4525dae..dc51db9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -60,6 +61,8 @@ import org.mockito.junit.MockitoJUnitRunner; */ @RunWith(MockitoJUnitRunner.class) public class TestFSConfigToCSConfigConverter { + private static final String CLUSTER_RESOURCE_STRING = + "vcores=20, memory-mb=240"; private static final Resource CLUSTER_RESOURCE = Resource.newInstance(16384, 16); private static final String FILE_PREFIX = "file:"; @@ -67,6 +70,10 @@ public class TestFSConfigToCSConfigConverter { prepareFileName("fair-scheduler-conversion.xml"); private static final String FS_INVALID_PLACEMENT_RULES_XML = prepareFileName("fair-scheduler-invalidplacementrules.xml"); + private static final String FS_ONLY_FAIR_POLICY_XML = + prepareFileName("fair-scheduler-onlyfairpolicy.xml"); + private static final String FS_MIXED_POLICY_XML = + prepareFileName("fair-scheduler-orderingpolicy-mixed.xml"); @Mock private FSConfigToCSConfigRuleHandler ruleHandler; @@ -216,20 +223,6 @@ public class TestFSConfigToCSConfigConverter { } @Test - public void testMixedQueueOrderingPolicy() throws Exception { - expectedException.expect(ConversionException.class); - expectedException.expectMessage( - "DRF ordering policy cannot be used together with fifo/fair"); - String absolutePath = - new File("src/test/resources/fair-scheduler-orderingpolicy-mixed.xml") - .getAbsolutePath(); - config.set(FairSchedulerConfiguration.ALLOCATION_FILE, - FILE_PREFIX + absolutePath); - - converter.convert(config); - } - - @Test public void testQueueMaxChildCapacityNotSupported() throws Exception { expectedException.expect(UnsupportedPropertyException.class); expectedException.expectMessage("test"); @@ -277,7 +270,7 @@ public class TestFSConfigToCSConfigConverter { @Test public void testConvertFSConfigurationClusterResource() throws Exception { FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .build(); converter.convert(params); assertEquals("Resource", Resource.newInstance(240, 20), @@ -288,7 +281,7 @@ public class TestFSConfigToCSConfigConverter { public void testConvertFSConfigPctModeUsedAndClusterResourceDefined() throws Exception { FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .build(); converter.convert(params); assertEquals("Resource", Resource.newInstance(240, 20), @@ -394,7 +387,7 @@ public class TestFSConfigToCSConfigConverter { @Test public void testConvertCheckOutputDir() throws Exception { FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .build(); converter.convert(params); @@ -419,7 +412,7 @@ public class TestFSConfigToCSConfigConverter { throws Exception { FSConfigToCSConfigConverterParams params = createParamsBuilder(YARN_SITE_XML_NO_REF_TO_FS_XML) - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .build(); expectedException.expect(PreconditionException.class); @@ -430,7 +423,7 @@ public class TestFSConfigToCSConfigConverter { @Test public void testInvalidFairSchedulerXml() throws Exception { FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .withFairSchedulerXmlConfig(FAIR_SCHEDULER_XML_INVALID) .build(); @@ -442,7 +435,7 @@ public class TestFSConfigToCSConfigConverter { public void testInvalidYarnSiteXml() throws Exception { FSConfigToCSConfigConverterParams params = createParamsBuilder(YARN_SITE_XML_INVALID) - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .build(); expectedException.expect(RuntimeException.class); @@ -464,7 +457,7 @@ public class TestFSConfigToCSConfigConverter { public void testConversionWhenInvalidPlacementRulesIgnored() throws Exception { FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .withFairSchedulerXmlConfig(FS_INVALID_PLACEMENT_RULES_XML) .build(); @@ -479,6 +472,38 @@ public class TestFSConfigToCSConfigConverter { // expected: no exception } + @Test + public void testConversionWhenOnlyFairPolicyIsUsed() throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource(CLUSTER_RESOURCE_STRING) + .withFairSchedulerXmlConfig(FS_ONLY_FAIR_POLICY_XML) + .build(); + + converter.convert(params); + + Configuration convertedConfig = converter.getYarnSiteConfig(); + + assertEquals("Resource calculator class shouldn't be set", null, + convertedConfig.getClass( + CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, null)); + } + + @Test + public void testConversionWhenMixedPolicyIsUsed() throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource(CLUSTER_RESOURCE_STRING) + .withFairSchedulerXmlConfig(FS_MIXED_POLICY_XML) + .build(); + + converter.convert(params); + + Configuration convertedConfig = converter.getYarnSiteConfig(); + + assertEquals("Resource calculator type", DominantResourceCalculator.class, + convertedConfig.getClass( + CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, null)); + } + private Configuration getConvertedCSConfig() { ByteArrayInputStream input = new ByteArrayInputStream(csConfigOut.toByteArray()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java index b6caadd..4539e1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java @@ -25,6 +25,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.conve import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.SPECIFIED_NOT_FIRST; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_APPS_DEFAULT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_RUNNING_APPS; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.FAIR_AS_DRF; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -86,6 +87,7 @@ public class TestFSConfigToCSConfigRuleHandler { rules.put(SPECIFIED_NOT_FIRST, WARNING); rules.put(USER_MAX_APPS_DEFAULT, WARNING); rules.put(USER_MAX_RUNNING_APPS, WARNING); + rules.put(FAIR_AS_DRF, WARNING); ruleHandler = new FSConfigToCSConfigRuleHandler(rules, createDefaultConversionOptions()); @@ -111,6 +113,8 @@ public class TestFSConfigToCSConfigRuleHandler { rules.put(SPECIFIED_NOT_FIRST, ABORT); rules.put(USER_MAX_APPS_DEFAULT, ABORT); rules.put(USER_MAX_RUNNING_APPS, ABORT); + rules.put(USER_MAX_RUNNING_APPS, ABORT); + rules.put(FAIR_AS_DRF, ABORT); rules.put(MAX_CHILD_QUEUE_LIMIT, "1"); ruleHandler = new FSConfigToCSConfigRuleHandler(rules, @@ -126,6 +130,7 @@ public class TestFSConfigToCSConfigRuleHandler { expectAbort(() -> ruleHandler.handleSpecifiedNotFirstRule()); expectAbort(() -> ruleHandler.handleUserMaxApps()); expectAbort(() -> ruleHandler.handleUserMaxAppsDefault()); + expectAbort(() -> ruleHandler.handleFairAsDrf("test")); } @Test(expected = ConversionException.class) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java index 3507c83..2221364 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java @@ -20,6 +20,9 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import java.io.File; import java.io.IOException; @@ -401,6 +404,23 @@ public class TestFSQueueConverter { } @Test + public void testQueueUnsupportedMixedOrderingPolicy() throws IOException { + converter = builder.withDrfUsed(true).build(); + String absolutePath = + new File("src/test/resources/fair-scheduler-orderingpolicy-mixed.xml") + .getAbsolutePath(); + config.set(FairSchedulerConfiguration.ALLOCATION_FILE, + FILE_PREFIX + absolutePath); + fs.close(); + fs = createFairScheduler(); + rootQueue = fs.getQueueManager().getRootQueue(); + + converter.convertQueueHierarchy(rootQueue); + + verify(ruleHandler, times(6)).handleFairAsDrf(anyString()); + } + + @Test public void testQueueMaxChildCapacityNotSupported() { converter = builder.build(); expectedException.expect(UnsupportedPropertyException.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java index edb9a4e..0bd8fb1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java @@ -20,6 +20,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.junit.Before; import org.junit.Test; @@ -50,7 +52,7 @@ public class TestFSYarnSiteConverter { yarnConfig.setInt( FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_SLEEP_MS, 666); - converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); assertTrue("Cont. scheduling", yarnConvertedConfig.getBoolean( CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false)); @@ -65,7 +67,7 @@ public class TestFSYarnSiteConverter { yarnConfig.setInt("yarn.resource-types.memory-mb.increment-allocation", 11); yarnConfig.setInt("yarn.resource-types.vcores.increment-allocation", 5); - converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); assertEquals("Memory alloc increment", 11, yarnConvertedConfig.getInt("yarn.scheduler.minimum-allocation-mb", @@ -83,7 +85,7 @@ public class TestFSYarnSiteConverter { FairSchedulerConfiguration.WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS, 321); - converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); assertTrue("Preemption enabled", yarnConvertedConfig.getBoolean( @@ -103,7 +105,7 @@ public class TestFSYarnSiteConverter { public void testSiteAssignMultipleConversion() { yarnConfig.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); - converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); assertTrue("Assign multiple", yarnConvertedConfig.getBoolean( @@ -115,7 +117,7 @@ public class TestFSYarnSiteConverter { public void testSiteMaxAssignConversion() { yarnConfig.setInt(FairSchedulerConfiguration.MAX_ASSIGN, 111); - converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); assertEquals("Max assign", 111, yarnConvertedConfig.getInt( @@ -129,7 +131,7 @@ public class TestFSYarnSiteConverter { yarnConfig.set(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, "321.321"); - converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); assertEquals("Locality threshold node", "123.123", yarnConvertedConfig.get( @@ -138,4 +140,23 @@ public class TestFSYarnSiteConverter { yarnConvertedConfig.get( CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY)); } + + @Test + public void testSiteDrfEnabledConversion() { + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, true); + + assertEquals("Resource calculator type", DominantResourceCalculator.class, + yarnConvertedConfig.getClass( + CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, null)); + } + + @Test + public void testSiteDrfDisabledConversion() { + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); + + assertEquals("Resource calculator type", DefaultResourceCalculator.class, + yarnConvertedConfig.getClass( + CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + CapacitySchedulerConfiguration.DEFAULT_RESOURCE_CALCULATOR_CLASS)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-onlyfairpolicy.xml similarity index 86% copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-onlyfairpolicy.xml index 3a2a593..e03be5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-onlyfairpolicy.xml @@ -19,7 +19,7 @@ <allocations> <queue name="root"> <weight>1.0</weight> - <schedulingPolicy>drf</schedulingPolicy> + <schedulingPolicy>fair</schedulingPolicy> <aclSubmitApps>alice,bob,joe,john hadoop_users</aclSubmitApps> <aclAdministerApps>alice,bob,joe,john hadoop_users</aclAdministerApps> <queue name="default"> @@ -28,10 +28,10 @@ </queue> <queue name="users" type="parent"> <weight>1.0</weight> - <schedulingPolicy>drf</schedulingPolicy> + <schedulingPolicy>fair</schedulingPolicy> <queue name="john"> <weight>1.0</weight> - <schedulingPolicy>drf</schedulingPolicy> + <schedulingPolicy>fair</schedulingPolicy> <aclSubmitApps>john </aclSubmitApps> <aclAdministerApps>john </aclAdministerApps> <maxContainerAllocation>vcores=2,memory-mb=8192</maxContainerAllocation> @@ -40,7 +40,7 @@ <maxResources>memory-mb=50.0%, vcores=50.0%</maxResources> <weight>3.0</weight> <allowPreemptionFrom>false</allowPreemptionFrom> - <schedulingPolicy>drf</schedulingPolicy> + <schedulingPolicy>fair</schedulingPolicy> <aclSubmitApps>joe </aclSubmitApps> <aclAdministerApps>joe </aclAdministerApps> </queue> @@ -55,26 +55,31 @@ <maxRunningApps>2</maxRunningApps> <weight>3.0</weight> <allowPreemptionFrom>false</allowPreemptionFrom> - <schedulingPolicy>drf</schedulingPolicy> + <schedulingPolicy>fair</schedulingPolicy> <aclSubmitApps>alice </aclSubmitApps> <aclAdministerApps>alice </aclAdministerApps> <maxAMShare>0.15</maxAMShare> + <reservation>memory-mb=16384, vcores=4</reservation> </queue> <queue name="bob"> <maxResources>memory-mb=8192, vcores=2</maxResources> <weight>1.0</weight> - <schedulingPolicy>drf</schedulingPolicy> + <schedulingPolicy>fair</schedulingPolicy> <aclSubmitApps>bob </aclSubmitApps> <aclAdministerApps>bob </aclAdministerApps> <maxAMShare>-1.0</maxAMShare> </queue> </queue> </queue> + <user name="alice"> + <maxRunningApps>30</maxRunningApps> + </user> + <userMaxAppsDefault>10</userMaxAppsDefault> <defaultFairSharePreemptionTimeout>23</defaultFairSharePreemptionTimeout> <defaultMinSharePreemptionTimeout>24</defaultMinSharePreemptionTimeout> <defaultFairSharePreemptionThreshold>0.12</defaultFairSharePreemptionThreshold> <queueMaxAppsDefault>15</queueMaxAppsDefault> - <defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy> + <defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy> <queueMaxAMShareDefault>0.16</queueMaxAMShareDefault> <queuePlacementPolicy> <rule name="nestedUserQueue" create="false"> diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml index 3a2a593..778fb86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml @@ -31,7 +31,7 @@ <schedulingPolicy>drf</schedulingPolicy> <queue name="john"> <weight>1.0</weight> - <schedulingPolicy>drf</schedulingPolicy> + <schedulingPolicy>fair</schedulingPolicy> <aclSubmitApps>john </aclSubmitApps> <aclAdministerApps>john </aclAdministerApps> <maxContainerAllocation>vcores=2,memory-mb=8192</maxContainerAllocation> @@ -40,7 +40,7 @@ <maxResources>memory-mb=50.0%, vcores=50.0%</maxResources> <weight>3.0</weight> <allowPreemptionFrom>false</allowPreemptionFrom> - <schedulingPolicy>drf</schedulingPolicy> + <schedulingPolicy>fair</schedulingPolicy> <aclSubmitApps>joe </aclSubmitApps> <aclAdministerApps>joe </aclAdministerApps> </queue> @@ -55,7 +55,7 @@ <maxRunningApps>2</maxRunningApps> <weight>3.0</weight> <allowPreemptionFrom>false</allowPreemptionFrom> - <schedulingPolicy>drf</schedulingPolicy> + <schedulingPolicy>fair</schedulingPolicy> <aclSubmitApps>alice </aclSubmitApps> <aclAdministerApps>alice </aclAdministerApps> <maxAMShare>0.15</maxAMShare> @@ -63,7 +63,7 @@ <queue name="bob"> <maxResources>memory-mb=8192, vcores=2</maxResources> <weight>1.0</weight> - <schedulingPolicy>drf</schedulingPolicy> + <schedulingPolicy>fair</schedulingPolicy> <aclSubmitApps>bob </aclSubmitApps> <aclAdministerApps>bob </aclAdministerApps> <maxAMShare>-1.0</maxAMShare> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org