[48/50] [abbrv] hadoop git commit: YARN-7707. [GPG] Policy generator framework. Contributed by Young Chen
YARN-7707. [GPG] Policy generator framework. Contributed by Young Chen Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b97ad374 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b97ad374 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b97ad374 Branch: refs/heads/YARN-7402 Commit: b97ad374035275d4c53dbba4f3cd2f51f6c6e7de Parents: eac28b5 Author: Botong Huang Authored: Fri Mar 23 17:07:10 2018 -0700 Committer: Botong Huang Committed: Mon Nov 12 15:09:38 2018 -0800 -- .../hadoop/yarn/conf/YarnConfiguration.java | 36 +- .../src/main/resources/yarn-default.xml | 40 +++ .../utils/FederationStateStoreFacade.java | 13 + .../pom.xml | 18 + .../globalpolicygenerator/GPGContext.java | 4 + .../globalpolicygenerator/GPGContextImpl.java | 10 + .../globalpolicygenerator/GPGPolicyFacade.java | 220 .../server/globalpolicygenerator/GPGUtils.java | 80 + .../GlobalPolicyGenerator.java | 17 + .../policygenerator/GlobalPolicy.java | 76 + .../policygenerator/NoOpGlobalPolicy.java | 36 ++ .../policygenerator/PolicyGenerator.java| 261 ++ .../UniformWeightedLocalityGlobalPolicy.java| 71 .../policygenerator/package-info.java | 24 ++ .../TestGPGPolicyFacade.java| 202 +++ .../policygenerator/TestPolicyGenerator.java| 338 +++ .../src/test/resources/schedulerInfo1.json | 134 .../src/test/resources/schedulerInfo2.json | 196 +++ 18 files changed, 1775 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b97ad374/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 0b35b9f..c42ebb1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3427,7 +3427,7 @@ public class YarnConfiguration extends Configuration { public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED = false; - private static final String FEDERATION_GPG_PREFIX = + public static final String FEDERATION_GPG_PREFIX = FEDERATION_PREFIX + "gpg."; // The number of threads to use for the GPG scheduled executor service @@ -3445,6 +3445,40 @@ public class YarnConfiguration extends Configuration { FEDERATION_GPG_PREFIX + "subcluster.heartbeat.expiration-ms"; public static final long DEFAULT_GPG_SUBCLUSTER_EXPIRATION_MS = 180; + public static final String FEDERATION_GPG_POLICY_PREFIX = + FEDERATION_GPG_PREFIX + "policy.generator."; + + /** The interval at which the policy generator runs, default is one hour. */ + public static final String GPG_POLICY_GENERATOR_INTERVAL_MS = + FEDERATION_GPG_POLICY_PREFIX + "interval-ms"; + public static final long DEFAULT_GPG_POLICY_GENERATOR_INTERVAL_MS = -1; + + /** + * The configured policy generator class, runs NoOpGlobalPolicy by + * default. + */ + public static final String GPG_GLOBAL_POLICY_CLASS = + FEDERATION_GPG_POLICY_PREFIX + "class"; + public static final String DEFAULT_GPG_GLOBAL_POLICY_CLASS = + "org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator." + + "NoOpGlobalPolicy"; + + /** + * Whether or not the policy generator is running in read only (won't modify + * policies), default is false. + */ + public static final String GPG_POLICY_GENERATOR_READONLY = + FEDERATION_GPG_POLICY_PREFIX + "readonly"; + public static final boolean DEFAULT_GPG_POLICY_GENERATOR_READONLY = + false; + + /** + * Which sub-clusters the policy generator should blacklist. + */ + public static final String GPG_POLICY_GENERATOR_BLACKLIST = + FEDERATION_GPG_POLICY_PREFIX + "blacklist"; + + // Other Configs http://git-wip-us.apache.org/repos/asf/hadoop/blob/b97ad374/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
[48/50] [abbrv] hadoop git commit: YARN-7707. [GPG] Policy generator framework. Contributed by Young Chen
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5da8ca6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo2.json -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo2.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo2.json new file mode 100644 index 000..2ff879e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo2.json @@ -0,0 +1,196 @@ + { + "type": "capacityScheduler", + "capacity": 100.0, + "usedCapacity": 0.0, + "maxCapacity": 100.0, + "queueName": "root", + "queues": { +"queue": [ + { +"type": "capacitySchedulerLeafQueueInfo", +"capacity": 100.0, +"usedCapacity": 0.0, +"maxCapacity": 100.0, +"absoluteCapacity": 100.0, +"absoluteMaxCapacity": 100.0, +"absoluteUsedCapacity": 0.0, +"numApplications": 484, +"queueName": "default", +"state": "RUNNING", +"resourcesUsed": { + "memory": 0, + "vCores": 0 +}, +"hideReservationQueues": false, +"nodeLabels": [ + "*" +], +"numActiveApplications": 484, +"numPendingApplications": 0, +"numContainers": 0, +"maxApplications": 1, +"maxApplicationsPerUser": 1, +"userLimit": 100, +"users": { + "user": [ +{ + "username": "Default", + "resourcesUsed": { +"memory": 0, +"vCores": 0 + }, + "numPendingApplications": 0, + "numActiveApplications": 468, + "AMResourceUsed": { +"memory": 30191616, +"vCores": 468 + }, + "userResourceLimit": { +"memory": 31490048, +"vCores": 7612 + } +} + ] +}, +"userLimitFactor": 1.0, +"AMResourceLimit": { + "memory": 31490048, + "vCores": 7612 +}, +"usedAMResource": { + "memory": 30388224, + "vCores": 532 +}, +"userAMResourceLimit": { + "memory": 31490048, + "vCores": 7612 +}, +"preemptionDisabled": true + }, + { +"type": "capacitySchedulerLeafQueueInfo", +"capacity": 100.0, +"usedCapacity": 0.0, +"maxCapacity": 100.0, +"absoluteCapacity": 100.0, +"absoluteMaxCapacity": 100.0, +"absoluteUsedCapacity": 0.0, +"numApplications": 484, +"queueName": "default2", +"state": "RUNNING", +"resourcesUsed": { + "memory": 0, + "vCores": 0 +}, +"hideReservationQueues": false, +"nodeLabels": [ + "*" +], +"numActiveApplications": 484, +"numPendingApplications": 0, +"numContainers": 0, +"maxApplications": 1, +"maxApplicationsPerUser": 1, +"userLimit": 100, +"users": { + "user": [ +{ + "username": "Default", + "resourcesUsed": { +"memory": 0, +"vCores": 0 + }, + "numPendingApplications": 0, + "numActiveApplications": 468, + "AMResourceUsed": { +"memory": 30191616, +"vCores": 468 + }, + "userResourceLimit": { +"memory": 31490048, +"vCores": 7612 + } +} + ] +}, +"userLimitFactor": 1.0, +"AMResourceLimit": { + "memory": 31490048, + "vCores": 7612 +}, +"usedAMResource": { + "memory": 30388224, + "vCores": 532 +}, +"userAMResourceLimit": { + "memory": 31490048, + "vCores": 7612 +}, +"preemptionDisabled": true + } +] + }, + "health": { +"lastrun": 1517951638085, +"operationsInfo": { + "entry": { +"key":