[43/50] [abbrv] hadoop git commit: YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong)
YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eac28b59 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eac28b59 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eac28b59 Branch: refs/heads/YARN-7402 Commit: eac28b599575be5291f89faa46fa081b3f7b0676 Parents: af0c804 Author: Botong Huang Authored: Thu Feb 1 14:43:48 2018 -0800 Committer: Botong Huang Committed: Mon Nov 12 15:09:38 2018 -0800 -- .../dev-support/findbugs-exclude.xml| 5 + .../hadoop/yarn/conf/YarnConfiguration.java | 18 +++ .../src/main/resources/yarn-default.xml | 24 .../store/impl/MemoryFederationStateStore.java | 13 ++ .../utils/FederationStateStoreFacade.java | 41 ++- .../GlobalPolicyGenerator.java | 92 ++- .../subclustercleaner/SubClusterCleaner.java| 109 + .../subclustercleaner/package-info.java | 19 +++ .../TestSubClusterCleaner.java | 118 +++ 9 files changed, 409 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac28b59/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index dd42129..507e116 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -387,6 +387,11 @@ + + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac28b59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index e88d594..0b35b9f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3427,6 +3427,24 @@ public class YarnConfiguration extends Configuration { public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED = false; + private static final String FEDERATION_GPG_PREFIX = + FEDERATION_PREFIX + "gpg."; + + // The number of threads to use for the GPG scheduled executor service + public static final String GPG_SCHEDULED_EXECUTOR_THREADS = + FEDERATION_GPG_PREFIX + "scheduled.executor.threads"; + public static final int DEFAULT_GPG_SCHEDULED_EXECUTOR_THREADS = 10; + + // The interval at which the subcluster cleaner runs, -1 means disabled + public static final String GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = + FEDERATION_GPG_PREFIX + "subcluster.cleaner.interval-ms"; + public static final long DEFAULT_GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = -1; + + // The expiration time for a subcluster heartbeat, default is 30 minutes + public static final String GPG_SUBCLUSTER_EXPIRATION_MS = + FEDERATION_GPG_PREFIX + "subcluster.heartbeat.expiration-ms"; + public static final long DEFAULT_GPG_SUBCLUSTER_EXPIRATION_MS = 180; + // Other Configs http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac28b59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index f5493bc..4083ff0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3647,6 +3647,30 @@ + The number of threads to use for the GPG scheduled executor service. + +yarn.federation.gpg.scheduled.executor.threads +10 + + + + + The interval at which the subcluster cleaner runs, -1 means disabled. + +yarn.federation.gpg.subcluster.cleaner.interval-ms +-1 + + + + + The expiration time for a subcluster heartbeat, default is 30 minutes. + +yarn.federation.gpg.subcluster.heartbeat.
[42/50] [abbrv] hadoop git commit: YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong)
YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33c2916d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33c2916d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33c2916d Branch: refs/heads/YARN-7402 Commit: 33c2916d6360e5abf4efaf924612c4daa1061aff Parents: 6bea5ee Author: Botong Huang Authored: Thu Feb 1 14:43:48 2018 -0800 Committer: Botong Huang Committed: Wed Sep 19 13:47:32 2018 -0700 -- .../dev-support/findbugs-exclude.xml| 5 + .../hadoop/yarn/conf/YarnConfiguration.java | 18 +++ .../src/main/resources/yarn-default.xml | 24 .../store/impl/MemoryFederationStateStore.java | 13 ++ .../utils/FederationStateStoreFacade.java | 41 ++- .../GlobalPolicyGenerator.java | 92 ++- .../subclustercleaner/SubClusterCleaner.java| 109 + .../subclustercleaner/package-info.java | 19 +++ .../TestSubClusterCleaner.java | 118 +++ 9 files changed, 409 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/33c2916d/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 216c3bd..9fcafad 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -387,6 +387,11 @@ + + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/33c2916d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a82801d..5c7bf26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3365,6 +3365,24 @@ public class YarnConfiguration extends Configuration { public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED = false; + private static final String FEDERATION_GPG_PREFIX = + FEDERATION_PREFIX + "gpg."; + + // The number of threads to use for the GPG scheduled executor service + public static final String GPG_SCHEDULED_EXECUTOR_THREADS = + FEDERATION_GPG_PREFIX + "scheduled.executor.threads"; + public static final int DEFAULT_GPG_SCHEDULED_EXECUTOR_THREADS = 10; + + // The interval at which the subcluster cleaner runs, -1 means disabled + public static final String GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = + FEDERATION_GPG_PREFIX + "subcluster.cleaner.interval-ms"; + public static final long DEFAULT_GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = -1; + + // The expiration time for a subcluster heartbeat, default is 30 minutes + public static final String GPG_SUBCLUSTER_EXPIRATION_MS = + FEDERATION_GPG_PREFIX + "subcluster.heartbeat.expiration-ms"; + public static final long DEFAULT_GPG_SUBCLUSTER_EXPIRATION_MS = 180; + // Other Configs http://git-wip-us.apache.org/repos/asf/hadoop/blob/33c2916d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0700902..7df0a67 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3611,6 +3611,30 @@ + The number of threads to use for the GPG scheduled executor service. + +yarn.federation.gpg.scheduled.executor.threads +10 + + + + + The interval at which the subcluster cleaner runs, -1 means disabled. + +yarn.federation.gpg.subcluster.cleaner.interval-ms +-1 + + + + + The expiration time for a subcluster heartbeat, default is 30 minutes. + +yarn.federation.gpg.subcluster.heartbeat.
[48/50] [abbrv] hadoop git commit: YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong)
YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f833e1b3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f833e1b3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f833e1b3 Branch: refs/heads/YARN-7402 Commit: f833e1b300758e7c7622e2ca93c2dd164ec6d73d Parents: 48a8379 Author: Botong Huang Authored: Thu Feb 1 14:43:48 2018 -0800 Committer: Botong Huang Committed: Thu Aug 2 09:59:48 2018 -0700 -- .../dev-support/findbugs-exclude.xml| 5 + .../hadoop/yarn/conf/YarnConfiguration.java | 18 +++ .../src/main/resources/yarn-default.xml | 24 .../store/impl/MemoryFederationStateStore.java | 13 ++ .../utils/FederationStateStoreFacade.java | 41 ++- .../GlobalPolicyGenerator.java | 92 ++- .../subclustercleaner/SubClusterCleaner.java| 109 + .../subclustercleaner/package-info.java | 19 +++ .../TestSubClusterCleaner.java | 118 +++ 9 files changed, 409 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f833e1b3/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 216c3bd..9fcafad 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -387,6 +387,11 @@ + + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/f833e1b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index bbf877f..ec88411 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3342,6 +3342,24 @@ public class YarnConfiguration extends Configuration { public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED = false; + private static final String FEDERATION_GPG_PREFIX = + FEDERATION_PREFIX + "gpg."; + + // The number of threads to use for the GPG scheduled executor service + public static final String GPG_SCHEDULED_EXECUTOR_THREADS = + FEDERATION_GPG_PREFIX + "scheduled.executor.threads"; + public static final int DEFAULT_GPG_SCHEDULED_EXECUTOR_THREADS = 10; + + // The interval at which the subcluster cleaner runs, -1 means disabled + public static final String GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = + FEDERATION_GPG_PREFIX + "subcluster.cleaner.interval-ms"; + public static final long DEFAULT_GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = -1; + + // The expiration time for a subcluster heartbeat, default is 30 minutes + public static final String GPG_SUBCLUSTER_EXPIRATION_MS = + FEDERATION_GPG_PREFIX + "subcluster.heartbeat.expiration-ms"; + public static final long DEFAULT_GPG_SUBCLUSTER_EXPIRATION_MS = 180; + // Other Configs http://git-wip-us.apache.org/repos/asf/hadoop/blob/f833e1b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 2cc842f..66493f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3533,6 +3533,30 @@ + The number of threads to use for the GPG scheduled executor service. + +yarn.federation.gpg.scheduled.executor.threads +10 + + + + + The interval at which the subcluster cleaner runs, -1 means disabled. + +yarn.federation.gpg.subcluster.cleaner.interval-ms +-1 + + + + + The expiration time for a subcluster heartbeat, default is 30 minutes. + +yarn.federation.gpg.subcluster.heartbeat.e
[50/50] [abbrv] hadoop git commit: YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong)
YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa3ee34c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa3ee34c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa3ee34c Branch: refs/heads/YARN-7402 Commit: fa3ee34c7b74889bcbfd2effb999757c73994dd4 Parents: 43b8c2d Author: Botong Huang Authored: Thu Feb 1 14:43:48 2018 -0800 Committer: Botong Huang Committed: Fri Jul 13 17:42:58 2018 -0700 -- .../dev-support/findbugs-exclude.xml| 5 + .../hadoop/yarn/conf/YarnConfiguration.java | 18 +++ .../src/main/resources/yarn-default.xml | 24 .../store/impl/MemoryFederationStateStore.java | 13 ++ .../utils/FederationStateStoreFacade.java | 41 ++- .../GlobalPolicyGenerator.java | 92 ++- .../subclustercleaner/SubClusterCleaner.java| 109 + .../subclustercleaner/package-info.java | 19 +++ .../TestSubClusterCleaner.java | 118 +++ 9 files changed, 409 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa3ee34c/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 5cc81e5..406a8b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -387,6 +387,11 @@ + + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa3ee34c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 9156c2d..b3a4ccb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3335,6 +3335,24 @@ public class YarnConfiguration extends Configuration { public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED = false; + private static final String FEDERATION_GPG_PREFIX = + FEDERATION_PREFIX + "gpg."; + + // The number of threads to use for the GPG scheduled executor service + public static final String GPG_SCHEDULED_EXECUTOR_THREADS = + FEDERATION_GPG_PREFIX + "scheduled.executor.threads"; + public static final int DEFAULT_GPG_SCHEDULED_EXECUTOR_THREADS = 10; + + // The interval at which the subcluster cleaner runs, -1 means disabled + public static final String GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = + FEDERATION_GPG_PREFIX + "subcluster.cleaner.interval-ms"; + public static final long DEFAULT_GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = -1; + + // The expiration time for a subcluster heartbeat, default is 30 minutes + public static final String GPG_SUBCLUSTER_EXPIRATION_MS = + FEDERATION_GPG_PREFIX + "subcluster.heartbeat.expiration-ms"; + public static final long DEFAULT_GPG_SUBCLUSTER_EXPIRATION_MS = 180; + // Other Configs http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa3ee34c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 2cc842f..66493f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3533,6 +3533,30 @@ + The number of threads to use for the GPG scheduled executor service. + +yarn.federation.gpg.scheduled.executor.threads +10 + + + + + The interval at which the subcluster cleaner runs, -1 means disabled. + +yarn.federation.gpg.subcluster.cleaner.interval-ms +-1 + + + + + The expiration time for a subcluster heartbeat, default is 30 minutes. + +yarn.federation.gpg.subcluster.heartbeat.
[46/50] [abbrv] hadoop git commit: YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong)
YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46a4a945 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46a4a945 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46a4a945 Branch: refs/heads/YARN-7402 Commit: 46a4a945732afdefec9828d1c43b77d32609bb8a Parents: bca8e9b Author: Botong Huang Authored: Thu Feb 1 14:43:48 2018 -0800 Committer: Botong Huang Committed: Tue May 29 10:48:40 2018 -0700 -- .../dev-support/findbugs-exclude.xml| 5 + .../hadoop/yarn/conf/YarnConfiguration.java | 18 +++ .../src/main/resources/yarn-default.xml | 24 .../store/impl/MemoryFederationStateStore.java | 13 ++ .../utils/FederationStateStoreFacade.java | 41 ++- .../GlobalPolicyGenerator.java | 92 ++- .../subclustercleaner/SubClusterCleaner.java| 109 + .../subclustercleaner/package-info.java | 19 +++ .../TestSubClusterCleaner.java | 118 +++ 9 files changed, 409 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/46a4a945/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 5841361..bf2e376 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -380,6 +380,11 @@ + + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/46a4a945/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index f7f82f8..7c78e0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3326,6 +3326,24 @@ public class YarnConfiguration extends Configuration { public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED = false; + private static final String FEDERATION_GPG_PREFIX = + FEDERATION_PREFIX + "gpg."; + + // The number of threads to use for the GPG scheduled executor service + public static final String GPG_SCHEDULED_EXECUTOR_THREADS = + FEDERATION_GPG_PREFIX + "scheduled.executor.threads"; + public static final int DEFAULT_GPG_SCHEDULED_EXECUTOR_THREADS = 10; + + // The interval at which the subcluster cleaner runs, -1 means disabled + public static final String GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = + FEDERATION_GPG_PREFIX + "subcluster.cleaner.interval-ms"; + public static final long DEFAULT_GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = -1; + + // The expiration time for a subcluster heartbeat, default is 30 minutes + public static final String GPG_SUBCLUSTER_EXPIRATION_MS = + FEDERATION_GPG_PREFIX + "subcluster.heartbeat.expiration-ms"; + public static final long DEFAULT_GPG_SUBCLUSTER_EXPIRATION_MS = 180; + // Other Configs http://git-wip-us.apache.org/repos/asf/hadoop/blob/46a4a945/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index b0ffc48..8a450d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3524,6 +3524,30 @@ + The number of threads to use for the GPG scheduled executor service. + +yarn.federation.gpg.scheduled.executor.threads +10 + + + + + The interval at which the subcluster cleaner runs, -1 means disabled. + +yarn.federation.gpg.subcluster.cleaner.interval-ms +-1 + + + + + The expiration time for a subcluster heartbeat, default is 30 minutes. + +yarn.federation.gpg.subcluster.heartbeat.
hadoop git commit: YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong)
Repository: hadoop Updated Branches: refs/heads/YARN-7402 1702dfa7f -> fd03fd45d YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fd03fd45 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fd03fd45 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fd03fd45 Branch: refs/heads/YARN-7402 Commit: fd03fd45df84b45aefee347a6c1f89a05cd7fe68 Parents: 1702dfa Author: Botong Huang Authored: Thu Feb 1 14:43:48 2018 -0800 Committer: Botong Huang Committed: Thu Feb 1 14:43:48 2018 -0800 -- .../dev-support/findbugs-exclude.xml| 5 + .../hadoop/yarn/conf/YarnConfiguration.java | 18 +++ .../src/main/resources/yarn-default.xml | 24 .../store/impl/MemoryFederationStateStore.java | 13 ++ .../utils/FederationStateStoreFacade.java | 41 ++- .../GlobalPolicyGenerator.java | 92 ++- .../subclustercleaner/SubClusterCleaner.java| 109 + .../subclustercleaner/package-info.java | 19 +++ .../TestSubClusterCleaner.java | 118 +++ 9 files changed, 409 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd03fd45/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 6a10312..d4ab8f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -380,6 +380,11 @@ + + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd03fd45/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 271b666..eabe413 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3160,6 +3160,24 @@ public class YarnConfiguration extends Configuration { public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED = false; + private static final String FEDERATION_GPG_PREFIX = + FEDERATION_PREFIX + "gpg."; + + // The number of threads to use for the GPG scheduled executor service + public static final String GPG_SCHEDULED_EXECUTOR_THREADS = + FEDERATION_GPG_PREFIX + "scheduled.executor.threads"; + public static final int DEFAULT_GPG_SCHEDULED_EXECUTOR_THREADS = 10; + + // The interval at which the subcluster cleaner runs, -1 means disabled + public static final String GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = + FEDERATION_GPG_PREFIX + "subcluster.cleaner.interval-ms"; + public static final long DEFAULT_GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = -1; + + // The expiration time for a subcluster heartbeat, default is 30 minutes + public static final String GPG_SUBCLUSTER_EXPIRATION_MS = + FEDERATION_GPG_PREFIX + "subcluster.heartbeat.expiration-ms"; + public static final long DEFAULT_GPG_SUBCLUSTER_EXPIRATION_MS = 180; + // Other Configs http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd03fd45/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 017799a..899c210 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3403,6 +3403,30 @@ + The number of threads to use for the GPG scheduled executor service. + +yarn.federation.gpg.scheduled.executor.threads +10 + + + + + The interval at which the subcluster cleaner runs, -1 means disabled. + +yarn.federation.gpg.subcluster.cleaner.interval-ms +-1 + + + + + The expiration time for a subcluster he