git commit: YARN-2531. Added a configuration for admins to be able to override app-configs and enforce/not-enforce strict control of per-container cpu usage. Contributed by Varun Vasudev.
Repository: hadoop Updated Branches: refs/heads/trunk 0c26412be -> 9f6891d9e YARN-2531. Added a configuration for admins to be able to override app-configs and enforce/not-enforce strict control of per-container cpu usage. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f6891d9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f6891d9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f6891d9 Branch: refs/heads/trunk Commit: 9f6891d9ef7064d121305ca783eb62586c8aa018 Parents: 0c26412 Author: Vinod Kumar Vavilapalli Authored: Tue Sep 16 10:14:46 2014 -0700 Committer: Vinod Kumar Vavilapalli Committed: Tue Sep 16 10:14:46 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 4 + .../hadoop/yarn/conf/YarnConfiguration.java | 10 ++ .../src/main/resources/yarn-default.xml | 10 ++ .../util/CgroupsLCEResourcesHandler.java| 29 +++- .../util/TestCgroupsLCEResourcesHandler.java| 165 --- 5 files changed, 189 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6891d9/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b41ad82..66623ec 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -84,6 +84,10 @@ Release 2.6.0 - UNRELEASED failures should be ignored towards counting max-attempts. (Xuan Gong via vinodkv) +YARN-2531. Added a configuration for admins to be able to override app-configs +and enforce/not-enforce strict control of per-container cpu usage. (Varun +Vasudev via vinodkv) + IMPROVEMENTS YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6891d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a92b358..44a6fc3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -902,6 +902,16 @@ public class YarnConfiguration extends Configuration { public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH = NM_PREFIX + "linux-container-executor.cgroups.mount-path"; + /** + * Whether the apps should run in strict resource usage mode(not allowed to + * use spare CPU) + */ + public static final String NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = + NM_PREFIX + "linux-container-executor.cgroups.strict-resource-usage"; + public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = + false; + + /** * Interval of time the linux container executor should try cleaning up http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6891d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 55e5bca..3a7e94a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1040,6 +1040,16 @@ +This flag determines whether apps should run with strict resource limits +or be allowed to consume spare resources if they need them. For example, turning the +flag on will restrict apps to use only their share of CPU, even if the node has spare +CPU cycles. The default value is false i.e. use available resources. Please note that +turning this flag on may reduce job throughput on the cluster. + yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage +false + + + T-file compression types used to compress aggregated logs. yarn.nodemanager.log-aggregation.compression-type none http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6891d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/Cgr
git commit: YARN-2531. Added a configuration for admins to be able to override app-configs and enforce/not-enforce strict control of per-container cpu usage. Contributed by Varun Vasudev.
Repository: hadoop Updated Branches: refs/heads/branch-2 08abb320a -> 9d34dc87e YARN-2531. Added a configuration for admins to be able to override app-configs and enforce/not-enforce strict control of per-container cpu usage. Contributed by Varun Vasudev. (cherry picked from commit 9f6891d9ef7064d121305ca783eb62586c8aa018) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d34dc87 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d34dc87 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d34dc87 Branch: refs/heads/branch-2 Commit: 9d34dc87e1965625552e6bae81ad721fbee81338 Parents: 08abb32 Author: Vinod Kumar Vavilapalli Authored: Tue Sep 16 10:14:46 2014 -0700 Committer: Vinod Kumar Vavilapalli Committed: Tue Sep 16 10:15:37 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 4 + .../hadoop/yarn/conf/YarnConfiguration.java | 10 ++ .../src/main/resources/yarn-default.xml | 10 ++ .../util/CgroupsLCEResourcesHandler.java| 29 +++- .../util/TestCgroupsLCEResourcesHandler.java| 165 --- 5 files changed, 189 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d34dc87/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index da90f27..295bb9e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -57,6 +57,10 @@ Release 2.6.0 - UNRELEASED failures should be ignored towards counting max-attempts. (Xuan Gong via vinodkv) +YARN-2531. Added a configuration for admins to be able to override app-configs +and enforce/not-enforce strict control of per-container cpu usage. (Varun +Vasudev via vinodkv) + IMPROVEMENTS YARN-2242. Improve exception information on AM launch crashes. (Li Lu http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d34dc87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 9e88c97..f145f69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -902,6 +902,16 @@ public class YarnConfiguration extends Configuration { public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH = NM_PREFIX + "linux-container-executor.cgroups.mount-path"; + /** + * Whether the apps should run in strict resource usage mode(not allowed to + * use spare CPU) + */ + public static final String NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = + NM_PREFIX + "linux-container-executor.cgroups.strict-resource-usage"; + public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = + false; + + /** * Interval of time the linux container executor should try cleaning up http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d34dc87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index cfc00f5..6635c93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1039,6 +1039,16 @@ +This flag determines whether apps should run with strict resource limits +or be allowed to consume spare resources if they need them. For example, turning the +flag on will restrict apps to use only their share of CPU, even if the node has spare +CPU cycles. The default value is false i.e. use available resources. Please note that +turning this flag on may reduce job throughput on the cluster. + yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage +false + + + T-file compression types used to compress aggregated logs. yarn.nodemanager.log-aggregation.compression-type none http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d34dc87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-se
[06/10] git commit: YARN-2531. Added a configuration for admins to be able to override app-configs and enforce/not-enforce strict control of per-container cpu usage. Contributed by Varun Vasudev.
YARN-2531. Added a configuration for admins to be able to override app-configs and enforce/not-enforce strict control of per-container cpu usage. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f6891d9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f6891d9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f6891d9 Branch: refs/heads/HDFS-6581 Commit: 9f6891d9ef7064d121305ca783eb62586c8aa018 Parents: 0c26412 Author: Vinod Kumar Vavilapalli Authored: Tue Sep 16 10:14:46 2014 -0700 Committer: Vinod Kumar Vavilapalli Committed: Tue Sep 16 10:14:46 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 4 + .../hadoop/yarn/conf/YarnConfiguration.java | 10 ++ .../src/main/resources/yarn-default.xml | 10 ++ .../util/CgroupsLCEResourcesHandler.java| 29 +++- .../util/TestCgroupsLCEResourcesHandler.java| 165 --- 5 files changed, 189 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6891d9/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b41ad82..66623ec 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -84,6 +84,10 @@ Release 2.6.0 - UNRELEASED failures should be ignored towards counting max-attempts. (Xuan Gong via vinodkv) +YARN-2531. Added a configuration for admins to be able to override app-configs +and enforce/not-enforce strict control of per-container cpu usage. (Varun +Vasudev via vinodkv) + IMPROVEMENTS YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6891d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a92b358..44a6fc3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -902,6 +902,16 @@ public class YarnConfiguration extends Configuration { public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH = NM_PREFIX + "linux-container-executor.cgroups.mount-path"; + /** + * Whether the apps should run in strict resource usage mode(not allowed to + * use spare CPU) + */ + public static final String NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = + NM_PREFIX + "linux-container-executor.cgroups.strict-resource-usage"; + public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = + false; + + /** * Interval of time the linux container executor should try cleaning up http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6891d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 55e5bca..3a7e94a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1040,6 +1040,16 @@ +This flag determines whether apps should run with strict resource limits +or be allowed to consume spare resources if they need them. For example, turning the +flag on will restrict apps to use only their share of CPU, even if the node has spare +CPU cycles. The default value is false i.e. use available resources. Please note that +turning this flag on may reduce job throughput on the cluster. + yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage +false + + + T-file compression types used to compress aggregated logs. yarn.nodemanager.log-aggregation.compression-type none http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6891d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
[12/22] git commit: YARN-2531. Added a configuration for admins to be able to override app-configs and enforce/not-enforce strict control of per-container cpu usage. Contributed by Varun Vasudev.
YARN-2531. Added a configuration for admins to be able to override app-configs and enforce/not-enforce strict control of per-container cpu usage. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f6891d9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f6891d9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f6891d9 Branch: refs/heads/HDFS-6584 Commit: 9f6891d9ef7064d121305ca783eb62586c8aa018 Parents: 0c26412 Author: Vinod Kumar Vavilapalli Authored: Tue Sep 16 10:14:46 2014 -0700 Committer: Vinod Kumar Vavilapalli Committed: Tue Sep 16 10:14:46 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 4 + .../hadoop/yarn/conf/YarnConfiguration.java | 10 ++ .../src/main/resources/yarn-default.xml | 10 ++ .../util/CgroupsLCEResourcesHandler.java| 29 +++- .../util/TestCgroupsLCEResourcesHandler.java| 165 --- 5 files changed, 189 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6891d9/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b41ad82..66623ec 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -84,6 +84,10 @@ Release 2.6.0 - UNRELEASED failures should be ignored towards counting max-attempts. (Xuan Gong via vinodkv) +YARN-2531. Added a configuration for admins to be able to override app-configs +and enforce/not-enforce strict control of per-container cpu usage. (Varun +Vasudev via vinodkv) + IMPROVEMENTS YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6891d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a92b358..44a6fc3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -902,6 +902,16 @@ public class YarnConfiguration extends Configuration { public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH = NM_PREFIX + "linux-container-executor.cgroups.mount-path"; + /** + * Whether the apps should run in strict resource usage mode(not allowed to + * use spare CPU) + */ + public static final String NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = + NM_PREFIX + "linux-container-executor.cgroups.strict-resource-usage"; + public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = + false; + + /** * Interval of time the linux container executor should try cleaning up http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6891d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 55e5bca..3a7e94a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1040,6 +1040,16 @@ +This flag determines whether apps should run with strict resource limits +or be allowed to consume spare resources if they need them. For example, turning the +flag on will restrict apps to use only their share of CPU, even if the node has spare +CPU cycles. The default value is false i.e. use available resources. Please note that +turning this flag on may reduce job throughput on the cluster. + yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage +false + + + T-file compression types used to compress aggregated logs. yarn.nodemanager.log-aggregation.compression-type none http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f6891d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java