hadoop git commit: YARN-7617. Add a flag in distributed shell to automatically PROMOTE opportunistic containers to guaranteed once they are started. Contributed by Weiwei Yang.
Repository: hadoop
Updated Branches:
refs/heads/trunk 5e81f32d1 -> 928964102
YARN-7617. Add a flag in distributed shell to automatically PROMOTE
opportunistic containers to guaranteed once they are started. Contributed by
Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92896410
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92896410
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92896410
Branch: refs/heads/trunk
Commit: 928964102029e96406f5482e8900802f38164501
Parents: 5e81f32
Author: Weiwei Yang
Authored: Mon Dec 18 10:07:16 2017 +0800
Committer: Weiwei Yang
Committed: Mon Dec 18 10:07:16 2017 +0800
--
.../distributedshell/ApplicationMaster.java | 49 +++-
.../applications/distributedshell/Client.java | 11 +
.../site/markdown/OpportunisticContainers.md.vm | 2 +-
3 files changed, 59 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92896410/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 926de50..b3fa0ff 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -93,6 +93,8 @@ import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.UpdatedContainer;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
@@ -247,6 +249,8 @@ public class ApplicationMaster {
// Execution type of the containers.
// Default GUARANTEED.
private ExecutionType containerType = ExecutionType.GUARANTEED;
+ // Whether to automatically promote opportunistic containers.
+ private boolean autoPromoteContainers = false;
// Resource profile for the container
private String containerResourceProfile = "";
@@ -420,6 +424,9 @@ public class ApplicationMaster {
"Environment for shell script. Specified as env_key=env_val pairs");
opts.addOption("container_type", true,
"Container execution type, GUARANTEED or OPPORTUNISTIC");
+opts.addOption("promote_opportunistic_after_start", false,
+"Flag to indicate whether to automatically promote opportunistic"
++ " containers to guaranteed.");
opts.addOption("container_memory", true,
"Amount of memory in MB to be requested to run the shell command");
opts.addOption("container_vcores", true,
@@ -576,6 +583,9 @@ public class ApplicationMaster {
}
containerType = ExecutionType.valueOf(containerTypeStr);
}
+if (cliParser.hasOption("promote_opportunistic_after_start")) {
+ autoPromoteContainers = true;
+}
containerMemory = Integer.parseInt(cliParser.getOptionValue(
"container_memory", "-1"));
containerVirtualCores = Integer.parseInt(cliParser.getOptionValue(
@@ -977,7 +987,24 @@ public class ApplicationMaster {
@Override
public void onContainersUpdated(
-List containers) {}
+List containers) {
+ for (UpdatedContainer container : containers) {
+LOG.info("Container {} updated, updateType={}, resource={}, "
++ "execType={}",
+container.getContainer().getId(),
+container.getUpdateType().toString(),
+container.getContainer().getResource().toString(),
+container.getContainer().getExecutionType());
+
+// TODO Remove this line with finalized updateContainer API.
+// Currently nm client needs to notify the NM to update container
+// execution type via NMClient#updateContaine
hadoop git commit: YARN-7617. Add a flag in distributed shell to automatically PROMOTE opportunistic containers to guaranteed once they are started. Contributed by Weiwei Yang.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0 90636cf12 -> de2156dc9
YARN-7617. Add a flag in distributed shell to automatically PROMOTE
opportunistic containers to guaranteed once they are started. Contributed by
Weiwei Yang.
(cherry picked from commit 928964102029e96406f5482e8900802f38164501)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de2156dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de2156dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de2156dc
Branch: refs/heads/branch-3.0
Commit: de2156dc900b9a84e20e98edfd54ea5a19840511
Parents: 90636cf
Author: Weiwei Yang
Authored: Mon Dec 18 10:07:16 2017 +0800
Committer: Weiwei Yang
Committed: Mon Dec 18 10:27:17 2017 +0800
--
.../distributedshell/ApplicationMaster.java | 49 +++-
.../applications/distributedshell/Client.java | 13 +-
.../site/markdown/OpportunisticContainers.md.vm | 2 +-
3 files changed, 60 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de2156dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index b8a805b..1fa9f20 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -92,6 +92,8 @@ import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.UpdatedContainer;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
@@ -243,6 +245,8 @@ public class ApplicationMaster {
// Execution type of the containers.
// Default GUARANTEED.
private ExecutionType containerType = ExecutionType.GUARANTEED;
+ // Whether to automatically promote opportunistic containers.
+ private boolean autoPromoteContainers = false;
// Counter for completed containers ( complete denotes successful or failed )
private AtomicInteger numCompletedContainers = new AtomicInteger();
@@ -412,6 +416,9 @@ public class ApplicationMaster {
"Environment for shell script. Specified as env_key=env_val pairs");
opts.addOption("container_type", true,
"Container execution type, GUARANTEED or OPPORTUNISTIC");
+opts.addOption("promote_opportunistic_after_start", false,
+"Flag to indicate whether to automatically promote opportunistic"
++ " containers to guaranteed.");
opts.addOption("container_memory", true,
"Amount of memory in MB to be requested to run the shell command");
opts.addOption("container_vcores", true,
@@ -566,6 +573,9 @@ public class ApplicationMaster {
}
containerType = ExecutionType.valueOf(containerTypeStr);
}
+if (cliParser.hasOption("promote_opportunistic_after_start")) {
+ autoPromoteContainers = true;
+}
containerMemory = Integer.parseInt(cliParser.getOptionValue(
"container_memory", "10"));
containerVirtualCores = Integer.parseInt(cliParser.getOptionValue(
@@ -964,7 +974,24 @@ public class ApplicationMaster {
@Override
public void onContainersUpdated(
-List containers) {}
+List containers) {
+ for (UpdatedContainer container : containers) {
+LOG.info("Container {} updated, updateType={}, resource={}, "
++ "execType={}",
+container.getContainer().getId(),
+container.getUpdateType().toString(),
+container.getContainer().getResource().toString(),
+container.getContainer().getExecutionType());
+
+// TODO Remove this line with finalized updateC
