Repository: hadoop Updated Branches: refs/heads/branch-2 f7ac0fd80 -> 3d401206c
YARN-5483. Optimize RMAppAttempt#pullJustFinishedContainers. Contributed by sandflee Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d401206 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d401206 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d401206 Branch: refs/heads/branch-2 Commit: 3d401206cf1eca5360f7f53165732a73d644a1fb Parents: f7ac0fd Author: Jason Lowe <jl...@apache.org> Authored: Wed Aug 10 18:06:56 2016 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Wed Aug 10 18:06:56 2016 +0000 ---------------------------------------------------------------------- .../rmapp/attempt/RMAppAttemptImpl.java | 38 ++++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d401206/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 9ec0f82..bcc840c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -28,6 +28,7 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -144,14 +145,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { private SecretKey clientTokenMasterKey = null; private ConcurrentMap<NodeId, List<ContainerStatus>> - justFinishedContainers = - new ConcurrentHashMap<NodeId, List<ContainerStatus>>(); + justFinishedContainers = new ConcurrentHashMap<>(); // Tracks the previous finished containers that are waiting to be // verified as received by the AM. If the AM sends the next allocate // request it implicitly acks this list. private ConcurrentMap<NodeId, List<ContainerStatus>> - finishedContainersSentToAM = - new ConcurrentHashMap<NodeId, List<ContainerStatus>>(); + finishedContainersSentToAM = new ConcurrentHashMap<>(); private volatile Container masterContainer; private float progress = 0; @@ -759,7 +758,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { public List<ContainerStatus> getJustFinishedContainers() { this.readLock.lock(); try { - List<ContainerStatus> returnList = new ArrayList<ContainerStatus>(); + List<ContainerStatus> returnList = new ArrayList<>(); for (Collection<ContainerStatus> containerStatusList : justFinishedContainers.values()) { returnList.addAll(containerStatusList); @@ -798,7 +797,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { this.writeLock.lock(); try { - List<ContainerStatus> returnList = new ArrayList<ContainerStatus>(); + List<ContainerStatus> returnList = new ArrayList<>(); // A new allocate means the AM received the previously sent // finishedContainers. We can ack this to NM now @@ -806,15 +805,17 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { // Mark every containerStatus as being sent to AM though we may return // only the ones that belong to the current attempt - boolean keepContainersAcressAttempts = this.submissionContext + boolean keepContainersAcrossAppAttempts = this.submissionContext .getKeepContainersAcrossApplicationAttempts(); - for (NodeId nodeId:justFinishedContainers.keySet()) { - - // Clear and get current values - List<ContainerStatus> finishedContainers = justFinishedContainers.put - (nodeId, new ArrayList<ContainerStatus>()); + for (Map.Entry<NodeId, List<ContainerStatus>> entry : + justFinishedContainers.entrySet()) { + NodeId nodeId = entry.getKey(); + List<ContainerStatus> finishedContainers = entry.getValue(); + if (finishedContainers.isEmpty()) { + continue; + } - if (keepContainersAcressAttempts) { + if (keepContainersAcrossAppAttempts) { returnList.addAll(finishedContainers); } else { // Filter out containers from previous attempt @@ -826,12 +827,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } } - if (!finishedContainers.isEmpty()) { - finishedContainersSentToAM.putIfAbsent(nodeId, - new ArrayList<ContainerStatus>()); - finishedContainersSentToAM.get(nodeId).addAll(finishedContainers); - } + finishedContainersSentToAM.putIfAbsent(nodeId, + new ArrayList<ContainerStatus>()); + finishedContainersSentToAM.get(nodeId).addAll(finishedContainers); } + justFinishedContainers.clear(); return returnList; } finally { @@ -1866,7 +1866,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { finishedContainersSentToAM.put(nodeId, new ArrayList<ContainerStatus>()); List<ContainerId> containerIdList = - new ArrayList<ContainerId>(currentSentContainers.size()); + new ArrayList<>(currentSentContainers.size()); for (ContainerStatus containerStatus : currentSentContainers) { containerIdList.add(containerStatus.getContainerId()); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
