[2/2] hadoop git commit: YARN-6706. Refactor ContainerScheduler to make oversubscription change easier. (Haibo Chen via asuresh)
YARN-6706. Refactor ContainerScheduler to make oversubscription change easier.
(Haibo Chen via asuresh)
(cherry picked from commit 5b007921cdf01ecc8ed97c164b7d327b8304c529)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7803a93f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7803a93f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7803a93f
Branch: refs/heads/branch-2
Commit: 7803a93fff80e923b67261cc44814eaf8f505fb3
Parents: ac20d7e
Author: Arun Suresh
Authored: Mon Jul 17 14:07:23 2017 -0700
Committer: Arun Suresh
Committed: Tue Sep 12 16:33:05 2017 -0700
--
.../scheduler/ContainerScheduler.java | 134 +--
.../TestContainerManagerRecovery.java | 2 +-
.../TestContainerSchedulerQueuing.java | 85
3 files changed, 176 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7803a93f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 0177b15..b2ff7cd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -266,7 +266,9 @@ public class ContainerScheduler extends AbstractService
implements
// decrement only if it was a running container
Container completedContainer = runningContainers.remove(container
.getContainerId());
-if (completedContainer != null) {
+// only a running container releases resources upon completion
+boolean resourceReleased = completedContainer != null;
+if (resourceReleased) {
this.utilizationTracker.subtractContainerResource(container);
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
@@ -292,8 +294,7 @@ public class ContainerScheduler extends AbstractService
implements
boolean resourcesAvailable = true;
while (cIter.hasNext() && resourcesAvailable) {
Container container = cIter.next();
- if (this.utilizationTracker.hasResourcesAvailable(container)) {
-startAllocatedContainer(container);
+ if (tryStartContainer(container)) {
cIter.remove();
} else {
resourcesAvailable = false;
@@ -302,50 +303,95 @@ public class ContainerScheduler extends AbstractService
implements
return resourcesAvailable;
}
- @VisibleForTesting
- protected void scheduleContainer(Container container) {
-if (maxOppQueueLength <= 0) {
- startAllocatedContainer(container);
- return;
+ private boolean tryStartContainer(Container container) {
+boolean containerStarted = false;
+if (resourceAvailableToStartContainer(container)) {
+ startContainer(container);
+ containerStarted = true;
}
-if (queuedGuaranteedContainers.isEmpty() &&
-queuedOpportunisticContainers.isEmpty() &&
-this.utilizationTracker.hasResourcesAvailable(container)) {
- startAllocatedContainer(container);
+return containerStarted;
+ }
+
+ /**
+ * Check if there is resource available to start a given container
+ * immediately. (This can be extended to include overallocated resources)
+ * @param container the container to start
+ * @return true if container can be launched directly
+ */
+ private boolean resourceAvailableToStartContainer(Container container) {
+return this.utilizationTracker.hasResourcesAvailable(container);
+ }
+
+ private boolean enqueueContainer(Container container) {
+boolean isGuaranteedContainer = container.getContainerTokenIdentifier().
+getExecutionType() == ExecutionType.GUARANTEED;
+
+boolean isQueued;
+if (isGuaranteedContainer) {
+ queuedGuaranteedContainers.put(container.getContainerId(), container);
+ isQueued = true;
} else {
- LOG.info("No available resources for container {} to start its execution
"
- + "immediately.", container.getContainerId());
- bo
[29/50] [abbrv] hadoop git commit: YARN-6706. Refactor ContainerScheduler to make oversubscription change easier. (Haibo Chen via asuresh)
YARN-6706. Refactor ContainerScheduler to make oversubscription change easier.
(Haibo Chen via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b007921
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b007921
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b007921
Branch: refs/heads/HDFS-7240
Commit: 5b007921cdf01ecc8ed97c164b7d327b8304c529
Parents: ed27f2b
Author: Arun Suresh
Authored: Mon Jul 17 14:07:23 2017 -0700
Committer: Arun Suresh
Committed: Mon Jul 17 14:11:14 2017 -0700
--
.../scheduler/ContainerScheduler.java | 135 +--
.../TestContainerManagerRecovery.java | 2 +-
.../TestContainerSchedulerQueuing.java | 85
3 files changed, 177 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b007921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 24530b3..19243ac 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -192,7 +192,9 @@ public class ContainerScheduler extends AbstractService
implements
// decrement only if it was a running container
Container completedContainer = runningContainers.remove(container
.getContainerId());
-if (completedContainer != null) {
+// only a running container releases resources upon completion
+boolean resourceReleased = completedContainer != null;
+if (resourceReleased) {
this.utilizationTracker.subtractContainerResource(container);
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
@@ -218,8 +220,7 @@ public class ContainerScheduler extends AbstractService
implements
boolean resourcesAvailable = true;
while (cIter.hasNext() && resourcesAvailable) {
Container container = cIter.next();
- if (this.utilizationTracker.hasResourcesAvailable(container)) {
-startAllocatedContainer(container);
+ if (tryStartContainer(container)) {
cIter.remove();
} else {
resourcesAvailable = false;
@@ -228,50 +229,95 @@ public class ContainerScheduler extends AbstractService
implements
return resourcesAvailable;
}
- @VisibleForTesting
- protected void scheduleContainer(Container container) {
-if (maxOppQueueLength <= 0) {
- startAllocatedContainer(container);
- return;
+ private boolean tryStartContainer(Container container) {
+boolean containerStarted = false;
+if (resourceAvailableToStartContainer(container)) {
+ startContainer(container);
+ containerStarted = true;
}
-if (queuedGuaranteedContainers.isEmpty() &&
-queuedOpportunisticContainers.isEmpty() &&
-this.utilizationTracker.hasResourcesAvailable(container)) {
- startAllocatedContainer(container);
+return containerStarted;
+ }
+
+ /**
+ * Check if there is resource available to start a given container
+ * immediately. (This can be extended to include overallocated resources)
+ * @param container the container to start
+ * @return true if container can be launched directly
+ */
+ private boolean resourceAvailableToStartContainer(Container container) {
+return this.utilizationTracker.hasResourcesAvailable(container);
+ }
+
+ private boolean enqueueContainer(Container container) {
+boolean isGuaranteedContainer = container.getContainerTokenIdentifier().
+getExecutionType() == ExecutionType.GUARANTEED;
+
+boolean isQueued;
+if (isGuaranteedContainer) {
+ queuedGuaranteedContainers.put(container.getContainerId(), container);
+ isQueued = true;
} else {
- LOG.info("No available resources for container {} to start its execution
"
- + "immediately.", container.getContainerId());
- boolean isQueued = true;
- if (container.getContainerTokenIdentifi
hadoop git commit: YARN-6706. Refactor ContainerScheduler to make oversubscription change easier. (Haibo Chen via asuresh)
Repository: hadoop
Updated Branches:
refs/heads/YARN-1011 8e458246c -> 4c501b46d
YARN-6706. Refactor ContainerScheduler to make oversubscription change easier.
(Haibo Chen via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c501b46
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c501b46
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c501b46
Branch: refs/heads/YARN-1011
Commit: 4c501b46dd3674bdd2471a431014bc6d12ffd703
Parents: 8e45824
Author: Arun Suresh
Authored: Mon Jul 17 14:07:23 2017 -0700
Committer: Haibo Chen
Committed: Mon Jul 17 14:25:51 2017 -0700
--
.../scheduler/ContainerScheduler.java | 135 +--
.../TestContainerManagerRecovery.java | 2 +-
.../TestContainerSchedulerQueuing.java | 85
3 files changed, 177 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c501b46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 24530b3..19243ac 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -192,7 +192,9 @@ public class ContainerScheduler extends AbstractService
implements
// decrement only if it was a running container
Container completedContainer = runningContainers.remove(container
.getContainerId());
-if (completedContainer != null) {
+// only a running container releases resources upon completion
+boolean resourceReleased = completedContainer != null;
+if (resourceReleased) {
this.utilizationTracker.subtractContainerResource(container);
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
@@ -218,8 +220,7 @@ public class ContainerScheduler extends AbstractService
implements
boolean resourcesAvailable = true;
while (cIter.hasNext() && resourcesAvailable) {
Container container = cIter.next();
- if (this.utilizationTracker.hasResourcesAvailable(container)) {
-startAllocatedContainer(container);
+ if (tryStartContainer(container)) {
cIter.remove();
} else {
resourcesAvailable = false;
@@ -228,50 +229,95 @@ public class ContainerScheduler extends AbstractService
implements
return resourcesAvailable;
}
- @VisibleForTesting
- protected void scheduleContainer(Container container) {
-if (maxOppQueueLength <= 0) {
- startAllocatedContainer(container);
- return;
+ private boolean tryStartContainer(Container container) {
+boolean containerStarted = false;
+if (resourceAvailableToStartContainer(container)) {
+ startContainer(container);
+ containerStarted = true;
}
-if (queuedGuaranteedContainers.isEmpty() &&
-queuedOpportunisticContainers.isEmpty() &&
-this.utilizationTracker.hasResourcesAvailable(container)) {
- startAllocatedContainer(container);
+return containerStarted;
+ }
+
+ /**
+ * Check if there is resource available to start a given container
+ * immediately. (This can be extended to include overallocated resources)
+ * @param container the container to start
+ * @return true if container can be launched directly
+ */
+ private boolean resourceAvailableToStartContainer(Container container) {
+return this.utilizationTracker.hasResourcesAvailable(container);
+ }
+
+ private boolean enqueueContainer(Container container) {
+boolean isGuaranteedContainer = container.getContainerTokenIdentifier().
+getExecutionType() == ExecutionType.GUARANTEED;
+
+boolean isQueued;
+if (isGuaranteedContainer) {
+ queuedGuaranteedContainers.put(container.getContainerId(), container);
+ isQueued = true;
} else {
- LOG.info("No available resources for container {} to start its execution
"
- + "immediately.", container.getContainerI
hadoop git commit: YARN-6706. Refactor ContainerScheduler to make oversubscription change easier. (Haibo Chen via asuresh)
Repository: hadoop
Updated Branches:
refs/heads/trunk ed27f2b2c -> 5b007921c
YARN-6706. Refactor ContainerScheduler to make oversubscription change easier.
(Haibo Chen via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b007921
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b007921
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b007921
Branch: refs/heads/trunk
Commit: 5b007921cdf01ecc8ed97c164b7d327b8304c529
Parents: ed27f2b
Author: Arun Suresh
Authored: Mon Jul 17 14:07:23 2017 -0700
Committer: Arun Suresh
Committed: Mon Jul 17 14:11:14 2017 -0700
--
.../scheduler/ContainerScheduler.java | 135 +--
.../TestContainerManagerRecovery.java | 2 +-
.../TestContainerSchedulerQueuing.java | 85
3 files changed, 177 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b007921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 24530b3..19243ac 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -192,7 +192,9 @@ public class ContainerScheduler extends AbstractService
implements
// decrement only if it was a running container
Container completedContainer = runningContainers.remove(container
.getContainerId());
-if (completedContainer != null) {
+// only a running container releases resources upon completion
+boolean resourceReleased = completedContainer != null;
+if (resourceReleased) {
this.utilizationTracker.subtractContainerResource(container);
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
@@ -218,8 +220,7 @@ public class ContainerScheduler extends AbstractService
implements
boolean resourcesAvailable = true;
while (cIter.hasNext() && resourcesAvailable) {
Container container = cIter.next();
- if (this.utilizationTracker.hasResourcesAvailable(container)) {
-startAllocatedContainer(container);
+ if (tryStartContainer(container)) {
cIter.remove();
} else {
resourcesAvailable = false;
@@ -228,50 +229,95 @@ public class ContainerScheduler extends AbstractService
implements
return resourcesAvailable;
}
- @VisibleForTesting
- protected void scheduleContainer(Container container) {
-if (maxOppQueueLength <= 0) {
- startAllocatedContainer(container);
- return;
+ private boolean tryStartContainer(Container container) {
+boolean containerStarted = false;
+if (resourceAvailableToStartContainer(container)) {
+ startContainer(container);
+ containerStarted = true;
}
-if (queuedGuaranteedContainers.isEmpty() &&
-queuedOpportunisticContainers.isEmpty() &&
-this.utilizationTracker.hasResourcesAvailable(container)) {
- startAllocatedContainer(container);
+return containerStarted;
+ }
+
+ /**
+ * Check if there is resource available to start a given container
+ * immediately. (This can be extended to include overallocated resources)
+ * @param container the container to start
+ * @return true if container can be launched directly
+ */
+ private boolean resourceAvailableToStartContainer(Container container) {
+return this.utilizationTracker.hasResourcesAvailable(container);
+ }
+
+ private boolean enqueueContainer(Container container) {
+boolean isGuaranteedContainer = container.getContainerTokenIdentifier().
+getExecutionType() == ExecutionType.GUARANTEED;
+
+boolean isQueued;
+if (isGuaranteedContainer) {
+ queuedGuaranteedContainers.put(container.getContainerId(), container);
+ isQueued = true;
} else {
- LOG.info("No available resources for container {} to start its execution
"
- + "immediately.", container.getContainerId());
-
