[hadoop] branch branch-2.10 updated: YARN-8222. Fix potential NPE when gets RMApp from RM context. Contributed by Tao Yang.
This is an automated email from the ASF dual-hosted git repository.
epayne pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new 5f2047d YARN-8222. Fix potential NPE when gets RMApp from RM context.
Contributed by Tao Yang.
5f2047d is described below
commit 5f2047d491f3e168326257cc63e2cb5b0ddbcd6a
Author: Weiwei Yang
AuthorDate: Wed May 2 17:54:46 2018 +0800
YARN-8222. Fix potential NPE when gets RMApp from RM context. Contributed
by Tao Yang.
(cherry picked from commit 251f528814c4a4647cac0af6effb9a73135db180)
---
.../rmcontainer/RMContainerImpl.java | 30 --
.../scheduler/SchedulerApplicationAttempt.java | 13 +-
2 files changed, 23 insertions(+), 20 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index 76933f1..3c85df9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
@@ -709,21 +710,22 @@ public class RMContainerImpl implements RMContainer {
private static void updateAttemptMetrics(RMContainerImpl container) {
// If this is a preempted container, update preemption metrics
Resource resource = container.getContainer().getResource();
- RMAppAttempt rmAttempt = container.rmContext.getRMApps()
- .get(container.getApplicationAttemptId().getApplicationId())
- .getCurrentAppAttempt();
-
- if (rmAttempt != null) {
-long usedMillis = container.finishTime - container.creationTime;
-rmAttempt.getRMAppAttemptMetrics()
-.updateAggregateAppResourceUsage(resource, usedMillis);
-// If this is a preempted container, update preemption metrics
-if (ContainerExitStatus.PREEMPTED == container.finishedStatus
-.getExitStatus()) {
+ RMApp app = container.rmContext.getRMApps()
+ .get(container.getApplicationAttemptId().getApplicationId());
+ if (app != null) {
+RMAppAttempt rmAttempt = app.getCurrentAppAttempt();
+if (rmAttempt != null) {
+ long usedMillis = container.finishTime - container.creationTime;
rmAttempt.getRMAppAttemptMetrics()
- .updatePreemptionInfo(resource, container);
- rmAttempt.getRMAppAttemptMetrics()
- .updateAggregatePreemptedAppResourceUsage(resource, usedMillis);
+ .updateAggregateAppResourceUsage(resource, usedMillis);
+ // If this is a preempted container, update preemption metrics
+ if (ContainerExitStatus.PREEMPTED == container.finishedStatus
+ .getExitStatus()) {
+rmAttempt.getRMAppAttemptMetrics()
+.updatePreemptionInfo(resource, container);
+rmAttempt.getRMAppAttemptMetrics()
+.updateAggregatePreemptedAppResourceUsage(resource,
usedMillis);
+ }
}
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 5e85047..0b01024 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++
[hadoop] branch branch-3.3 updated: YARN-10869. CS considers only the default maximum-allocation-mb/vcore property as a maximum when it creates dynamic queues (#3504)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 7000458 YARN-10869. CS considers only the default
maximum-allocation-mb/vcore property as a maximum when it creates dynamic
queues (#3504)
7000458 is described below
commit 700045896c6175913d9071c132b533871dcbc20e
Author: Benjamin Teke
AuthorDate: Tue Oct 12 18:05:50 2021 +0200
YARN-10869. CS considers only the default maximum-allocation-mb/vcore
property as a maximum when it creates dynamic queues (#3504)
Co-authored-by: Benjamin Teke
---
.../scheduler/capacity/AbstractCSQueue.java| 50 --
.../capacity/CapacitySchedulerConfiguration.java | 14 ++
.../TestCapacitySchedulerAutoCreatedQueueBase.java | 32 --
.../TestCapacitySchedulerAutoQueueCreation.java| 33 +-
4 files changed, 102 insertions(+), 27 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index fda5e16..e7d9fa5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -100,7 +100,7 @@ public abstract class AbstractCSQueue implements CSQueue {
String defaultLabelExpression;
private String multiNodeSortingPolicyName = null;
- Map acls =
+ Map acls =
new HashMap();
volatile boolean reservationsContinueLooking;
private volatile boolean preemptionDisabled;
@@ -112,7 +112,7 @@ public abstract class AbstractCSQueue implements CSQueue {
volatile ResourceUsage queueUsage;
private final boolean fullPathQueueNamingPolicy = false;
-
+
// Track capacities like
used-capcity/abs-used-capacity/capacity/abs-capacity,
// etc.
QueueCapacities queueCapacities;
@@ -134,7 +134,7 @@ public abstract class AbstractCSQueue implements CSQueue {
protected CapacityConfigType capacityConfigType =
CapacityConfigType.NONE;
- private final RecordFactory recordFactory =
+ private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
protected CapacitySchedulerContext csContext;
protected YarnAuthorizationProvider authorizer = null;
@@ -250,12 +250,12 @@ public abstract class AbstractCSQueue implements CSQueue {
public QueueState getState() {
return state;
}
-
+
@Override
public CSQueueMetrics getMetrics() {
return metrics;
}
-
+
@Override
public String getQueueShortName() {
return queueName;
@@ -283,7 +283,7 @@ public abstract class AbstractCSQueue implements CSQueue {
public void setParent(CSQueue newParentQueue) {
this.parent = newParentQueue;
}
-
+
public Set getAccessibleNodeLabels() {
return accessibleLabels;
}
@@ -344,7 +344,7 @@ public abstract class AbstractCSQueue implements CSQueue {
public String getDefaultNodeLabelExpression() {
return defaultLabelExpression;
}
-
+
void setupQueueConfigs(Resource clusterResource)
throws IOException {
setupQueueConfigs(clusterResource, csContext.getConfiguration());
@@ -471,8 +471,14 @@ public abstract class AbstractCSQueue implements CSQueue {
private void setupMaximumAllocation(CapacitySchedulerConfiguration csConf) {
String myQueuePath = getQueuePath();
+/* YARN-10869: When using AutoCreatedLeafQueues, the passed configuration
+* object is a cloned one containing only the template configs
+* (see ManagedParentQueue#getLeafQueueConfigs). To ensure that the actual
+* cluster maximum allocation is fetched the original config object should
+* be used.
+*/
Resource clusterMax = ResourceUtils
-.fetchMaximumAllocationFromConfig(csConf);
+.fetchMaximumAllocationFromConfig(this.csContext.getConfiguration());
Resource queueMax = csConf.getQueueMaximumAllocation(myQueuePath);
maximumAllocation = Resources.clone(
@@ -740,7 +746,7 @@ public abstract class AbstractCSQueue implements CSQueue {
stats.setReservedContainers(getMetrics().getReservedContainers());
return stats;
}
-
+
public Map getQueueConfigurations() {
Map queueConfigurations = new HashMap<>();
Set nodeLabels = getNodeLabelsForQueue();
@@ -776,12 +782,12 @@ public abstract class
[hadoop] branch branch-2.10 updated: HADOOP-17964. Increase Java heap size for running Maven in Dockerfile of branch-2.10. (#3545)
This is an automated email from the ASF dual-hosted git repository. iwasakims pushed a commit to branch branch-2.10 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-2.10 by this push: new b8dc05e HADOOP-17964. Increase Java heap size for running Maven in Dockerfile of branch-2.10. (#3545) b8dc05e is described below commit b8dc05efcd1450f94c930fa6c1c794e0e2421037 Author: Masatake Iwasaki AuthorDate: Wed Oct 13 00:53:03 2021 +0900 HADOOP-17964. Increase Java heap size for running Maven in Dockerfile of branch-2.10. (#3545) --- dev-support/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 24c36f0..cf163d9e 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -203,7 +203,7 @@ RUN curl -L -s -S \ ### # Avoid out of memory errors in builds ### -ENV MAVEN_OPTS -Xms256m -Xmx1536m -Dhttps.protocols=TLSv1.2 -Dhttps.cipherSuites=TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 +ENV MAVEN_OPTS -Xms256m -Xmx2048m -XX:MaxPermSize=512m -Dhttps.protocols=TLSv1.2 -Dhttps.cipherSuites=TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 ### - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8c271ca YARN-6862. Nodemanager resource usage metrics sometimes are
negative. Contributed by Benjamin Teke
8c271ca is described below
commit 8c271caa81db683fe4b92853bd8ca5da060e6593
Author: Szilard Nemeth
AuthorDate: Tue Oct 12 17:32:40 2021 +0200
YARN-6862. Nodemanager resource usage metrics sometimes are negative.
Contributed by Benjamin Teke
---
.../monitor/ContainersMonitorImpl.java | 8
.../MockCPUResourceCalculatorProcessTree.java | 10 +
...> MockMemoryResourceCalculatorProcessTree.java} | 47 +++---
.../monitor/MockResourceCalculatorProcessTree.java | 6 +++
.../TestContainersMonitorResourceChange.java | 24 ---
5 files changed, 75 insertions(+), 20 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index a83ae3a..deccded 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -537,6 +537,14 @@ public class ContainersMonitorImpl extends AbstractService
implements
pTree.updateProcessTree();// update process-tree
long currentVmemUsage = pTree.getVirtualMemorySize();
long currentPmemUsage = pTree.getRssMemorySize();
+if (currentVmemUsage < 0 || currentPmemUsage < 0) {
+ // YARN-6862/YARN-5021 If the container just exited or for
+ // another reason the physical/virtual memory is UNAVAILABLE (-1)
+ // the values shouldn't be aggregated.
+ LOG.info("Skipping monitoring container {} because "
+ + "memory usage is not available.", containerId);
+ continue;
+}
// if machine has 6 cores and 3 are used,
// cpuUsagePercentPerCore should be 300%
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
index eb35c91..49161f3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
@@ -57,6 +57,16 @@ public class MockCPUResourceCalculatorProcessTree
}
@Override
+ public long getVirtualMemorySize(int olderThanAge) {
+return 0;
+ }
+
+ @Override
+ public long getRssMemorySize(int olderThanAge) {
+return 0;
+ }
+
+ @Override
public float getCpuUsagePercent() {
long cpu = this.cpuPercentage;
// First getter call will be returned with -1, and other calls will
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java
similarity index 50%
copy from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
copy to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java
index eb35c91..ea45ac4 100644
---
