[hadoop] branch trunk updated: YARN-11050 (#3805)
This is an automated email from the ASF dual-hosted git repository.
zhuqi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 0714142 YARN-11050 (#3805)
0714142 is described below
commit 07141426e08e1586833018a8f367d27cd5bcb8e2
Author: Szilard Nemeth <954799+szilard-nem...@users.noreply.github.com>
AuthorDate: Sat Dec 18 04:52:03 2021 +0100
YARN-11050 (#3805)
---
.../apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java | 2 +-
.../yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java | 2 +-
.../hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java | 2 +-
.../yarn/server/router/webapp/DefaultRequestInterceptorREST.java | 2 +-
.../hadoop/yarn/server/router/webapp/FederationInterceptorREST.java | 2 +-
.../apache/hadoop/yarn/server/router/webapp/RouterWebServices.java| 4 ++--
.../hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java | 4 ++--
.../hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java | 2 +-
.../yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java | 4 ++--
.../hadoop/yarn/server/router/webapp/TestRouterWebServices.java | 2 +-
.../hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java | 4 ++--
11 files changed, 15 insertions(+), 15 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
index 82ceed3..791e34a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
@@ -126,7 +126,7 @@ public final class RMWSConsts {
/** Path for {@code RMWebServiceProtocol#addToClusterNodeLabels}. */
public static final String ADD_NODE_LABELS = "/add-node-labels";
- /** Path for {@code RMWebServiceProtocol#removeFromCluserNodeLabels}. */
+ /** Path for {@code RMWebServiceProtocol#removeFromClusterNodeLabels}. */
public static final String REMOVE_NODE_LABELS = "/remove-node-labels";
/** Path for {@code RMWebServiceProtocol#getLabelsOnNode}. */
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java
index f2736e3..41fc4ea 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java
@@ -398,7 +398,7 @@ public interface RMWebServiceProtocol {
* @return Response containing the status code
* @throws Exception in case of bad request
*/
- Response removeFromCluserNodeLabels(Set oldNodeLabels,
+ Response removeFromClusterNodeLabels(Set oldNodeLabels,
HttpServletRequest hsr) throws Exception;
/**
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 041b37c..1dac043 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -1435,7 +1435,7 @@ public class RMWebServices extends WebServices implements
RMWebServiceProtocol {
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Override
- public Response removeFromCluserNodeLabels(
+ public Response removeFromClusterNodeLabels(
@QueryParam(RMWSConsts.LABELS) Set oldNodeLabels,
@Context HttpServletRequest hsr) throws Exception {
UserGroupInformation
[hadoop] branch branch-3.2.3 updated: HADOOP-17534. Update Jackson to 2.10.5 and Jackson databind to 2.10.5.1 (#2708)
This is an automated email from the ASF dual-hosted git repository. aajisaka pushed a commit to branch branch-3.2.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2.3 by this push: new 2fac763 HADOOP-17534. Update Jackson to 2.10.5 and Jackson databind to 2.10.5.1 (#2708) 2fac763 is described below commit 2fac7630e2aca3c24e61cae3debf61a85a8c4e0b Author: Akira Ajisaka AuthorDate: Mon Feb 22 17:33:02 2021 +0900 HADOOP-17534. Update Jackson to 2.10.5 and Jackson databind to 2.10.5.1 (#2708) Reviewed-by: Wei-Chiu Chuang (cherry picked from commit e119063a078f06523158dba3b9e16ca04768e920) (cherry picked from commit 07d3d5ea0b1e56c2f6027536d4c4667dbb9c8512) --- hadoop-project/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 8a4dcc7..14dc864 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -69,8 +69,8 @@ 1.9.13 -2.10.3 -2.10.3 +2.10.5 +2.10.5.1 4.5.13 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2.3 updated: HADOOP-13500. Synchronizing iteration of Configuration properties object (#3775)
This is an automated email from the ASF dual-hosted git repository.
aajisaka pushed a commit to branch branch-3.2.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2.3 by this push:
new f2d12b3 HADOOP-13500. Synchronizing iteration of Configuration
properties object (#3775)
f2d12b3 is described below
commit f2d12b32b9384d979be565a05b6f5b5de093e82e
Author: Dhananjay Badaya <11253243+dbada...@users.noreply.github.com>
AuthorDate: Fri Dec 17 12:35:46 2021 +0530
HADOOP-13500. Synchronizing iteration of Configuration properties object
(#3775)
Signed-off-by: Akira Ajisaka
(cherry picked from commit 4483607a4edc0f13264f4ea37abd90aba97e1ef0)
(cherry picked from commit 253b217efa1650bbd6da35b0eeeb3d7504171ecb)
---
.../java/org/apache/hadoop/conf/Configuration.java | 10 +---
.../org/apache/hadoop/conf/TestConfiguration.java | 29 ++
2 files changed, 35 insertions(+), 4 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 14af61c..7301af6 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -2906,11 +2906,13 @@ public class Configuration implements
Iterable>,
// methods that allow non-strings to be put into configurations are
removed,
// we could replace properties with a Map and get rid of
this
// code.
-Map result = new HashMap();
-for(Map.Entry item: getProps().entrySet()) {
- if (item.getKey() instanceof String &&
- item.getValue() instanceof String) {
+Properties props = getProps();
+Map result = new HashMap<>();
+synchronized (props) {
+ for (Map.Entry item : props.entrySet()) {
+if (item.getKey() instanceof String && item.getValue() instanceof
String) {
result.put((String) item.getKey(), (String) item.getValue());
+}
}
}
return result.entrySet().iterator();
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index aebefe0..209d0c9 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -38,6 +38,7 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.ConcurrentModificationException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -45,6 +46,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Pattern;
import static java.util.concurrent.TimeUnit.*;
@@ -2590,4 +2592,31 @@ public class TestConfiguration {
assertEquals(" prefix >cdata\nsuffix ", conf.get("cdata-whitespace"));
return conf;
}
+
+ @Test
+ public void testConcurrentModificationDuringIteration() throws
InterruptedException {
+Configuration configuration = new Configuration();
+new Thread(() -> {
+ while (true) {
+configuration.set(String.valueOf(Math.random()),
String.valueOf(Math.random()));
+ }
+}).start();
+
+AtomicBoolean exceptionOccurred = new AtomicBoolean(false);
+
+new Thread(() -> {
+ while (true) {
+try {
+ configuration.iterator();
+} catch (final ConcurrentModificationException e) {
+ exceptionOccurred.set(true);
+ break;
+}
+ }
+}).start();
+
+Thread.sleep(1000); //give enough time for threads to run
+
+assertFalse("ConcurrentModificationException occurred",
exceptionOccurred.get());
+ }
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-2.10 updated: HADOOP-15293. TestLogLevel fails on Java 9
This is an automated email from the ASF dual-hosted git repository.
aajisaka pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new cc84223 HADOOP-15293. TestLogLevel fails on Java 9
cc84223 is described below
commit cc842231dfcdbf2626594179da3ec3b363d53d78
Author: Takanobu Asanuma
AuthorDate: Fri Mar 9 10:20:35 2018 -0800
HADOOP-15293. TestLogLevel fails on Java 9
Signed-off-by: Akira Ajisaka
(cherry picked from commit 99ab511cbac570bea9d31a55898b95590a8e3159)
Conflicts:
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java
---
.../src/test/java/org/apache/hadoop/log/TestLogLevel.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java
index 30bf726..b29c7ce 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java
@@ -356,7 +356,7 @@ public class TestLogLevel extends KerberosSecurityTestcase {
fail("A HTTPS Client should not have succeeded in connecting to a " +
"HTTP server");
} catch (SSLException e) {
- GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
+ GenericTestUtils.assertExceptionContains("recognized SSL message", e);
}
}
@@ -374,7 +374,7 @@ public class TestLogLevel extends KerberosSecurityTestcase {
fail("A HTTPS Client should not have succeeded in connecting to a " +
"HTTP server");
} catch (SSLException e) {
- GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
+ GenericTestUtils.assertExceptionContains("recognized SSL message", e);
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-2.10 updated: HADOOP-13500. Synchronizing iteration of Configuration properties object (#3776)
This is an automated email from the ASF dual-hosted git repository.
aajisaka pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new ebf5697 HADOOP-13500. Synchronizing iteration of Configuration
properties object (#3776)
ebf5697 is described below
commit ebf569793b3736ad2c57236bef969529757677fb
Author: Dhananjay Badaya <11253243+dbada...@users.noreply.github.com>
AuthorDate: Fri Dec 17 21:37:38 2021 +0530
HADOOP-13500. Synchronizing iteration of Configuration properties object
(#3776)
Signed-off-by: Akira Ajisaka
---
.../java/org/apache/hadoop/conf/Configuration.java | 10 ---
.../org/apache/hadoop/conf/TestConfiguration.java | 35 ++
2 files changed, 41 insertions(+), 4 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 5d34a8e..efe34ef 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -2696,11 +2696,13 @@ public class Configuration implements
Iterable>,
// methods that allow non-strings to be put into configurations are
removed,
// we could replace properties with a Map and get rid of
this
// code.
-Map result = new HashMap();
-for(Map.Entry item: getProps().entrySet()) {
- if (item.getKey() instanceof String &&
- item.getValue() instanceof String) {
+Properties props = getProps();
+Map result = new HashMap<>();
+synchronized (props) {
+ for (Map.Entry item : props.entrySet()) {
+if (item.getKey() instanceof String && item.getValue() instanceof
String) {
result.put((String) item.getKey(), (String) item.getValue());
+}
}
}
return result.entrySet().iterator();
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 50bf56c..e2ad643 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -36,12 +36,14 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.ConcurrentModificationException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Pattern;
import static java.util.concurrent.TimeUnit.*;
@@ -2133,6 +2135,39 @@ public class TestConfiguration extends TestCase {
checkCDATA(os.toByteArray());
}
+ @Test
+ public void testConcurrentModificationDuringIteration() throws
InterruptedException {
+final Configuration configuration = new Configuration();
+new Thread(new Runnable() {
+ @Override
+ public void run() {
+while (true) {
+ configuration.set(String.valueOf(Math.random()),
String.valueOf(Math.random()));
+}
+ }
+}).start();
+
+final AtomicBoolean exceptionOccurred = new AtomicBoolean(false);
+
+new Thread(new Runnable() {
+ @Override
+ public void run() {
+while (true) {
+ try {
+configuration.iterator();
+ } catch (final ConcurrentModificationException e) {
+exceptionOccurred.set(true);
+break;
+ }
+}
+ }
+}).start();
+
+Thread.sleep(1000); //give enough time for threads to run
+
+assertFalse("ConcurrentModificationException occurred",
exceptionOccurred.get());
+ }
+
private static Configuration checkCDATA(byte[] bytes) {
Configuration conf = new Configuration(false);
conf.addResource(new ByteArrayInputStream(bytes));
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 01de8fd YARN-6862. Nodemanager resource usage metrics sometimes are
negative. Contributed by Benjamin Teke
01de8fd is described below
commit 01de8fd9d650a41aba850043f78e565de759a83e
Author: Szilard Nemeth
AuthorDate: Fri Dec 17 14:53:06 2021 +0100
YARN-6862. Nodemanager resource usage metrics sometimes are negative.
Contributed by Benjamin Teke
---
.../monitor/ContainersMonitorImpl.java | 8
.../MockCPUResourceCalculatorProcessTree.java | 10 +
...> MockMemoryResourceCalculatorProcessTree.java} | 47 +++---
.../monitor/MockResourceCalculatorProcessTree.java | 6 +++
.../TestContainersMonitorResourceChange.java | 24 ---
5 files changed, 75 insertions(+), 20 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index a7bf73f..d7da7d7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -540,6 +540,14 @@ public class ContainersMonitorImpl extends AbstractService
implements
pTree.updateProcessTree();// update process-tree
long currentVmemUsage = pTree.getVirtualMemorySize();
long currentPmemUsage = pTree.getRssMemorySize();
+if (currentVmemUsage < 0 || currentPmemUsage < 0) {
+ // YARN-6862/YARN-5021 If the container just exited or for
+ // another reason the physical/virtual memory is UNAVAILABLE (-1)
+ // the values shouldn't be aggregated.
+ LOG.info("Skipping monitoring container {} because "
+ + "memory usage is not available.", containerId);
+ continue;
+}
// if machine has 6 cores and 3 are used,
// cpuUsagePercentPerCore should be 300%
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
index eb35c91..49161f3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
@@ -57,6 +57,16 @@ public class MockCPUResourceCalculatorProcessTree
}
@Override
+ public long getVirtualMemorySize(int olderThanAge) {
+return 0;
+ }
+
+ @Override
+ public long getRssMemorySize(int olderThanAge) {
+return 0;
+ }
+
+ @Override
public float getCpuUsagePercent() {
long cpu = this.cpuPercentage;
// First getter call will be returned with -1, and other calls will
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java
similarity index 50%
copy from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
copy to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java
index eb35c91..ea45ac4 100644
---
[hadoop] branch branch-3.3 updated: YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 357423b YARN-6862. Nodemanager resource usage metrics sometimes are
negative. Contributed by Benjamin Teke
357423b is described below
commit 357423b57a93249aebd635eb461ba1cff63c7353
Author: Szilard Nemeth
AuthorDate: Fri Dec 17 14:51:41 2021 +0100
YARN-6862. Nodemanager resource usage metrics sometimes are negative.
Contributed by Benjamin Teke
---
.../monitor/ContainersMonitorImpl.java | 8
.../MockCPUResourceCalculatorProcessTree.java | 10 +
...> MockMemoryResourceCalculatorProcessTree.java} | 47 +++---
.../monitor/MockResourceCalculatorProcessTree.java | 6 +++
.../TestContainersMonitorResourceChange.java | 24 ---
5 files changed, 75 insertions(+), 20 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index a83ae3a..deccded 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -537,6 +537,14 @@ public class ContainersMonitorImpl extends AbstractService
implements
pTree.updateProcessTree();// update process-tree
long currentVmemUsage = pTree.getVirtualMemorySize();
long currentPmemUsage = pTree.getRssMemorySize();
+if (currentVmemUsage < 0 || currentPmemUsage < 0) {
+ // YARN-6862/YARN-5021 If the container just exited or for
+ // another reason the physical/virtual memory is UNAVAILABLE (-1)
+ // the values shouldn't be aggregated.
+ LOG.info("Skipping monitoring container {} because "
+ + "memory usage is not available.", containerId);
+ continue;
+}
// if machine has 6 cores and 3 are used,
// cpuUsagePercentPerCore should be 300%
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
index eb35c91..49161f3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
@@ -57,6 +57,16 @@ public class MockCPUResourceCalculatorProcessTree
}
@Override
+ public long getVirtualMemorySize(int olderThanAge) {
+return 0;
+ }
+
+ @Override
+ public long getRssMemorySize(int olderThanAge) {
+return 0;
+ }
+
+ @Override
public float getCpuUsagePercent() {
long cpu = this.cpuPercentage;
// First getter call will be returned with -1, and other calls will
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java
similarity index 50%
copy from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
copy to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java
index eb35c91..ea45ac4 100644
---
[hadoop] branch branch-2.10 updated: HADOOP-18049. Pin python-lazy-object-proxy to 1.6.0 in docker image (#3813)
This is an automated email from the ASF dual-hosted git repository. aajisaka pushed a commit to branch branch-2.10 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-2.10 by this push: new 2873606 HADOOP-18049. Pin python-lazy-object-proxy to 1.6.0 in docker image (#3813) 2873606 is described below commit 2873606fc9d1669bdac84048ac24e458a11424eb Author: Dhananjay Badaya <11253243+dbada...@users.noreply.github.com> AuthorDate: Fri Dec 17 14:32:26 2021 +0530 HADOOP-18049. Pin python-lazy-object-proxy to 1.6.0 in docker image (#3813) Signed-off-by: Akira Ajisaka --- dev-support/docker/Dockerfile | 3 ++- dev-support/docker/pkg-resolver/install-common-pkgs.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index cf163d9e..9e7242b 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -183,7 +183,8 @@ RUN apt-get -q update \ RUN pip2 install \ configparser==4.0.2 \ pylint==1.9.2 \ -isort==4.3.21 +isort==4.3.21 \ +lazy-object-proxy==1.6.0 # Install dateutil.parser diff --git a/dev-support/docker/pkg-resolver/install-common-pkgs.sh b/dev-support/docker/pkg-resolver/install-common-pkgs.sh index 1396b95..1ac35d9 100644 --- a/dev-support/docker/pkg-resolver/install-common-pkgs.sh +++ b/dev-support/docker/pkg-resolver/install-common-pkgs.sh @@ -19,4 +19,4 @@ ## # Install pylint and python-dateutil ## -pip2 install configparser==4.0.2 pylint==1.9.2 isort==4.3.21 python-dateutil==2.7.3 \ No newline at end of file +pip2 install configparser==4.0.2 pylint==1.9.2 isort==4.3.21 lazy-object-proxy==1.6.0 python-dateutil==2.7.3 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
