This is an automated email from the ASF dual-hosted git repository.
Hexiaoqiao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c7dfb267c02 YARN-11967. Strict memory enforcement should not skip
polling-based memory check when CGroups memory is disabled. (#8573).
Contributed by yanbin.zhang.
c7dfb267c02 is described below
commit c7dfb267c024b0ca7cabe41ed75dffd2993ee2ca
Author: singer-bin <[email protected]>
AuthorDate: Tue Jul 7 11:11:24 2026 +0800
YARN-11967. Strict memory enforcement should not skip polling-based memory
check when CGroups memory is disabled. (#8573). Contributed by yanbin.zhang.
Signed-off-by: He Xiaoqiao <[email protected]>
---
.../src/main/resources/yarn-default.xml | 4 ++
.../monitor/ContainersMonitorImpl.java | 25 ++++++++++++
.../monitor/TestContainersMonitor.java | 47 ++++++++++++++++++++++
3 files changed, 76 insertions(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index b128d0f0ef1..29ff936ab12 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1771,6 +1771,10 @@
<property>
<description>Whether YARN CGroups strict memory enforcement is enabled.
+ This only takes effect when the CGroups memory controller is enabled via
+ yarn.nodemanager.resource.memory.enabled. When CGroups memory is disabled
+ (the default), this flag has no effect and container memory limits are
+ enforced (if at all) by the polling-based ContainersMonitor.
</description>
<name>yarn.nodemanager.resource.memory.enforced</name>
<value>true</value>
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 0b4bd4a3fbd..9db7671cdd1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -203,7 +203,18 @@ protected void serviceInit(Configuration myConf) throws
Exception {
elasticMemoryEnforcement = this.conf.getBoolean(
YarnConfiguration.NM_ELASTIC_MEMORY_CONTROL_ENABLED,
YarnConfiguration.DEFAULT_NM_ELASTIC_MEMORY_CONTROL_ENABLED);
+ // CGroup-based strict memory enforcement (relying on the kernel OOM
+ // killer) is only in effect when the CGroups memory controller is actually
+ // enabled via yarn.nodemanager.resource.memory.enabled. The "enforced"
+ // flag alone (which defaults to true) does not write any CGroups memory
+ // hard limit unless the memory controller is enabled. Gating
+ // strictMemoryEnforcement on both flags prevents the polling-based memory
+ // check from being skipped (see checkLimit) when no CGroups memory limit
+ // is actually applied, which would otherwise leave containers unbounded.
strictMemoryEnforcement = conf.getBoolean(
+ YarnConfiguration.NM_MEMORY_RESOURCE_ENABLED,
+ YarnConfiguration.DEFAULT_NM_MEMORY_RESOURCE_ENABLED)
+ && conf.getBoolean(
YarnConfiguration.NM_MEMORY_RESOURCE_ENFORCED,
YarnConfiguration.DEFAULT_NM_MEMORY_RESOURCE_ENFORCED);
LOG.info("Physical memory check enabled: {}", pmemCheckEnabled);
@@ -1049,6 +1060,20 @@ public boolean isVmemCheckEnabled() {
return this.vmemCheckEnabled;
}
+ /**
+ * Is CGroup-based strict memory enforcement in effect? This is true only
+ * when both {@code yarn.nodemanager.resource.memory.enabled} and
+ * {@code yarn.nodemanager.resource.memory.enforced} are true. When it is
+ * true, the kernel OOM killer enforces the limit and the polling-based
+ * memory check is skipped.
+ *
+ * @return true if CGroup-based strict memory enforcement is in effect.
+ */
+ @VisibleForTesting
+ boolean isStrictMemoryEnforcementEnabled() {
+ return this.strictMemoryEnforcement;
+ }
+
@Override
public ResourceUtilization getContainersUtilization() {
return this.containersUtilization;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
index ac3ea2eb6c5..b961ef70b71 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
@@ -565,6 +565,53 @@ public void testContainerMonitorMemFlags() {
assertEquals(true, cm.isVmemCheckEnabled());
}
+ /**
+ * Verifies that CGroup-based strict memory enforcement (which skips the
+ * polling-based memory check, relying on the kernel OOM killer) is only
+ * considered active when the CGroups memory controller is actually enabled
+ * via {@code yarn.nodemanager.resource.memory.enabled}. With the historical
+ * behaviour, {@code yarn.nodemanager.resource.memory.enforced} alone
+ * (defaulting to true) was enough to skip the polling check, which left
+ * containers unbounded when CGroups memory was not enabled.
+ */
+ @Test
+ public void testStrictMemoryEnforcementRequiresMemoryEnabled()
+ throws Exception {
+ // memory.enabled defaults to false, memory.enforced defaults to true.
+ // Strict enforcement must NOT be considered active in this case, otherwise
+ // the polling-based memory check would be wrongly skipped.
+ ContainersMonitorImpl cm =
+ new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ cm.init(getConfForCM(true, false, 8192, 2.1f));
+ assertFalse(cm.isStrictMemoryEnforcementEnabled(),
+ "Strict memory enforcement must be off when "
+ + "yarn.nodemanager.resource.memory.enabled is false, "
+ + "even if yarn.nodemanager.resource.memory.enforced is true");
+
+ // memory.enabled = true, memory.enforced = true -> active.
+ cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ YarnConfiguration conf = getConfForCM(true, false, 8192, 2.1f);
+ conf.setBoolean(YarnConfiguration.NM_MEMORY_RESOURCE_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_MEMORY_RESOURCE_ENFORCED, true);
+ cm.init(conf);
+ assertTrue(cm.isStrictMemoryEnforcementEnabled(),
+ "Strict memory enforcement must be on when both memory.enabled "
+ + "and memory.enforced are true");
+
+ // memory.enabled = true, memory.enforced = false -> not active.
+ cm = new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ conf = getConfForCM(true, false, 8192, 2.1f);
+ conf.setBoolean(YarnConfiguration.NM_MEMORY_RESOURCE_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_MEMORY_RESOURCE_ENFORCED, false);
+ cm.init(conf);
+ assertFalse(cm.isStrictMemoryEnforcementEnabled(),
+ "Strict memory enforcement must be off when "
+ + "memory.enforced is false");
+ }
+
private YarnConfiguration getConfForCM(boolean pMemEnabled,
boolean vMemEnabled, int nmPmem, float vMemToPMemRatio) {
YarnConfiguration conf = new YarnConfiguration();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
