[hadoop] branch trunk updated: YARN.11590. RM process stuck after calling confStore.format() when ZK SSL/TLS is enabled, as netty thread waits indefinitely. Contributed by Ferenc Erdelyi
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2736f8856144 YARN.11590. RM process stuck after calling
confStore.format() when ZK SSL/TLS is enabled, as netty thread waits
indefinitely. Contributed by Ferenc Erdelyi
2736f8856144 is described below
commit 2736f8856144c8361be3821cd1c158ef2d9a2dec
Author: Szilard Nemeth
AuthorDate: Mon Oct 16 15:17:58 2023 -0400
YARN.11590. RM process stuck after calling confStore.format() when ZK
SSL/TLS is enabled, as netty thread waits indefinitely. Contributed by Ferenc
Erdelyi
---
.../hadoop/yarn/server/resourcemanager/ResourceManager.java | 9 +
.../scheduler/capacity/conf/YarnConfigurationStore.java | 2 +-
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 90eaed3d8a02..f358a22cee71 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -1903,10 +1903,11 @@ public class ResourceManager extends CompositeService
}
if (scheduler instanceof MutableConfScheduler && isConfigurationMutable) {
- YarnConfigurationStore confStore = YarnConfigurationStoreFactory
- .getStore(conf);
- confStore.initialize(conf, conf, rmContext);
- confStore.format();
+ try (YarnConfigurationStore confStore = YarnConfigurationStoreFactory
+ .getStore(conf)) {
+confStore.initialize(conf, conf, rmContext);
+confStore.format();
+ }
} else {
System.out.println(String.format("Scheduler Configuration format only " +
"supported by %s.", MutableConfScheduler.class.getSimpleName()));
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 0f44f439056b..2ec2ea442a64 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -44,7 +44,7 @@ import java.util.Map;
* {@code getPendingMutations}, and replay/confirm them via
* {@code confirmMutation} as in the normal case.
*/
-public abstract class YarnConfigurationStore {
+public abstract class YarnConfigurationStore implements AutoCloseable {
public static final Logger LOG =
LoggerFactory.getLogger(YarnConfigurationStore.class);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: MAPREDUCE-7456. Extend add-opens flag to container launch commands on JDK17 nodes. Contributed by Peter Szucs
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2d871fab7888 MAPREDUCE-7456. Extend add-opens flag to container launch
commands on JDK17 nodes. Contributed by Peter Szucs
2d871fab7888 is described below
commit 2d871fab7888aa531a199aa52bc1e0875f6a85f3
Author: Szilard Nemeth
AuthorDate: Wed Sep 27 22:33:21 2023 -0400
MAPREDUCE-7456. Extend add-opens flag to container launch commands on JDK17
nodes. Contributed by Peter Szucs
---
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 8 +
.../src/main/resources/yarn-default.xml| 11 +++
.../containermanager/launcher/ContainerLaunch.java | 8 +++--
.../localizer/ContainerLocalizer.java | 15 ++
.../launcher/TestContainerLaunch.java | 8 +++--
.../localizer/TestContainerLocalizer.java | 35 ++
6 files changed, 81 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 13d4c209110e..90a8978a228b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2232,6 +2232,14 @@ public class YarnConfiguration extends Configuration {
public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT =
"-Xmx256m";
+ /*
+ * Flag to indicate whether JDK17's required add-exports flags should be
added to
+ * container localizers regardless of the user specified JAVA_OPTS.
+ */
+ public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_KEY =
+ NM_PREFIX + "container-localizer.java.opts.add-exports-as-default";
+ public static final boolean
NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_DEFAULT = true;
+
/** The log level of container localizer process. */
public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL=
NM_PREFIX + "container-localizer.log.level";
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 2259b73fb651..9991e841d74b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1484,6 +1484,17 @@
-Xmx256m
+
+
yarn.nodemanager.container-localizer.java.opts.add-exports-as-default
+true
+Since on JDK17 it is no longer possible to use the reflection
API to
+ access public fields and methods, add-exports flags should be added to
container
+ localizers regardless of the user specified JAVA_OPTS. Setting this to
true will
+ add the flags to the container localizer on nodes with JDK17 or higher.
+ Defaults to true, but the setting has no effect on nodes using JDK16 and
before.
+
+
+
The log level for container localizer while it is an independent process.
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 143086db2aec..5466182b715d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -123,6 +123,11 @@ public class ContainerLaunch implements Callable {
private static final String PID_FILE_NAME_FMT = "%s.pid";
static final String EXIT_CODE_FILE_SUFFIX = ".exitcode";
+ private static final String ADDITIONAL_JDK17_PLUS_OPTIONS =
+ "--add-opens=java.base/java.lang=ALL-UNNAMED " +
+ "--add-exports=java.base/sun.net.dns=ALL-UNNAMED " +
+ "--add-exports=java.base/sun.net.util=ALL-UNNAMED";
+
protected final Dispatcher dispatcher;
protected final ContainerExecutor exec;
protected final Application app;
@@ -171,8 +176,7 @@ public class ContainerLaunch implements Callable {
File.pathS
[hadoop] branch trunk updated: YARN-11468. Zookeeper SSL/TLS support. Contributed by Ferenc Erdelyi
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d9cb76ac985a YARN-11468. Zookeeper SSL/TLS support. Contributed by
Ferenc Erdelyi
d9cb76ac985a is described below
commit d9cb76ac985a1f28208e29a6f1efa6d161720fb0
Author: Szilard Nemeth
AuthorDate: Wed Sep 27 18:21:45 2023 -0400
YARN-11468. Zookeeper SSL/TLS support. Contributed by Ferenc Erdelyi
---
.../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java| 4
.../hadoop-yarn-common/src/main/resources/yarn-default.xml | 6 ++
.../apache/hadoop/yarn/server/resourcemanager/ResourceManager.java | 3 ++-
3 files changed, 12 insertions(+), 1 deletion(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index bbb1ed6f8a7d..13d4c209110e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -857,6 +857,10 @@ public class YarnConfiguration extends Configuration {
/** Zookeeper interaction configs */
public static final String RM_ZK_PREFIX = RM_PREFIX + "zk-";
+ /** Enable Zookeeper SSL/TLS communication. */
+ public static final String RM_ZK_CLIENT_SSL_ENABLED = RM_ZK_PREFIX +
"client-ssl.enabled";
+ public static final boolean DEFAULT_RM_ZK_CLIENT_SSL_ENABLED = false;
+
public static final String RM_ZK_ADDRESS = RM_ZK_PREFIX + "address";
public static final String RM_ZK_NUM_RETRIES = RM_ZK_PREFIX + "num-retries";
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 9fa600db4b03..2259b73fb651 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -741,6 +741,12 @@
1048576
+
+Enable SSL/TLS encryption for the ZooKeeper
communication.
+yarn.resourcemanager.zk-client-ssl.enabled
+false
+
+
Name of the cluster. In a HA setting,
this is used to ensure the RM participates in leader
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 2730dde72fc1..90eaed3d8a02 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -427,7 +427,8 @@ public class ResourceManager extends CompositeService
authInfos.add(authInfo);
}
-manager.start(authInfos);
+manager.start(authInfos,
config.getBoolean(YarnConfiguration.RM_ZK_CLIENT_SSL_ENABLED,
+YarnConfiguration.DEFAULT_RM_ZK_CLIENT_SSL_ENABLED));
return manager;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-18870. CURATOR-599 change broke functionality introduced in HADOOP-18139 and HADOOP-18709. Contributed by Ferenc Erdelyi
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9342ecf6ccd HADOOP-18870. CURATOR-599 change broke functionality
introduced in HADOOP-18139 and HADOOP-18709. Contributed by Ferenc Erdelyi
9342ecf6ccd is described below
commit 9342ecf6ccd5c7ef443a0eb722852d2addc1d5db
Author: Szilard Nemeth
AuthorDate: Wed Sep 6 21:32:36 2023 -0400
HADOOP-18870. CURATOR-599 change broke functionality introduced in
HADOOP-18139 and HADOOP-18709. Contributed by Ferenc Erdelyi
---
.../hadoop/util/curator/ZKCuratorManager.java | 12 +++-
.../hadoop/util/curator/TestZKCuratorManager.java | 35 ++
2 files changed, 46 insertions(+), 1 deletion(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
index 81ee4663e1c..4df79774329 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
@@ -549,7 +549,17 @@ public final class ZKCuratorManager {
public ZooKeeper newZooKeeper(String connectString, int sessionTimeout,
Watcher watcher, boolean canBeReadOnly
) throws Exception {
- ZKClientConfig zkClientConfig = new ZKClientConfig();
+ return this.newZooKeeper(connectString, sessionTimeout,
+ watcher, canBeReadOnly, new ZKClientConfig());
+}
+
+@Override
+public ZooKeeper newZooKeeper(String connectString, int sessionTimeout,
+Watcher watcher, boolean canBeReadOnly, ZKClientConfig zkClientConfig
+) throws Exception {
+ if (zkClientConfig == null) {
+zkClientConfig = new ZKClientConfig();
+ }
if (zkPrincipal != null) {
LOG.info("Configuring zookeeper to use {} as the server principal",
zkPrincipal);
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
index fd15a0c2b1b..4365e43e491 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
@@ -22,10 +22,15 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.security.auth.login.AppConfigurationEntry;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
import org.apache.curator.test.TestingServer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -193,6 +198,36 @@ public class TestZKCuratorManager {
}
}
+ @Test
+ public void testCuratorFrameworkFactory() throws Exception{
+// By not explicitly calling the NewZooKeeper method validate that the
Curator override works.
+ZKClientConfig zkClientConfig = new ZKClientConfig();
+Configuration conf = new Configuration();
+conf.set(CommonConfigurationKeys.ZK_ADDRESS,
this.server.getConnectString());
+int numRetries = conf.getInt(CommonConfigurationKeys.ZK_NUM_RETRIES,
+CommonConfigurationKeys.ZK_NUM_RETRIES_DEFAULT);
+int zkSessionTimeout = conf.getInt(CommonConfigurationKeys.ZK_TIMEOUT_MS,
+CommonConfigurationKeys.ZK_TIMEOUT_MS_DEFAULT);
+int zkRetryInterval = conf.getInt(
+CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS,
+CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS_DEFAULT);
+RetryNTimes retryPolicy = new RetryNTimes(numRetries, zkRetryInterval);
+
+CuratorFramework client = CuratorFrameworkFactory.builder()
+.connectString(conf.get(CommonConfigurationKeys.ZK_ADDRESS))
+.zkClientConfig(zkClientConfig)
+.sessionTimeoutMs(zkSessionTimeout).retryPolicy(retryPolicy)
+.authorization(new ArrayList<>())
+.zookeeperFactory(new ZKCuratorManager.HadoopZookeeperFactory(
+"foo1", "bar1", "bar1.keytab", false,
+new ZKCuratorManager.TruststoreKeystore(conf))
+
+).build();
+client.start();
+
validateJaasConfiguration(ZKCuratorManager.HadoopZookeeperFactory.JAAS_CLIENT_ENTRY,
+"bar1", "bar1.keytab", client.getZookeeperClient().getZooKeeper());
+ }
+
private voi
[hadoop] branch trunk updated: MAPREDUCE-7441. Fix race condition in closing FadvisedFileRegion. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1c15987ee38 MAPREDUCE-7441. Fix race condition in closing
FadvisedFileRegion. Contributed by Benjamin Teke
1c15987ee38 is described below
commit 1c15987ee38b96cf2ef4ff383f3f86e082c50fab
Author: Szilard Nemeth
AuthorDate: Fri Jun 23 14:40:03 2023 -0400
MAPREDUCE-7441. Fix race condition in closing FadvisedFileRegion.
Contributed by Benjamin Teke
---
.../apache/hadoop/mapred/FadvisedFileRegion.java | 102 -
1 file changed, 58 insertions(+), 44 deletions(-)
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
index 9290a282e39..184b58e6c76 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
@@ -41,6 +41,7 @@ public class FadvisedFileRegion extends DefaultFileRegion {
private static final Logger LOG =
LoggerFactory.getLogger(FadvisedFileRegion.class);
+ private final Object closeLock = new Object();
private final boolean manageOsCache;
private final int readaheadLength;
private final ReadaheadPool readaheadPool;
@@ -51,12 +52,12 @@ public class FadvisedFileRegion extends DefaultFileRegion {
private final int shuffleBufferSize;
private final boolean shuffleTransferToAllowed;
private final FileChannel fileChannel;
-
- private ReadaheadRequest readaheadRequest;
+
+ private volatile ReadaheadRequest readaheadRequest;
public FadvisedFileRegion(RandomAccessFile file, long position, long count,
boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
- String identifier, int shuffleBufferSize,
+ String identifier, int shuffleBufferSize,
boolean shuffleTransferToAllowed) throws IOException {
super(file.getChannel(), position, count);
this.manageOsCache = manageOsCache;
@@ -73,97 +74,110 @@ public class FadvisedFileRegion extends DefaultFileRegion {
@Override
public long transferTo(WritableByteChannel target, long position)
- throws IOException {
-if (readaheadPool != null && readaheadLength > 0) {
- readaheadRequest = readaheadPool.readaheadStream(identifier, fd,
- position() + position, readaheadLength,
- position() + count(), readaheadRequest);
+ throws IOException {
+synchronized (closeLock) {
+ if (fd.valid()) {
+if (readaheadPool != null && readaheadLength > 0) {
+ readaheadRequest = readaheadPool.readaheadStream(identifier, fd,
+ position() + position, readaheadLength,
+ position() + count(), readaheadRequest);
+}
+
+if(this.shuffleTransferToAllowed) {
+ return super.transferTo(target, position);
+} else {
+ return customShuffleTransfer(target, position);
+}
+ } else {
+return 0L;
+ }
}
-
-if(this.shuffleTransferToAllowed) {
- return super.transferTo(target, position);
-} else {
- return customShuffleTransfer(target, position);
-}
+
}
/**
- * This method transfers data using local buffer. It transfers data from
- * a disk to a local buffer in memory, and then it transfers data from the
+ * This method transfers data using local buffer. It transfers data from
+ * a disk to a local buffer in memory, and then it transfers data from the
* buffer to the target. This is used only if transferTo is disallowed in
- * the configuration file. super.TransferTo does not perform well on Windows
- * due to a small IO request generated. customShuffleTransfer can control
- * the size of the IO requests by changing the size of the intermediate
+ * the configuration file. super.TransferTo does not perform well on Windows
+ * due to a small IO request generated. customShuffleTransfer can control
+ * the size of the IO requests by changing the size of the intermediate
* buffer.
*/
@VisibleForTesting
long customShuffleTransfer(WritableByteChannel target, long position)
- throws IOException {
+ throws IOException {
long actualCount = this.count - position;
if (actualCount < 0 || position < 0) {
throw new IllegalArgumentException(
- "position out of range: " + position +
- " (expected: 0 - " + (this.count -
[hadoop] branch trunk updated: HADOOP-18709. Add curator based ZooKeeper communication support over SSL/TLS into the common library. Contributed by Ferenc Erdelyi
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e0a339223a8 HADOOP-18709. Add curator based ZooKeeper communication
support over SSL/TLS into the common library. Contributed by Ferenc Erdelyi
e0a339223a8 is described below
commit e0a339223a89367d9391758ff62c99923369b5ee
Author: Szilard Nemeth
AuthorDate: Sun Jun 4 14:40:41 2023 -0400
HADOOP-18709. Add curator based ZooKeeper communication support over
SSL/TLS into the common library. Contributed by Ferenc Erdelyi
---
hadoop-common-project/hadoop-common/pom.xml| 8 +
.../apache/hadoop/fs/CommonConfigurationKeys.java | 8 +
.../hadoop/util/curator/ZKCuratorManager.java | 222 ++---
.../src/main/resources/core-default.xml| 29 +++
.../util/curator/TestSecureZKCuratorManager.java | 214
.../util/curator/resources/data/ssl/keystore.jks | Bin 0 -> 4160 bytes
.../util/curator/resources/data/ssl/truststore.jks | Bin 0 -> 994 bytes
7 files changed, 458 insertions(+), 23 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/pom.xml
b/hadoop-common-project/hadoop-common/pom.xml
index 58006c011da..426f7a4af41 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -340,6 +340,14 @@
+
+ io.netty
+ netty-handler
+
+
+ io.netty
+ netty-transport-native-epoll
+
io.dropwizard.metrics
metrics-core
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 9d6224366d1..63f494a20ac 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -417,6 +417,14 @@ public class CommonConfigurationKeys extends
CommonConfigurationKeysPublic {
/** How often to retry a ZooKeeper operation in milliseconds. */
public static final String ZK_RETRY_INTERVAL_MS =
ZK_PREFIX + "retry-interval-ms";
+ /** Keystore location for ZooKeeper client connection over SSL. */
+ public static final String ZK_SSL_KEYSTORE_LOCATION = ZK_PREFIX +
"ssl.keystore.location";
+ /** Keystore password for ZooKeeper client connection over SSL. */
+ public static final String ZK_SSL_KEYSTORE_PASSWORD = ZK_PREFIX +
"ssl.keystore.password";
+ /** Truststore location for ZooKeeper client connection over SSL. */
+ public static final String ZK_SSL_TRUSTSTORE_LOCATION = ZK_PREFIX +
"ssl.truststore.location";
+ /** Truststore password for ZooKeeper client connection over SSL. */
+ public static final String ZK_SSL_TRUSTSTORE_PASSWORD = ZK_PREFIX +
"ssl.truststore.password";
public static final intZK_RETRY_INTERVAL_MS_DEFAULT = 1000;
/** Default domain name resolver for hadoop to use. */
public static final String HADOOP_DOMAINNAME_RESOLVER_IMPL =
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
index c11b8683867..81ee4663e1c 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
+import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.AuthInfo;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
@@ -39,13 +40,17 @@ import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.client.ZKClientConfig;
+import org.apache.zookeeper.common.ClientX509Util;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.util.Preconditions;
+import javax.naming.ConfigurationException;
+
/**
* Helper class that provides utility methods specific to ZK operations.
*/
@@ -122,7 +127,7 @@ public final class ZKCuratorManager {
* Start the connection to the ZooKeeper ensemble.
* @throws IOException If the connection cannot be started.
*/
- public void start() throws IOException {
+ public void start() throws IOException{
this.start(new ArrayList<>());
[hadoop] branch trunk updated: YARN-11079. Make an AbstractParentQueue to store common ParentQueue and ManagedParentQueue functionality. Contributed by Susheel Gupta
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c7699d3dcd4 YARN-11079. Make an AbstractParentQueue to store common
ParentQueue and ManagedParentQueue functionality. Contributed by Susheel Gupta
c7699d3dcd4 is described below
commit c7699d3dcd4f8feaf2c5ae5943b8a4cec738e95d
Author: Szilard Nemeth
AuthorDate: Thu May 4 22:16:18 2023 -0400
YARN-11079. Make an AbstractParentQueue to store common ParentQueue and
ManagedParentQueue functionality. Contributed by Susheel Gupta
---
.../ProportionalCapacityPreemptionPolicy.java |6 +-
.../monitor/capacity/TempQueuePerPartition.java|3 +-
.../csmappingrule/MappingRuleValidationHelper.java |5 +-
.../scheduler/capacity/AbstractCSQueue.java|6 +-
.../capacity/AbstractManagedParentQueue.java |2 +-
.../{ParentQueue.java => AbstractParentQueue.java} | 233 +--
.../capacity/AutoCreatedQueueManagementPolicy.java |4 +-
.../capacity/CSMaxRunningAppsEnforcer.java | 12 +-
.../scheduler/capacity/CapacityScheduler.java |2 +-
.../capacity/CapacitySchedulerConfigValidator.java |4 +-
.../capacity/CapacitySchedulerQueueManager.java|9 +-
.../scheduler/capacity/ManagedParentQueue.java |2 +-
.../scheduler/capacity/ParentQueue.java| 1572 +---
.../scheduler/capacity/PlanQueue.java |1 +
.../GuaranteedOrZeroCapacityOverTimePolicy.java|8 +-
.../event/QueueManagementChangeEvent.java |9 +-
.../webapp/dao/CapacitySchedulerInfo.java |6 +-
.../webapp/dao/CapacitySchedulerQueueInfo.java |6 +-
.../dao/helper/CapacitySchedulerInfoHelper.java|3 +-
.../TestCapacitySchedulerNewQueueAutoCreation.java |4 +-
.../capacity/TestCapacitySchedulerQueues.java |2 +-
.../scheduler/capacity/TestChildQueueOrder.java|4 +-
.../scheduler/capacity/TestParentQueue.java|4 +-
23 files changed, 133 insertions(+), 1774 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 1c4d60962ee..443241a664a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -21,6 +21,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.StringUtils;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractParentQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -41,7 +42,6 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
.ManagedParentQueue;
-import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptableQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent;
@@ -630,9 +630,9 @@ public class ProportionalCapacityPreemptionPolicy
partitionToLookAt, killable, absCap, absMaxCap, partitionResource,
reserved, curQueue, effMinRes, effMaxRes);
- if (curQueue instanceof ParentQueue) {
+ if (curQueue instanceof AbstractParentQueue) {
String configuredOrderingPolicy =
-((ParentQueue) curQueue).getQueueOrderingPolicy().getConfigName();
+((AbstractParentQueue)
curQueue).getQueueOrderingPolicy().getConfigName();
// Recursively add children
for (CSQueue c : curQueue.getChildQueues()) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-y
[hadoop] branch trunk updated: MAPREDUCE-7434. Fix ShuffleHandler tests. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8f6be3678d1 MAPREDUCE-7434. Fix ShuffleHandler tests. Contributed by
Tamas Domok
8f6be3678d1 is described below
commit 8f6be3678d1113e3e7f5477c357fc81f62d460b8
Author: Szilard Nemeth
AuthorDate: Wed Mar 1 16:10:05 2023 +0100
MAPREDUCE-7434. Fix ShuffleHandler tests. Contributed by Tamas Domok
---
.../hadoop/mapred/TestShuffleChannelHandler.java | 2 +-
.../apache/hadoop/mapred/TestShuffleHandler.java | 44 +++---
.../hadoop/mapred/TestShuffleHandlerBase.java | 29 +++---
3 files changed, 47 insertions(+), 28 deletions(-)
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleChannelHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleChannelHandler.java
index 7fedc7bb2dc..66fa3de94f8 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleChannelHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleChannelHandler.java
@@ -225,7 +225,7 @@ public class TestShuffleChannelHandler extends
TestShuffleHandlerBase {
final ShuffleTest t = createShuffleTest();
final EmbeddedChannel shuffle = t.createShuffleHandlerChannelFileRegion();
-String dataFile = getDataFile(tempDir.toAbsolutePath().toString(),
TEST_ATTEMPT_2);
+String dataFile = getDataFile(TEST_USER,
tempDir.toAbsolutePath().toString(), TEST_ATTEMPT_2);
assertTrue("should delete", new File(dataFile).delete());
FullHttpRequest req = t.createRequest(getUri(TEST_JOB_ID, 0,
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 37a9210286c..cc46b49b113 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -29,6 +29,7 @@ import static
org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -41,6 +42,7 @@ import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
@@ -159,7 +161,7 @@ public class TestShuffleHandler extends
TestShuffleHandlerBase {
shuffleHandler.init(conf);
shuffleHandler.start();
final String port =
shuffleHandler.getConfig().get(SHUFFLE_PORT_CONFIG_KEY);
-final SecretKey secretKey = shuffleHandler.addTestApp();
+final SecretKey secretKey = shuffleHandler.addTestApp(TEST_USER);
// setup connections
HttpURLConnection[] conns = new HttpURLConnection[connAttempts];
@@ -237,7 +239,7 @@ public class TestShuffleHandler extends
TestShuffleHandlerBase {
shuffleHandler.init(conf);
shuffleHandler.start();
final String port =
shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY);
-final SecretKey secretKey = shuffleHandler.addTestApp();
+final SecretKey secretKey = shuffleHandler.addTestApp(TEST_USER);
HttpURLConnection conn1 = createRequest(
geURL(port, TEST_JOB_ID, 0, Collections.singletonList(TEST_ATTEMPT_1),
true),
@@ -278,18 +280,34 @@ public class TestShuffleHandler extends
TestShuffleHandlerBase {
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
UserGroupInformation.setConfiguration(conf);
+final String randomUser = "randomUser";
+final String attempt = "attempt_1_0004_m_04_0";
+generateMapOutput(randomUser, tempDir.toAbsolutePath().toString(), attempt,
+Arrays.asList(TEST_DATA_C, TEST_DATA_B, TEST_DATA_A));
+
ShuffleHandlerMock shuffleHandler = new ShuffleHandlerMock();
[hadoop] branch trunk updated: HADOOP-18602. Remove netty3 dependency
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b677d40ab5e HADOOP-18602. Remove netty3 dependency
b677d40ab5e is described below
commit b677d40ab5ee899a0264b1763d632221cefe44b4
Author: Szilard Nemeth
AuthorDate: Fri Jan 27 16:32:50 2023 +0100
HADOOP-18602. Remove netty3 dependency
---
LICENSE-binary | 1 -
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 -
hadoop-project/pom.xml | 8
3 files changed, 14 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index 95ceb0d97c1..432dc5d28f7 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -260,7 +260,6 @@ io.grpc:grpc-netty:1.26.0
io.grpc:grpc-protobuf:1.26.0
io.grpc:grpc-protobuf-lite:1.26.0
io.grpc:grpc-stub:1.26.0
-io.netty:netty:3.10.6.Final
io.netty:netty-all:4.1.77.Final
io.netty:netty-buffer:4.1.77.Final
io.netty:netty-codec:4.1.77.Final
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index b51c7154f7b..4e7efab06e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -175,11 +175,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd;>
slf4j-log4j12
provided
-
- io.netty
- netty
- compile
-
io.netty
netty-all
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 33350c929e8..aada4e9b6de 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -140,7 +140,6 @@
2.2.21
2.9.0
3.2.4
-3.10.6.Final
4.1.77.Final
1.1.8.2
1.7.1
@@ -1000,13 +999,6 @@
-
-
-io.netty
-netty
-${netty3.version}
-
-
io.netty
netty-all
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-6971. Clean up different ways to create resources. Contributed by Riya Khandelwal
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 815cde98103 YARN-6971. Clean up different ways to create resources.
Contributed by Riya Khandelwal
815cde98103 is described below
commit 815cde98103fe57dde4f6686626bc4fcc1278a2d
Author: Szilard Nemeth
AuthorDate: Wed Jan 25 17:28:29 2023 +0100
YARN-6971. Clean up different ways to create resources. Contributed by Riya
Khandelwal
---
.../v2/app/rm/TestRMContainerAllocator.java| 29
.../server/uam/UnmanagedApplicationManager.java| 4 +-
.../server/nodemanager/TestNodeStatusUpdater.java | 13 ++--
.../containermanager/BaseContainerManagerTest.java | 5 +-
.../containermanager/container/TestContainer.java | 3 +-
.../launcher/TestContainerLaunch.java | 3 +-
.../logaggregation/TestLogAggregationService.java | 3 +-
.../monitor/TestContainersMonitor.java | 3 +-
...estContainerSchedulerBehaviorCompatibility.java | 4 +-
...ContainerSchedulerOppContainersByResources.java | 8 +--
.../scheduler/TestContainerSchedulerQueuing.java | 78 +++---
.../TestNMContainerTokenSecretManager.java | 3 +-
.../server/nodemanager/webapp/MockContainer.java | 3 +-
.../server/nodemanager/webapp/TestNMWebServer.java | 4 +-
.../server/resourcemanager/ClientRMService.java| 3 +-
.../resourcemanager/DefaultAMSProcessor.java | 2 +-
.../scheduler/fair/FairSchedulerConfiguration.java | 3 +-
.../server/resourcemanager/QueueACLsTestBase.java | 4 +-
.../resourcemanager/ReservationACLsTestBase.java | 4 +-
.../resourcemanager/TestApplicationACLs.java | 6 +-
.../resourcemanager/TestClientRMService.java | 8 +--
.../server/resourcemanager/TestClientRMTokens.java | 5 +-
.../TestResourceTrackerService.java| 35 +-
.../TestRMDelegatedNodeLabelsUpdater.java | 4 +-
.../resourcetracker/TestNMExpiry.java | 4 +-
.../resourcetracker/TestNMReconnect.java | 10 +--
.../resourcetracker/TestRMNMRPCResponseId.java | 4 +-
.../rmapp/attempt/TestRMAppAttemptTransitions.java | 8 +--
.../rmcontainer/TestRMContainerImpl.java | 9 +--
.../scheduler/TestAbstractYarnScheduler.java | 5 +-
.../capacity/TestApplicationPriorityACLs.java | 4 +-
.../TestCapacitySchedulerAmbiguousLeafs.java | 4 +-
.../scheduler/fair/FairSchedulerTestBase.java | 3 +-
.../scheduler/fair/TestFairScheduler.java | 6 +-
.../fair/TestFairSchedulerConfiguration.java | 19 +++---
.../TestDominantResourceFairnessPolicy.java| 5 +-
.../scheduler/fifo/TestFifoScheduler.java | 10 +--
.../webapp/TestRMWebServicesNodes.java | 4 +-
38 files changed, 170 insertions(+), 162 deletions(-)
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index fd7a2f0ceda..fe2f3072141 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -148,6 +148,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.ControlledClock;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.util.SystemClock;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
@@ -502,8 +503,8 @@ public class TestRMContainerAllocator {
0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));
MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
appAttemptId, mockJob, SystemClock.getInstance());
-allocator.setMapResourceRequest(BuilderUtils.newResource(1024, 1));
-allocator.setReduceResourceRequest(BuilderUtils.newResource(1024, 1));
+allocator.setMapResourceRequest(Resources.createResource(1024));
+allocator.setReduceResourceRequest(Resources.createResource(1024));
RMContainerAllocator.AssignedRequests assignedRequests =
allocator.getAssignedRequests();
RMContainerAllocator.ScheduledRequests scheduledRequests =
@@ -557,8 +558,8 @@ public class TestRMContainerAllocator {
clock.setTime(1);
MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
appAttem
[hadoop] branch trunk updated: YARN-5607. Document TestContainerResourceUsage#waitForContainerCompletion. Contributed by Susheel Gupta
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 29f2230cb61 YARN-5607. Document
TestContainerResourceUsage#waitForContainerCompletion. Contributed by Susheel
Gupta
29f2230cb61 is described below
commit 29f2230cb615b975950691044a621ff008088cf5
Author: Szilard Nemeth
AuthorDate: Wed Jan 25 15:13:24 2023 +0100
YARN-5607. Document TestContainerResourceUsage#waitForContainerCompletion.
Contributed by Susheel Gupta
---
.../hadoop/yarn/server/resourcemanager/MockRM.java | 25 ++
.../TestContainerResourceUsage.java| 18 ++--
2 files changed, 27 insertions(+), 16 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 95db1171b35..316f8e06cb5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -1011,6 +1011,31 @@ public class MockRM extends ResourceManager {
LOG.info("app is removed from scheduler, " + appId);
}
+ /**
+ * Wait until a container has reached a completion state.
+ * The timeout is 20 seconds.
+ * @param nm A mock nodemanager
+ * @param rm A mock resourcemanager
+ * @param amContainerId The id of an am container
+ * @param container A container
+ * @throws Exception
+ * if interrupted while waiting for the completion transition
+ * or an unexpected error while MockNM is hearbeating.
+ */
+ public static void waitForContainerCompletion(MockRM rm, MockNM nm,
+ContainerId amContainerId, RMContainer container) throws Exception {
+ContainerId containerId = container.getContainerId();
+if (null != rm.scheduler.getRMContainer(containerId)) {
+ if (containerId.equals(amContainerId)) {
+rm.waitForState(nm, containerId, RMContainerState.COMPLETED);
+ } else {
+rm.waitForState(nm, containerId, RMContainerState.KILLED);
+ }
+} else {
+ rm.drainEvents();
+}
+ }
+
private void drainEventsImplicitly() {
if (!disableDrainEventsImplicitly) {
drainEvents();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
index 9a01087ce3a..54fd6509647 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
@@ -348,7 +348,7 @@ public class TestContainerResourceUsage {
// If keepRunningContainers is false, all live containers should now
// be completed. Calculate the resource usage metrics for all of them.
for (RMContainer c : rmContainers) {
-waitforContainerCompletion(rm, nm, amContainerId, c);
+MockRM.waitForContainerCompletion(rm, nm, amContainerId, c);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
memorySeconds += ru.getMemorySeconds();
vcoreSeconds += ru.getVcoreSeconds();
@@ -400,7 +400,7 @@ public class TestContainerResourceUsage {
// Calculate container usage metrics for second attempt.
for (RMContainer c : rmContainers) {
- waitforContainerCompletion(rm, nm, amContainerId, c);
+ MockRM.waitForContainerCompletion(rm, nm, amContainerId, c);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
memorySeconds += ru.getMemorySeconds();
vcoreSeconds += ru.getVcoreSeconds();
@@ -417,20 +417,6 @@ public class TestContainerResourceUsage {
return;
}
- private void waitforContainerCompletion(MockRM rm, MockNM nm,
- ContainerId amContainerId, RMContainer container) throws Exception {
-ContainerId containerId = container.getContainerId();
-if (null != rm.scheduler.getRMContainer(containerId)) {
- if (containerId.equals(amContainerId)) {
-rm.waitForState(nm, c
[hadoop] branch trunk updated: YARN-10946. AbstractCSQueue: Create separate class for constructing Queue API objects. Contributed by Peter Szucs
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5440c75c4ab YARN-10946. AbstractCSQueue: Create separate class for
constructing Queue API objects. Contributed by Peter Szucs
5440c75c4ab is described below
commit 5440c75c4abee87aeff6da3c7e3e586978cd27dc
Author: Szilard Nemeth
AuthorDate: Thu Dec 1 15:11:58 2022 +0100
YARN-10946. AbstractCSQueue: Create separate class for constructing Queue
API objects. Contributed by Peter Szucs
---
.../scheduler/capacity/AbstractCSQueue.java| 80 +-
.../scheduler/capacity/CSQueueInfoProvider.java| 117 +
2 files changed, 118 insertions(+), 79 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 1a5a1ce0fd4..7a3ec4f6f4e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -29,10 +29,8 @@ import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
-import org.apache.hadoop.yarn.api.records.QueueConfigurations;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
-import org.apache.hadoop.yarn.api.records.QueueStatistics;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -584,83 +582,7 @@ public abstract class AbstractCSQueue implements CSQueue {
// from schedulerApplicationAttempt, to avoid deadlock, sacrifice
// consistency here.
// TODO, improve this
-QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
-queueInfo.setQueueName(queuePath.getLeafName());
-queueInfo.setQueuePath(queuePath.getFullPath());
-
queueInfo.setAccessibleNodeLabels(queueNodeLabelsSettings.getAccessibleNodeLabels());
-queueInfo.setCapacity(queueCapacities.getCapacity());
-queueInfo.setMaximumCapacity(queueCapacities.getMaximumCapacity());
-queueInfo.setQueueState(getState());
-
queueInfo.setDefaultNodeLabelExpression(queueNodeLabelsSettings.getDefaultLabelExpression());
-queueInfo.setCurrentCapacity(getUsedCapacity());
-queueInfo.setQueueStatistics(getQueueStatistics());
-queueInfo.setPreemptionDisabled(preemptionSettings.isPreemptionDisabled());
-queueInfo.setIntraQueuePreemptionDisabled(
-getIntraQueuePreemptionDisabled());
-queueInfo.setQueueConfigurations(getQueueConfigurations());
-queueInfo.setWeight(queueCapacities.getWeight());
-
queueInfo.setMaxParallelApps(queueAppLifetimeSettings.getMaxParallelApps());
-return queueInfo;
- }
-
- public QueueStatistics getQueueStatistics() {
-// Deliberately doesn't use lock here, because this method will be invoked
-// from schedulerApplicationAttempt, to avoid deadlock, sacrifice
-// consistency here.
-// TODO, improve this
-QueueStatistics stats = recordFactory.newRecordInstance(
-QueueStatistics.class);
-stats.setNumAppsSubmitted(getMetrics().getAppsSubmitted());
-stats.setNumAppsRunning(getMetrics().getAppsRunning());
-stats.setNumAppsPending(getMetrics().getAppsPending());
-stats.setNumAppsCompleted(getMetrics().getAppsCompleted());
-stats.setNumAppsKilled(getMetrics().getAppsKilled());
-stats.setNumAppsFailed(getMetrics().getAppsFailed());
-stats.setNumActiveUsers(getMetrics().getActiveUsers());
-stats.setAvailableMemoryMB(getMetrics().getAvailableMB());
-stats.setAllocatedMemoryMB(getMetrics().getAllocatedMB());
-stats.setPendingMemoryMB(getMetrics().getPendingMB());
-stats.setReservedMemoryMB(getMetrics().getReservedMB());
-stats.setAvailableVCores(getMetrics().getAvailableVirtualCores());
-stats.setAllocatedVCores(getMetrics().getAllocatedVirtualCores());
-stats.setPendingVCores(getMetrics().getPendingVirtualCores());
-stats.setReservedVCores(getMetrics().getReservedVirtualCores());
-stats.setPendingContainers(getMetrics().getPendingContainers());
-stats.setAllocatedContainers(getMetrics
[hadoop] branch trunk updated: YARN-8262. get_executable in container-executor should provide meaningful error codes. Contributed by Susheel Gupta
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3c37a01654e YARN-8262. get_executable in container-executor should
provide meaningful error codes. Contributed by Susheel Gupta
3c37a01654e is described below
commit 3c37a01654e0ddca3d88a9fe7ba7b0830ac44ea6
Author: Szilard Nemeth
AuthorDate: Tue Nov 22 13:37:55 2022 +0100
YARN-8262. get_executable in container-executor should provide meaningful
error codes. Contributed by Susheel Gupta
---
.../server/nodemanager/LinuxContainerExecutor.java | 8 ++--
.../container-executor/impl/get_executable.c | 22 +++---
.../src/main/native/container-executor/impl/util.c | 10 ++
.../src/main/native/container-executor/impl/util.h | 7 ++-
4 files changed, 33 insertions(+), 14 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index e899215291b..ea4595dffc4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -175,8 +175,12 @@ public class LinuxContainerExecutor extends
ContainerExecutor {
COULD_NOT_CREATE_WORK_DIRECTORIES(35),
COULD_NOT_CREATE_APP_LOG_DIRECTORIES(36),
COULD_NOT_CREATE_TMP_DIRECTORIES(37),
-ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS(38);
-
+ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS(38),
+CANNOT_GET_EXECUTABLE_NAME_FROM_READLINK(80),
+TOO_LONG_EXECUTOR_PATH(81),
+CANNOT_GET_EXECUTABLE_NAME_FROM_KERNEL(82),
+CANNOT_GET_EXECUTABLE_NAME_FROM_PID(83),
+WRONG_PATH_OF_EXECUTABLE(84);
private final int code;
ExitCode(int exitCode) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
index e1ec293cd47..b027e51bd31 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
@@ -56,17 +56,17 @@ char *__get_exec_readproc(char *procfn) {
filename = malloc(EXECUTOR_PATH_MAX);
if (!filename) {
fprintf(ERRORFILE,"cannot allocate memory for filename before readlink:
%s\n",strerror(errno));
-exit(-1);
+exit(OUT_OF_MEMORY);
}
len = readlink(procfn, filename, EXECUTOR_PATH_MAX);
if (len == -1) {
-fprintf(ERRORFILE,"Can't get executable name from %s - %s\n", procfn,
+fprintf(ERRORFILE,"Cannot get executable name from %s - %s\n", procfn,
strerror(errno));
-exit(-1);
+exit(CANNOT_GET_EXECUTABLE_NAME_FROM_READLINK);
} else if (len >= EXECUTOR_PATH_MAX) {
fprintf(ERRORFILE,"Resolved path for %s [%s] is longer than %d
characters.\n",
procfn, filename, EXECUTOR_PATH_MAX);
-exit(-1);
+exit(TOO_LONG_EXECUTOR_PATH);
}
filename[len] = '\0';
return filename;
@@ -88,14 +88,14 @@ char *__get_exec_sysctl(int *mib)
len = sizeof(buffer);
if (sysctl(mib, 4, buffer, , NULL, 0) == -1) {
-fprintf(ERRORFILE,"Can't get executable name from kernel: %s\n",
+fprintf(ERRORFILE,"Cannot get executable name from kernel: %s\n",
strerror(errno));
-exit(-1);
+exit(CANNOT_GET_EXECUTABLE_NAME_FROM_KERNEL);
}
filename=malloc(EXECUTOR_PATH_MAX);
if (!filename) {
fprintf(ERRORFILE,"cannot allocate memory for filename after sysctl:
%s\n",strerror(errno));
-exit(-1);
+exit(OUT_OF_MEMORY);
}
snprintf(filename,EXECUTOR_PATH_MAX,"%s",buffer);
return filename;
@@ -120,13 +120,13 @@ char* get_executable(char *argv0) {
filename = malloc(PROC_PIDPATHINFO_MAXSIZE);
if (!filename) {
fprintf(ERRORFILE,"cannot allocate memory for filename before
proc_pidpath: %s\n",strerror(errno));
-exit(-1);
+exit(OUT_OF_MEMORY);
}
pid = getpid();
if (proc_pidpath(pid,filename,PROC_PIDPATHINFO_MAXSIZE) <= 0) {
-fprintf(ERRORFILE,"Can't get executable name
[hadoop] branch trunk updated: YARN-11369. Commons.compress throws an IllegalArgumentException with large uids after 1.21. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 142df247eda YARN-11369. Commons.compress throws an
IllegalArgumentException with large uids after 1.21. Contributed by Benjamin
Teke
142df247eda is described below
commit 142df247edab25f0f1e829ea8559fb42a925af7b
Author: Szilard Nemeth
AuthorDate: Wed Nov 16 13:07:05 2022 +0100
YARN-11369. Commons.compress throws an IllegalArgumentException with large
uids after 1.21. Contributed by Benjamin Teke
---
.../main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java | 2 ++
1 file changed, 2 insertions(+)
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
index 862d68ebc0a..52b6dde3794 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
@@ -331,6 +331,8 @@ public class FrameworkUploader implements Runnable {
LOG.info("Compressing tarball");
try (TarArchiveOutputStream out = new TarArchiveOutputStream(
targetStream)) {
+ // Workaround for the compress issue present from 1.21: COMPRESS-587
+ out.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR);
for (String fullPath : filteredInputFiles) {
LOG.info("Adding " + fullPath);
File file = new File(fullPath);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10005. Code improvements in MutableCSConfigurationProvider. Contributed by Peter Szucs
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 22c9f28f4d8 YARN-10005. Code improvements in
MutableCSConfigurationProvider. Contributed by Peter Szucs
22c9f28f4d8 is described below
commit 22c9f28f4d85a92d23024b03c9094edb8e56f369
Author: Szilard Nemeth
AuthorDate: Sat Nov 12 18:35:07 2022 +0100
YARN-10005. Code improvements in MutableCSConfigurationProvider.
Contributed by Peter Szucs
---
.../conf/ConfigurationUpdateAssembler.java | 181
.../conf/MutableCSConfigurationProvider.java | 188 ++---
.../conf/TestConfigurationUpdateAssembler.java | 173 +++
3 files changed, 370 insertions(+), 172 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java
new file mode 100644
index 000..88c93019680
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY;
+
+public final class ConfigurationUpdateAssembler {
+
+ private ConfigurationUpdateAssembler() {
+ }
+
+ public static Map constructKeyValueConfUpdate(
+ CapacitySchedulerConfiguration proposedConf,
+ SchedConfUpdateInfo mutationInfo) throws IOException {
+
+Map confUpdate = new HashMap<>();
+for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
+ removeQueue(queueToRemove, proposedConf, confUpdate);
+}
+for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
+ addQueue(addQueueInfo, proposedConf, confUpdate);
+}
+for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
+ updateQueue(updateQueueInfo, proposedConf, confUpdate);
+}
+for (Map.Entry global : mutationInfo.getGlobalParams()
+.entrySet()) {
+ confUpdate.put(global.getKey(), global.getValue());
+}
+return confUpdate;
+ }
+
+ private static void removeQueue(
+ String queueToRemove, CapacitySchedulerConfiguration proposedConf,
+ Map confUpdate) throws IOException {
+if (queueToRemove == null) {
+ return;
+}
+if (queueToRemove.lastIndexOf('.') == -1) {
+ throw new IOException("Can't remove queue " + queueToRemove);
+}
+String queueName = queueToRemove.substring(
+queueToRemove.lastIndexOf('.') + 1);
+List siblingQueues = getSiblingQueues(queueToRemove,
+proposedConf);
+if (!siblingQueues.contains(queueName)) {
+ throw new IOException("Queue " + queueToRemove + " not found");
+}
+siblingQueues.remove(queueName);
+String parentQueuePath = queueToRemove.substring(0, queueToRemove
+.lastIndexOf('.'));
+proposedConf.setQueues(parentQueuePath, siblingQ
[hadoop] branch trunk updated: YARN-10680. Revisit try blocks without catch blocks but having finally blocks. Contributed by Susheel Gupta
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b0d5182c310 YARN-10680. Revisit try blocks without catch blocks but
having finally blocks. Contributed by Susheel Gupta
b0d5182c310 is described below
commit b0d5182c31043d70d614d3378c42974d6f468b27
Author: Szilard Nemeth
AuthorDate: Sat Oct 15 21:51:08 2022 +0200
YARN-10680. Revisit try blocks without catch blocks but having finally
blocks. Contributed by Susheel Gupta
---
.../apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java | 10 +-
.../org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java | 10 +-
.../org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java | 8 +++-
3 files changed, 25 insertions(+), 3 deletions(-)
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index 18e12cca05f..6cc4cf7597e 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -36,6 +36,8 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.sls.SLSRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@Private
@Unstable
@@ -45,6 +47,7 @@ public class SLSCapacityScheduler extends CapacityScheduler
implements
private final SLSSchedulerCommons schedulerCommons;
private Configuration conf;
private SLSRunner runner;
+ private static final Logger LOG =
LoggerFactory.getLogger(SLSCapacityScheduler.class);
public SLSCapacityScheduler() {
schedulerCommons = new SLSSchedulerCommons(this);
@@ -105,7 +108,12 @@ public class SLSCapacityScheduler extends
CapacityScheduler implements
@Override
public void handle(SchedulerEvent schedulerEvent) {
-schedulerCommons.handle(schedulerEvent);
+try {
+ schedulerCommons.handle(schedulerEvent);
+} catch(Exception e) {
+ LOG.error("Caught exception while handling scheduler event", e);
+ throw e;
+}
}
@Override
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
index 1b4d5ced69b..beb411025f8 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -31,6 +31,8 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.sls.SLSRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.List;
@@ -40,6 +42,7 @@ public class SLSFairScheduler extends FairScheduler
implements SchedulerWrapper, Configurable {
private final SLSSchedulerCommons schedulerCommons;
private SLSRunner runner;
+ private static final Logger LOG =
LoggerFactory.getLogger(SLSFairScheduler.class);
public SLSFairScheduler() {
schedulerCommons = new SLSSchedulerCommons(this);
@@ -63,7 +66,12 @@ public class SLSFairScheduler extends FairScheduler
@Override
public void handle(SchedulerEvent schedulerEvent) {
-schedulerCommons.handle(schedulerEvent);
+try {
+ schedulerCommons.handle(schedulerEvent);
+} catch (Exception e){
+ LOG.error("Caught exception while handling scheduler event", e);
+ throw e;
+}
}
@Override
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
index a1e530a6f77..f66cf4384d9 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
@@ -178,7 +178,7 @@ public abstract class SchedulerMetrics {
pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000,
TimeUnit.MILLISECONDS);
-// a thread to output metrics for real-tiem tracking
+// a thread to output metrics fo
[hadoop] branch trunk updated: YARN-6766. Add helper method in FairSchedulerAppsBlock to print app info. Contributed by Riya Khandelwal
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 0c515b0ef09 YARN-6766. Add helper method in FairSchedulerAppsBlock to
print app info. Contributed by Riya Khandelwal
0c515b0ef09 is described below
commit 0c515b0ef09b21fcd3e800bf54c1a7920ebf23fc
Author: Szilard Nemeth
AuthorDate: Mon Oct 10 15:28:27 2022 +0200
YARN-6766. Add helper method in FairSchedulerAppsBlock to print app info.
Contributed by Riya Khandelwal
---
.../webapp/FairSchedulerAppsBlock.java | 26 +-
1 file changed, 16 insertions(+), 10 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index f6202cbcc51..c74e2ae3e1b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -129,6 +129,12 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
return true;
}
+ private static String printAppInfo(long value) {
+if (value == -1) {
+ return "N/A";
+}
+return String.valueOf(value);
+ }
@Override public void render(Block html) {
TBODY> tbody = html.
@@ -193,16 +199,16 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
.append(appInfo.getFinishTime()).append("\",\"")
.append(appInfo.getState()).append("\",\"")
.append(appInfo.getFinalStatus()).append("\",\"")
- .append(appInfo.getRunningContainers() == -1 ? "N/A" : String
- .valueOf(appInfo.getRunningContainers())).append("\",\"")
- .append(appInfo.getAllocatedVCores() == -1 ? "N/A" : String
-.valueOf(appInfo.getAllocatedVCores())).append("\",\"")
- .append(appInfo.getAllocatedMB() == -1 ? "N/A" : String
-.valueOf(appInfo.getAllocatedMB())).append("\",\"")
- .append(appInfo.getReservedVCores() == -1 ? "N/A" : String
-.valueOf(appInfo.getReservedVCores())).append("\",\"")
- .append(appInfo.getReservedMB() == -1 ? "N/A" : String
-.valueOf(appInfo.getReservedMB())).append("\",\"")
+ .append(printAppInfo(appInfo.getRunningContainers()))
+ .append("\",\"")
+ .append(printAppInfo(appInfo.getAllocatedVCores()))
+ .append("\",\"")
+ .append(printAppInfo(appInfo.getAllocatedMB()))
+ .append("\",\"")
+ .append(printAppInfo(appInfo.getReservedVCores()))
+ .append("\",\"")
+ .append(printAppInfo(appInfo.getReservedMB()))
+ .append("\",\"")
// Progress bar
.append("
[hadoop] branch trunk updated: YARN-11211. QueueMetrics leaks Configuration objects when validation API is called multiple times. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new f4b635c4dcb YARN-11211. QueueMetrics leaks Configuration objects when
validation API is called multiple times. Contributed by Andras Gyori
f4b635c4dcb is described below
commit f4b635c4dcb44fb180605d2c540b52eef7b8bde0
Author: Szilard Nemeth
AuthorDate: Thu Jul 21 14:20:34 2022 +0200
YARN-11211. QueueMetrics leaks Configuration objects when validation API is
called multiple times. Contributed by Andras Gyori
---
.../scheduler/PartitionQueueMetrics.java| 2 +-
.../server/resourcemanager/scheduler/QueueMetrics.java | 17 ++---
.../scheduler/capacity/CSQueueMetrics.java | 7 ++-
.../capacity/CapacitySchedulerConfigValidator.java | 3 +++
4 files changed, 24 insertions(+), 5 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java
index 02eaa7bd9b7..150abdf51d0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java
@@ -78,7 +78,7 @@ public class PartitionQueueMetrics extends QueueMetrics {
metrics = new PartitionQueueMetrics(this.metricsSystem, this.queueName,
null, false, this.conf, this.partition);
users.put(userName, metrics);
- metricsSystem.register(
+ registerMetrics(
pSourceName(partitionJMXStr).append(qSourceName(queueName))
.append(",user=").append(userName).toString(),
"Metrics for user '" + userName + "' in queue '" + queueName + "'",
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
index d41cbed0b10..0bfee4d3350 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
@@ -289,7 +289,7 @@ public class QueueMetrics implements MetricsSource {
metrics =
new QueueMetrics(metricsSystem, queueName, null, false, conf);
users.put(userName, metrics);
- metricsSystem.register(
+ registerMetrics(
sourceName(queueName).append(",user=").append(userName).toString(),
"Metrics for user '"+ userName +"' in queue '"+ queueName +"'",
metrics.tag(QUEUE_INFO, queueName).tag(USER_INFO, userName));
@@ -334,7 +334,7 @@ public class QueueMetrics implements MetricsSource {
QueueMetrics queueMetrics =
new PartitionQueueMetrics(metricsSystem, this.queueName, parentQueue,
this.enableUserMetrics, this.conf, partition);
- metricsSystem.register(
+ registerMetrics(
pSourceName(partitionJMXStr).append(qSourceName(this.queueName))
.toString(),
"Metrics for queue: " + this.queueName,
@@ -378,7 +378,7 @@ public class QueueMetrics implements MetricsSource {
// Register with the MetricsSystems
if (metricsSystem != null) {
-metricsSystem.register(pSourceName(partitionJMXStr).toString(),
+registerMetrics(pSourceName(partitionJMXStr).toString(),
"Metrics for partition: " + partitionJMXStr,
(PartitionQueueMetrics) metrics.tag(PARTITION_INFO,
partitionJMXStr));
@@ -1359,4 +1359,15 @@ public class QueueMetrics implements MetricsSource {
}
}
}
+
+ protected void registerMetrics(String sourceName, String desc, QueueMetrics
metrics) {
+MetricsSource source = metricsSystem.getSource(sourceName);
+// Unregister metrics if a source is already present
+if (source != null) {
+ LOG.info("Unregistering source " + sourceName);
+
[hadoop] branch trunk updated: YARN-11185. Pending app metrics are increased doubly when a queue reaches its max-parallel-apps limit. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3a66348fda2 YARN-11185. Pending app metrics are increased doubly when
a queue reaches its max-parallel-apps limit. Contributed by Andras Gyori
3a66348fda2 is described below
commit 3a66348fda29b18054853baed6664fec92dc5cc5
Author: Szilard Nemeth
AuthorDate: Mon Jun 20 15:03:58 2022 +0200
YARN-11185. Pending app metrics are increased doubly when a queue reaches
its max-parallel-apps limit. Contributed by Andras Gyori
---
.../scheduler/capacity/AbstractLeafQueue.java | 4 +-
.../capacity/TestCapacitySchedulerApps.java| 77 ++
2 files changed, 51 insertions(+), 30 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
index ac5c8a15167..08fedb578ca 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
@@ -578,6 +578,8 @@ public class AbstractLeafQueue extends AbstractCSQueue {
public void submitApplicationAttempt(FiCaSchedulerApp application,
String userName, boolean isMoveApp) {
// Careful! Locking order is important!
+boolean isAppAlreadySubmitted = applicationAttemptMap.containsKey(
+application.getApplicationAttemptId());
writeLock.lock();
try {
// TODO, should use getUser, use this method just to avoid UT failure
@@ -591,7 +593,7 @@ public class AbstractLeafQueue extends AbstractCSQueue {
}
// We don't want to update metrics for move app
-if (!isMoveApp) {
+if (!isMoveApp && !isAppAlreadySubmitted) {
boolean unmanagedAM = application.getAppSchedulingInfo() != null &&
application.getAppSchedulingInfo().isUnmanagedAM();
usageTracker.getMetrics().submitAppAttempt(userName, unmanagedAM);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java
index ea22c24b355..d192e7dcc69 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
@@ -116,6 +117,8 @@ import static org.mockito.Mockito.when;
public class TestCapacitySchedulerApps {
+ public static final int MAX_PARALLEL_APPS = 5;
+ public static final String USER_0 = "user_0";
private ResourceManager resourceManager = null;
private RMContext mockContext;
@@ -237,18 +240,7 @@ public class TestCapacitySchedulerApps {
YarnScheduler scheduler = rm.getResourceScheduler();
// submit an app
-MockRMAppSubmissionData data =
-MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
-.withAppName("test-move-1")
-.withUser("user_0")
-.withAcls(null)
-.withQueue("a1")
-.withUnmanagedAM(false)
-.build();
-RMApp app = MockRMAppSubmitter.submit(rm, data);
-ApplicationAttemptId appAttemptId =
-rm.getApplicationReport(app.getApplicationId())
-.getCurrentApplicationAttemptId();
+ApplicationAttemptId appAttemptId = submitApp(rm);
// check preconditions
List appsInA1 = scheduler.getAppsInQueue("a1");
@@ -1020,18 +1012,7 @@ public class TestCapacitySchedulerApps {
(AbstractYarnScheduler) rm.getResourceScheduler();
// submit an app
-MockRMAppSubmissionData data =
-MockRMAppSubmissionData.Bui
[hadoop] branch branch-3.3 updated: YARN-11141. Capacity Scheduler does not support ambiguous queue names when moving application across queues. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 90ec4418c76 YARN-11141. Capacity Scheduler does not support ambiguous
queue names when moving application across queues. Contributed by Andras Gyori
90ec4418c76 is described below
commit 90ec4418c769c1aefe236d9cb79ce3c3da1c2601
Author: Szilard Nemeth
AuthorDate: Wed May 18 14:34:08 2022 +0200
YARN-11141. Capacity Scheduler does not support ambiguous queue names when
moving application across queues. Contributed by Andras Gyori
---
.../resourcemanager/scheduler/capacity/CapacityScheduler.java | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index d0d95c388a6..ba1ab940f5a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2531,7 +2531,12 @@ public class CapacityScheduler extends
if (application == null) {
throw new YarnException("App to be moved " + appId + " not found.");
}
- String sourceQueueName = application.getQueue().getQueueName();
+ if (!(application.getQueue() instanceof CSQueue)) {
+throw new YarnException("Source queue is not a Capacity Scheduler
queue");
+ }
+
+ CSQueue csQueue = (CSQueue) application.getQueue();
+ String sourceQueueName = csQueue.getQueuePath();
LeafQueue source =
this.queueManager.getAndCheckLeafQueue(sourceQueueName);
String destQueueName = handleMoveToPlanQueue(targetQueueName);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 4f112e3138c YARN-11126. ZKConfigurationStore Java deserialisation
vulnerability. Contributed by Tamas Domok
4f112e3138c is described below
commit 4f112e3138cba4f74a00f48b268b4180a3eed4f1
Author: Szilard Nemeth
AuthorDate: Wed May 18 14:25:35 2022 +0200
YARN-11126. ZKConfigurationStore Java deserialisation vulnerability.
Contributed by Tamas Domok
---
.../capacity/conf/ZKConfigurationStore.java| 5 ++--
.../capacity/conf/TestZKConfigurationStore.java| 35 ++
2 files changed, 38 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
index 703193b1b29..d55435a4c4b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import org.apache.commons.io.serialization.ValidatingObjectInputStream;
import
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.slf4j.Logger;
@@ -35,7 +36,6 @@ import org.apache.zookeeper.data.ACL;
import java.io.IOException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.HashMap;
import java.util.LinkedList;
@@ -314,7 +314,8 @@ public class ZKConfigurationStore extends
YarnConfigurationStore {
private static Object deserializeObject(byte[] bytes) throws Exception {
try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
-ObjectInputStream ois = new ObjectInputStream(bais);) {
+ ValidatingObjectInputStream ois = new
ValidatingObjectInputStream(bais);) {
+ ois.accept(LinkedList.class, LogMutation.class, HashMap.class,
String.class);
return ois.readObject();
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index 880ba77fa51..155996d11fe 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -42,15 +42,18 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.Yar
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.io.IOException;
import java.io.ByteArrayOutputStream;
import java.io.ObjectOutputStream;
import java.util.Arrays;
+import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
@@ -67,6 +70,9 @@ public class TestZKConfigurationStore extends
LoggerFactory.getLogger(TestZKConfigurationStore.class);
private static final int ZK_TIMEOUT_MS = 1;
+ private static final String DESERIALIZATION_VULNERABILITY_FILEPATH =
+ "/tmp/ZK_DESERIALIZATION_VULNERABILITY";
+
private TestingServer curatorTestingServer;
private CuratorFramework curatorFramework;
private ResourceManager rm;
@@ -401,6 +407,35 @@ public class TestZKConfigurationStore extends
rm2.close();
}
+ @Test(timeout = 3000)
+ @SuppressWarnings("checkstyle:linelength")
+ public void testDeserializationIsNotVulnerabl
[hadoop] branch branch-3.2 updated: YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 45801fba8b0 YARN-11126. ZKConfigurationStore Java deserialisation
vulnerability. Contributed by Tamas Domok
45801fba8b0 is described below
commit 45801fba8b00257ab32c02a7d1a05948ba687a49
Author: Szilard Nemeth
AuthorDate: Wed May 18 14:23:56 2022 +0200
YARN-11126. ZKConfigurationStore Java deserialisation vulnerability.
Contributed by Tamas Domok
---
.../capacity/conf/ZKConfigurationStore.java| 5 ++--
.../capacity/conf/TestZKConfigurationStore.java| 35 ++
2 files changed, 38 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
index 6f3612c28da..cacf32b3911 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.io.serialization.ValidatingObjectInputStream;
import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,7 +36,6 @@ import org.apache.zookeeper.data.ACL;
import java.io.IOException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.HashMap;
import java.util.LinkedList;
@@ -317,7 +317,8 @@ public class ZKConfigurationStore extends
YarnConfigurationStore {
private static Object deserializeObject(byte[] bytes) throws Exception {
try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
-ObjectInputStream ois = new ObjectInputStream(bais);) {
+ ValidatingObjectInputStream ois = new
ValidatingObjectInputStream(bais);) {
+ ois.accept(LinkedList.class, LogMutation.class, HashMap.class,
String.class);
return ois.readObject();
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index 40454a4621b..cb1da9c8963 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -41,15 +41,18 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.io.IOException;
import java.io.ByteArrayOutputStream;
import java.io.ObjectOutputStream;
import java.util.Arrays;
+import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
@@ -66,6 +69,9 @@ public class TestZKConfigurationStore extends
LoggerFactory.getLogger(TestZKConfigurationStore.class);
private static final int ZK_TIMEOUT_MS = 1;
+ private static final String DESERIALIZATION_VULNERABILITY_FILEPATH =
+ "/tmp/ZK_DESERIALIZATION_VULNERABILITY";
+
private TestingServer curatorTestingServer;
private CuratorFramework curatorFramework;
private ResourceManager rm;
@@ -398,6 +404,35 @@ public class TestZKConfigurationStore extends
rm2.close();
}
+ @Test(timeout = 3000)
+ @SuppressWarnings("checkstyle:linelength")
+ public void testDeserializationIsNotVulnerabl
[hadoop] branch branch-3.3 updated: YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new b4550b33564 YARN-10850. TimelineService v2 lists containers for all
attempts when filtering for one. Contributed by Benjamin Teke
b4550b33564 is described below
commit b4550b3356439f012ca6f97383889e236067e689
Author: Szilard Nemeth
AuthorDate: Wed May 18 14:08:41 2022 +0200
YARN-10850. TimelineService v2 lists containers for all attempts when
filtering for one. Contributed by Benjamin Teke
---
.../hadoop/yarn/client/api/impl/AHSv2ClientImpl.java | 5 ++---
.../client/api/impl/TimelineReaderClientImpl.java| 16 ++--
.../api/impl/TestTimelineReaderClientImpl.java | 20 +++-
3 files changed, 35 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
index b6a0c591c90..5fd3ee21fc5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
@@ -137,9 +137,8 @@ public class AHSv2ClientImpl extends AHSClient {
ApplicationId appId = applicationAttemptId.getApplicationId();
ApplicationReport appReport = getApplicationReport(appId);
Map filters = new HashMap<>();
-filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" +
-applicationAttemptId.toString() +
-"\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}");
+filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq "
++ "{\"type\":\"YARN_APPLICATION_ATTEMPT\",\"id\":\"" +
applicationAttemptId + "\"}");
List entities = readerClient.getContainerEntities(
appId, "ALL", filters, 0, null);
List containers =
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
index d5af8a0df56..de30fb84be0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
@@ -38,7 +38,10 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
import java.net.URI;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -208,12 +211,21 @@ public class TimelineReaderClientImpl extends
TimelineReaderClient {
return Arrays.asList(entity);
}
+ @VisibleForTesting
+ protected String encodeValue(String value) throws
UnsupportedEncodingException {
+// Since URLEncoder doesn't use and doesn't have an option for
percent-encoding
+// (as specified in RFC 3986) the spaces are encoded to + signs, which
need to be replaced
+// manually
+return URLEncoder.encode(value, StandardCharsets.UTF_8.toString())
+.replaceAll("\\+", "%20");
+ }
+
private void mergeFilters(MultivaluedMap defaults,
- Map filters) {
+Map filters) throws
UnsupportedEncodingException {
if (filters != null && !filters.isEmpty()) {
for (Map.Entry entry : filters.entrySet()) {
if (!defaults.containsKey(entry.getKey())) {
- defaults.add(entry.getKey(), filters.get(entry.getValue()));
+ defaults.add(entry.getKey(), encodeValue(entry.getValue()));
}
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
index f668472256a..757aeb8c31d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/j
[hadoop] branch branch-3.2 updated: YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 1ae834fe0b1 YARN-10850. TimelineService v2 lists containers for all
attempts when filtering for one. Contributed by Benjamin Teke
1ae834fe0b1 is described below
commit 1ae834fe0b1efb28a4fec56e9532b88864392cbe
Author: Szilard Nemeth
AuthorDate: Wed May 18 14:04:51 2022 +0200
YARN-10850. TimelineService v2 lists containers for all attempts when
filtering for one. Contributed by Benjamin Teke
---
.../hadoop/yarn/client/api/impl/AHSv2ClientImpl.java | 5 ++---
.../client/api/impl/TimelineReaderClientImpl.java| 16 ++--
.../api/impl/TestTimelineReaderClientImpl.java | 20 +++-
3 files changed, 35 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
index e797c281625..3422493aa1a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
@@ -129,9 +129,8 @@ public class AHSv2ClientImpl extends AHSClient {
applicationAttemptId) throws YarnException, IOException {
ApplicationId appId = applicationAttemptId.getApplicationId();
Map filters = new HashMap<>();
-filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" +
-applicationAttemptId.toString() +
-"\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}");
+filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq "
++ "{\"type\":\"YARN_APPLICATION_ATTEMPT\",\"id\":\"" +
applicationAttemptId + "\"}");
List entities = readerClient.getContainerEntities(
appId, "ALL", filters, 0, null);
List containers =
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
index db53f93136f..aafe4c3db11 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
@@ -38,7 +38,10 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
import java.net.URI;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -208,12 +211,21 @@ public class TimelineReaderClientImpl extends
TimelineReaderClient {
return Arrays.asList(entity);
}
+ @VisibleForTesting
+ protected String encodeValue(String value) throws
UnsupportedEncodingException {
+// Since URLEncoder doesn't use and doesn't have an option for
percent-encoding
+// (as specified in RFC 3986) the spaces are encoded to + signs, which
need to be replaced
+// manually
+return URLEncoder.encode(value, StandardCharsets.UTF_8.toString())
+.replaceAll("\\+", "%20");
+ }
+
private void mergeFilters(MultivaluedMap defaults,
- Map filters) {
+Map filters) throws
UnsupportedEncodingException {
if (filters != null && !filters.isEmpty()) {
for (Map.Entry entry : filters.entrySet()) {
if (!defaults.containsKey(entry.getKey())) {
- defaults.add(entry.getKey(), filters.get(entry.getValue()));
+ defaults.add(entry.getKey(), encodeValue(entry.getValue()));
}
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
index f668472256a..d6ca25f9501 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/tes
[hadoop] branch trunk updated: YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5e2f4339fad YARN-11126. ZKConfigurationStore Java deserialisation
vulnerability. Contributed by Tamas Domok
5e2f4339fad is described below
commit 5e2f4339fadc88f20543915fc9b0aaeaf4f9e7bf
Author: Szilard Nemeth
AuthorDate: Thu May 12 13:42:06 2022 +0200
YARN-11126. ZKConfigurationStore Java deserialisation vulnerability.
Contributed by Tamas Domok
---
.../capacity/conf/ZKConfigurationStore.java| 5 ++--
.../capacity/conf/TestZKConfigurationStore.java| 35 ++
2 files changed, 38 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
index 71226c300a8..ad8fb97a7a6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import org.apache.commons.io.serialization.ValidatingObjectInputStream;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.slf4j.Logger;
@@ -35,7 +36,6 @@ import org.apache.zookeeper.data.ACL;
import java.io.IOException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.HashMap;
import java.util.LinkedList;
@@ -314,7 +314,8 @@ public class ZKConfigurationStore extends
YarnConfigurationStore {
private static Object deserializeObject(byte[] bytes) throws Exception {
try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
-ObjectInputStream ois = new ObjectInputStream(bais);) {
+ ValidatingObjectInputStream ois = new
ValidatingObjectInputStream(bais);) {
+ ois.accept(LinkedList.class, LogMutation.class, HashMap.class,
String.class);
return ois.readObject();
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index 880ba77fa51..155996d11fe 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -42,15 +42,18 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.Yar
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.io.IOException;
import java.io.ByteArrayOutputStream;
import java.io.ObjectOutputStream;
import java.util.Arrays;
+import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
@@ -67,6 +70,9 @@ public class TestZKConfigurationStore extends
LoggerFactory.getLogger(TestZKConfigurationStore.class);
private static final int ZK_TIMEOUT_MS = 1;
+ private static final String DESERIALIZATION_VULNERABILITY_FILEPATH =
+ "/tmp/ZK_DESERIALIZATION_VULNERABILITY";
+
private TestingServer curatorTestingServer;
private CuratorFramework curatorFramework;
private ResourceManager rm;
@@ -401,6 +407,35 @@ public class TestZKConfigurationStore extends
rm2.close();
}
+ @Test(timeout = 3000)
+ @SuppressWarnings("checkstyle:linelength")
+ public void testDeserializationIsNotVulnerable() throws Exception {
+confStore.initialize(conf,
[hadoop] branch trunk updated: MAPREDUCE-7379. RMContainerRequestor#makeRemoteRequest has confusing log message. Contributed by Ashutosh Gupta
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new f143e994285 MAPREDUCE-7379. RMContainerRequestor#makeRemoteRequest has
confusing log message. Contributed by Ashutosh Gupta
f143e994285 is described below
commit f143e994285543e2f1a779274f826f38a78a27be
Author: Szilard Nemeth
AuthorDate: Wed May 11 16:55:19 2022 +0200
MAPREDUCE-7379. RMContainerRequestor#makeRemoteRequest has confusing log
message. Contributed by Ashutosh Gupta
---
.../hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java | 10 --
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
index 61cc2eb898e..16ca585e9b4 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
@@ -213,12 +213,10 @@ public abstract class RMContainerRequestor extends
RMCommunicator {
allocateResponse.getCompletedContainersStatuses().size();
if (ask.size() > 0 || release.size() > 0) {
- LOG.info("getResources() for " + applicationId + ":" + " ask="
- + ask.size() + " release= " + release.size() + " newContainers="
- + allocateResponse.getAllocatedContainers().size()
- + " finishedContainers=" + numCompletedContainers
- + " resourcelimit=" + availableResources + " knownNMs="
- + clusterNmCount);
+ LOG.info("applicationId={}: ask={} release={} newContainers={}
finishedContainers={}"
+ + " resourceLimit={} knownNMs={}", applicationId, ask.size(),
release.size(),
+ allocateResponse.getAllocatedContainers().size(),
numCompletedContainers,
+ availableResources, clusterNmCount);
}
ask.clear();
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new eeedc0c7e41 YARN-10850. TimelineService v2 lists containers for all
attempts when filtering for one. Contributed by Benjamin Teke
eeedc0c7e41 is described below
commit eeedc0c7e41c7932169af2b05556ca155c765b22
Author: Szilard Nemeth
AuthorDate: Wed May 11 14:39:42 2022 +0200
YARN-10850. TimelineService v2 lists containers for all attempts when
filtering for one. Contributed by Benjamin Teke
---
.../hadoop/yarn/client/api/impl/AHSv2ClientImpl.java | 5 ++---
.../client/api/impl/TimelineReaderClientImpl.java| 16 ++--
.../api/impl/TestTimelineReaderClientImpl.java | 20 +++-
3 files changed, 35 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
index e3653baa2b7..26f45baaee0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
@@ -137,9 +137,8 @@ public class AHSv2ClientImpl extends AHSClient {
ApplicationId appId = applicationAttemptId.getApplicationId();
ApplicationReport appReport = getApplicationReport(appId);
Map filters = new HashMap<>();
-filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" +
-applicationAttemptId.toString() +
-"\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}");
+filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq "
++ "{\"type\":\"YARN_APPLICATION_ATTEMPT\",\"id\":\"" +
applicationAttemptId + "\"}");
List entities = readerClient.getContainerEntities(
appId, "ALL", filters, 0, null);
List containers =
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
index 29609b955fb..71bf13220b0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
@@ -38,7 +38,10 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
import java.net.URI;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -208,12 +211,21 @@ public class TimelineReaderClientImpl extends
TimelineReaderClient {
return Arrays.asList(entity);
}
+ @VisibleForTesting
+ protected String encodeValue(String value) throws
UnsupportedEncodingException {
+// Since URLEncoder doesn't use and doesn't have an option for
percent-encoding
+// (as specified in RFC 3986) the spaces are encoded to + signs, which
need to be replaced
+// manually
+return URLEncoder.encode(value, StandardCharsets.UTF_8.toString())
+.replaceAll("\\+", "%20");
+ }
+
private void mergeFilters(MultivaluedMap defaults,
- Map filters) {
+Map filters) throws
UnsupportedEncodingException {
if (filters != null && !filters.isEmpty()) {
for (Map.Entry entry : filters.entrySet()) {
if (!defaults.containsKey(entry.getKey())) {
- defaults.add(entry.getKey(), filters.get(entry.getValue()));
+ defaults.add(entry.getKey(), encodeValue(entry.getValue()));
}
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
index f668472256a..757aeb8c31d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/j
[hadoop] branch trunk updated: YARN-11141. Capacity Scheduler does not support ambiguous queue names when moving application across queues. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b62d6ce6fd6 YARN-11141. Capacity Scheduler does not support ambiguous
queue names when moving application across queues. Contributed by Andras Gyori
b62d6ce6fd6 is described below
commit b62d6ce6fd6cd568e9b9a8729a03c7f773875fca
Author: Szilard Nemeth
AuthorDate: Wed May 11 14:27:34 2022 +0200
YARN-11141. Capacity Scheduler does not support ambiguous queue names when
moving application across queues. Contributed by Andras Gyori
---
.../scheduler/capacity/CapacityScheduler.java | 7 +++-
.../capacity/CapacitySchedulerQueueHelpers.java| 30
.../capacity/CapacitySchedulerTestUtilities.java | 11 ++
.../capacity/TestCapacitySchedulerApps.java| 41 ++
4 files changed, 88 insertions(+), 1 deletion(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 44e80a6c234..cf5034ba228 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2678,7 +2678,12 @@ public class CapacityScheduler extends
if (application == null) {
throw new YarnException("App to be moved " + appId + " not found.");
}
- String sourceQueueName = application.getQueue().getQueueName();
+ if (!(application.getQueue() instanceof CSQueue)) {
+throw new YarnException("Source queue is not a Capacity Scheduler
queue");
+ }
+
+ CSQueue csQueue = (CSQueue) application.getQueue();
+ String sourceQueueName = csQueue.getQueuePath();
AbstractLeafQueue source =
this.queueManager.getAndCheckLeafQueue(sourceQueueName);
String destQueueName = handleMoveToPlanQueue(targetQueueName);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java
index bc7e2b317ac..7e362731f86 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java
@@ -29,6 +29,7 @@ public final class CapacitySchedulerQueueHelpers {
public static final String A = CapacitySchedulerConfiguration.ROOT + ".a";
public static final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+ public static final String A_CHILD = A + ".a";
public static final String A1 = A + ".a1";
public static final String A2 = A + ".a2";
public static final String B1 = B + ".b1";
@@ -89,6 +90,35 @@ public final class CapacitySchedulerQueueHelpers {
return conf;
}
+ /**
+ * @param conf, to be modified
+ * @return CS configuration which has deleted all children of queue(b)
+ * root
+ * / \
+ *ab
+ * / \
+ * a1 a2
+ */
+ public static CapacitySchedulerConfiguration setupQueueConfAmbiguousQueue(
+ CapacitySchedulerConfiguration conf) {
+
+// Define top-level queues
+conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+new String[]{"a", "b"});
+
+conf.setCapacity(A, A_CAPACITY);
+conf.setCapacity(B, B_CAPACITY);
+
+// Define 2nd-level queues
+conf.setQueues(A, new String[]{"a", "a1"});
+conf.setCapacity(A_CHILD, A1_CAPACITY);
+conf.setUserLimitFactor(A1, 100.0f);
+conf.setCapacity(A1, A2_CAPACITY);
+conf.setUserLimitFactor(A2, 100.0f);
+
+return conf;
+ }
+
/**
* @param conf, to be modified
* @return CS configuration which has deleted all childred of que
[hadoop] branch trunk updated: YARN-11086. Add space in debug log of ParentQueue. Contributed by Junfan Zhang.
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e2701e2 YARN-11086. Add space in debug log of ParentQueue.
Contributed by Junfan Zhang.
e2701e2 is described below
commit e2701e227f82b89c9dcde53ef8fa49f757a61fa8
Author: Szilard Nemeth
AuthorDate: Mon Mar 21 14:37:10 2022 +0100
YARN-11086. Add space in debug log of ParentQueue. Contributed by Junfan
Zhang.
---
.../yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 87ebc0b..c0ad7f8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -465,7 +465,7 @@ public class ParentQueue extends AbstractCSQueue {
"numChildQueue= " + childQueues.size() + ", " +
getCapacityOrWeightString() + ", " +
"absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " +
-"usedResources=" + usageTracker.getQueueUsage().getUsed() +
+"usedResources=" + usageTracker.getQueueUsage().getUsed() + ", " +
"usedCapacity=" + getUsedCapacity() + ", " +
"numApps=" + getNumApplications() + ", " +
"numContainers=" + getNumContainers();
@@ -1148,7 +1148,7 @@ public class ParentQueue extends AbstractCSQueue {
StringBuilder sb = new StringBuilder();
for (CSQueue q : childQueues) {
sb.append(q.getQueuePath() +
- "usedCapacity=(" + q.getUsedCapacity() + "), " +
+ " usedCapacity=(" + q.getUsedCapacity() + "), " +
" label=("
+ StringUtils.join(q.getAccessibleNodeLabels().iterator(), ",")
+ ")");
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11089. Fix typo in RM audit log. Contributed by Junfan Zhang.
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4537b34 YARN-11089. Fix typo in RM audit log. Contributed by Junfan
Zhang.
4537b34 is described below
commit 4537b34e1c088f2b6d61c9bad8b82438cd94b944
Author: Szilard Nemeth
AuthorDate: Mon Mar 21 14:18:27 2022 +0100
YARN-11089. Fix typo in RM audit log. Contributed by Junfan Zhang.
---
.../org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 7ebcd65..928eeb1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -294,7 +294,7 @@ public class RMAppManager implements
EventHandler,
protected void writeAuditLog(ApplicationId appId) {
RMApp app = rmContext.getRMApps().get(appId);
-String operation = "UNKONWN";
+String operation = "UNKNOWN";
boolean success = false;
switch (app.getState()) {
case FAILED:
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11067. Resource overcommitment due to incorrect resource normalisation logical order. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new ed65aa2 YARN-11067. Resource overcommitment due to incorrect resource
normalisation logical order. Contributed by Andras Gyori
ed65aa2 is described below
commit ed65aa23240b3dd6b56e86e5f0e9d38069fb3b01
Author: Szilard Nemeth
AuthorDate: Thu Mar 10 22:22:58 2022 +0100
YARN-11067. Resource overcommitment due to incorrect resource normalisation
logical order. Contributed by Andras Gyori
---
.../scheduler/capacity/ParentQueue.java| 37 -
.../TestAbsoluteResourceConfiguration.java | 47 ++
2 files changed, 64 insertions(+), 20 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index c624aab..87ebc0b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1294,17 +1294,24 @@ public class ParentQueue extends AbstractCSQueue {
private void calculateEffectiveResourcesAndCapacity(String label,
Resource clusterResource) {
+// Update effective resources for my self;
+if (rootQueue) {
+ Resource resourceByLabel = labelManager.getResourceByLabel(label,
clusterResource);
+ usageTracker.getQueueResourceQuotas().setEffectiveMinResource(label,
resourceByLabel);
+ usageTracker.getQueueResourceQuotas().setEffectiveMaxResource(label,
resourceByLabel);
+} else {
+ super.updateEffectiveResources(clusterResource);
+}
+
+recalculateEffectiveMinRatio(label, clusterResource);
+ }
+
+ private void recalculateEffectiveMinRatio(String label, Resource
clusterResource) {
// For root queue, ensure that max/min resource is updated to latest
// cluster resource.
-Resource resourceByLabel = labelManager.getResourceByLabel(label,
-clusterResource);
-
-/*
- * == Below logic are added to calculate effectiveMinRatioPerResource ==
- */
+Resource resourceByLabel = labelManager.getResourceByLabel(label,
clusterResource);
-// Total configured min resources of direct children of this given parent
-// queue
+// Total configured min resources of direct children of this given parent
queue
Resource configuredMinResources = Resource.newInstance(0L, 0);
for (CSQueue childQueue : getChildQueues()) {
Resources.addTo(configuredMinResources,
@@ -1312,8 +1319,7 @@ public class ParentQueue extends AbstractCSQueue {
}
// Factor to scale down effective resource: When cluster has sufficient
-// resources, effective_min_resources will be same as configured
-// min_resources.
+// resources, effective_min_resources will be same as configured
min_resources.
Resource numeratorForMinRatio = null;
if (getQueuePath().equals("root")) {
if (!resourceByLabel.equals(Resources.none()) &&
Resources.lessThan(resourceCalculator,
@@ -1324,21 +1330,12 @@ public class ParentQueue extends AbstractCSQueue {
if (Resources.lessThan(resourceCalculator, clusterResource,
usageTracker.getQueueResourceQuotas().getEffectiveMinResource(label),
configuredMinResources)) {
-numeratorForMinRatio = usageTracker.getQueueResourceQuotas()
-.getEffectiveMinResource(label);
+numeratorForMinRatio =
usageTracker.getQueueResourceQuotas().getEffectiveMinResource(label);
}
}
effectiveMinResourceRatio.put(label, getEffectiveMinRatio(
configuredMinResources, numeratorForMinRatio));
-
-// Update effective resources for my self;
-if (rootQueue) {
- usageTracker.getQueueResourceQuotas().setEffectiveMinResource(label,
resourceByLabel);
- usageTracker.getQueueResourceQuotas().setEffectiveMaxResource(label,
resourceByLabel);
-} else{
- super.updateEffectiveResources(clusterResource);
-}
}
private Map getEffectiveMinRatio(
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager
[hadoop] branch trunk updated: YARN-10049. FIFOOrderingPolicy Improvements. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 481da19 YARN-10049. FIFOOrderingPolicy Improvements. Contributed by
Benjamin Teke
481da19 is described below
commit 481da19494fe13ca42651305b519e0670cafabf0
Author: Szilard Nemeth
AuthorDate: Thu Mar 10 22:15:35 2022 +0100
YARN-10049. FIFOOrderingPolicy Improvements. Contributed by Benjamin Teke
---
.../scheduler/policy/FifoComparator.java | 5 ++
.../scheduler/policy/TestFifoOrderingPolicy.java | 91 --
2 files changed, 71 insertions(+), 25 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
index 112c50f..c62b738 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
@@ -29,6 +29,11 @@ public class FifoComparator
@Override
public int compare(SchedulableEntity r1, SchedulableEntity r2) {
int res = r1.compareInputOrderTo(r2);
+
+if (res == 0) {
+ res = (int) Math.signum(r1.getStartTime() - r2.getStartTime());
+}
+
return res;
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
index 7ec2c01..62bc712 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
@@ -18,16 +18,19 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy;
-import java.util.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
-import org.junit.Assert;
-import org.junit.Test;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.junit.Assert;
+import org.junit.Test;
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class TestFifoOrderingPolicy {
+public
+class TestFifoOrderingPolicy {
@Test
public void testFifoOrderingPolicy() {
@@ -36,13 +39,17 @@ public class TestFifoOrderingPolicy {
MockSchedulableEntity r1 = new MockSchedulableEntity();
MockSchedulableEntity r2 = new MockSchedulableEntity();
-assertThat(policy.getComparator().compare(r1, r2)).isEqualTo(0);
+assertEquals("The comparator should return 0 because the entities are
created with " +
+"the same values.", 0,
+policy.getComparator().compare(r1, r2));
r1.setSerial(1);
-assertThat(policy.getComparator().compare(r1, r2)).isEqualTo(1);
+assertEquals("The lhs entity has a larger serial, the comparator return " +
+"value should be 1.", 1, policy.getComparator().compare(r1, r2));
r2.setSerial(2);
-assertThat(policy.getComparator().compare(r1, r2)).isEqualTo(-1);
+Assert.assertEquals("The rhs entity has a larger serial, the comparator
return " +
+"value should be -1.", -1, policy.getComparator().compare(r1, r2));
}
@Test
@@ -63,46 +70,80 @@ public class TestFifoOrderingPolicy {
schedOrder.addSchedulableEntity(msp3);
//Assignment, oldest to youngest
-
checkSerials(schedOrder.getAssignmentIterator(IteratorSelector.EMPTY_ITERATOR_SELECTOR),
new long[]{1, 2, 3});
+checkSerials(Arrays.asList(1L, 2L, 3L), schedOrder.getAssignmentIterator(
+IteratorSelector.EMPTY_ITERATOR_SELECTOR));
//Preemption, youngest to oldest
-checkSerials(schedOrder.getPreemptionIterator(), new long[]{3, 2, 1});
+checkSerials(Arrays.asList(3L, 2L, 1L),
schedOrder.getPreemptionIterator());
}
- public void checkSerials(
[hadoop] branch trunk updated: YARN-10918. Simplify method: CapacitySchedulerQueueManager#parseQueue. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new db8ae4b YARN-10918. Simplify method:
CapacitySchedulerQueueManager#parseQueue. Contributed by Andras Gyori
db8ae4b is described below
commit db8ae4b65448c506c9234641b2c1f9b8e894dc18
Author: Szilard Nemeth
AuthorDate: Wed Mar 9 19:36:09 2022 +0100
YARN-10918. Simplify method: CapacitySchedulerQueueManager#parseQueue.
Contributed by Andras Gyori
---
.../scheduler/capacity/CSQueue.java| 7 ++
.../capacity/CapacitySchedulerQueueManager.java| 116 -
.../scheduler/capacity/PlanQueue.java | 19
.../scheduler/capacity/ReservationQueue.java | 6 ++
4 files changed, 74 insertions(+), 74 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 90cb4f3..e2aeaab 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -95,6 +95,13 @@ public interface CSQueue extends SchedulerQueue {
*/
QueuePath getQueuePathObject();
+ /**
+ * Checks whether the queue is a dynamic queue (created dynamically in the
fashion of auto queue
+ * creation v2).
+ * @return true, if it is a dynamic queue, false otherwise
+ */
+ boolean isDynamicQueue();
+
public PrivilegedEntity getPrivilegedEntity();
Resource getMaximumAllocation();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index 50e8d52..29e5de0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -43,13 +43,11 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.Permission;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueueManager;
-import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
import
org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.classification.VisibleForTesting;
@@ -231,99 +229,62 @@ public class CapacitySchedulerQueueManager implements
SchedulerQueueManager<
* @throws IOException
*/
static CSQueue parseQueue(
- CapacitySchedulerQueueContext queueContext,
- CapacitySchedulerConfiguration conf,
- CSQueue parent, String queueName,
- CSQueueStore newQueues,
- CSQueueStore oldQueues,
+ CapacitySchedulerQueueContext queueContext,
CapacitySchedulerConfiguration conf,
+ CSQueue parent, String queueName, CSQueueStore newQueues, CSQueueStore
oldQueues,
QueueHook hook) throws IOException {
CSQueue queue;
-String fullQueueName = (parent == null) ?
-queueName :
-(parent.getQueuePath() + "." + queueName);
+String fullQueueName = (parent == null) ? queueName :
+(QueuePath.createFromQueues(parent.getQueuePath(),
queueName).getFullPath());
String[] staticChildQueueNames = conf.getQueues(fullQueueName);
List ch
[hadoop] branch trunk updated: YARN-10945. Add javadoc to all methods of AbstractCSQueue. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2ece950 YARN-10945. Add javadoc to all methods of AbstractCSQueue.
Contributed by Andras Gyori
2ece950 is described below
commit 2ece95064b9bf11ec6a165baf6de7ced4986747f
Author: Szilard Nemeth
AuthorDate: Wed Mar 9 19:29:53 2022 +0100
YARN-10945. Add javadoc to all methods of AbstractCSQueue. Contributed by
Andras Gyori
---
.../scheduler/capacity/AbstractCSQueue.java| 85 +-
1 file changed, 81 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index b0ab336..87fa9da 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -77,6 +77,10 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import static
org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NO_LABEL;
import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
+/**
+ * Provides implementation of {@code CSQueue} methods common for every queue
class in Capacity
+ * Scheduler.
+ */
public abstract class AbstractCSQueue implements CSQueue {
private static final Logger LOG =
LoggerFactory.getLogger(AbstractCSQueue.class);
@@ -106,11 +110,9 @@ public abstract class AbstractCSQueue implements CSQueue {
CSQueueUsageTracker usageTracker;
public enum CapacityConfigType {
-// FIXME, from what I can see, Percentage mode can almost apply to weighted
-// and percentage mode at the same time, there's only small area need to be
-// changed, we need to rename "PERCENTAGE" to "PERCENTAGE" and "WEIGHT"
NONE, PERCENTAGE, ABSOLUTE_RESOURCE
};
+
protected CapacityConfigType capacityConfigType =
CapacityConfigType.NONE;
@@ -175,6 +177,9 @@ public abstract class AbstractCSQueue implements CSQueue {
return new QueuePath(parent.getQueuePath(), queueName);
}
+ /**
+ * Sets up capacity and weight values from configuration.
+ */
protected void setupConfigurableCapacities() {
CSQueueUtils.loadCapacitiesByLabelsFromConf(queuePath, queueCapacities,
queueContext.getConfiguration(),
this.queueNodeLabelsSettings.getConfiguredNodeLabels());
@@ -272,6 +277,12 @@ public abstract class AbstractCSQueue implements CSQueue {
return queueNodeLabelsSettings.getAccessibleNodeLabels();
}
+ /**
+ * Checks whether the user has the required permission to execute the action
of {@code QueueACL}.
+ * @param acl the access type the user is checked for
+ * @param user UGI of the user
+ * @return true, if the user has permission, false otherwise
+ */
@Override
public boolean hasAccess(QueueACL acl, UserGroupInformation user) {
return authorizer.checkPermission(
@@ -319,6 +330,11 @@ public abstract class AbstractCSQueue implements CSQueue {
return this.queueNodeLabelsSettings.getDefaultLabelExpression();
}
+ /**
+ * Initialize queue properties that are based on configuration.
+ * @param clusterResource overall resource of the cluster
+ * @throws IOException if configuration is set in a way that is inconsistent
+ */
protected void setupQueueConfigs(Resource clusterResource) throws
IOException {
@@ -460,6 +476,11 @@ public abstract class AbstractCSQueue implements CSQueue {
}
}
+ /**
+ * Initializes configured minimum and maximum capacity from configuration,
if capacity is defined
+ * in ABSOLUTE node.
+ * @param clusterResource overall resource of the cluster
+ */
protected void updateConfigurableResourceLimits(Resource clusterResource) {
for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) {
final Resource minResource = getMinimumAbsoluteResource(getQueuePath(),
label);
@@ -647,6 +668,13 @@ public abstract class AbstractCSQueue implements CSQueue {
return queueAllocationSettings.getMinimumAllocation();
}
+ /**
+ * Increments resource usage of the queue and all related statistics and
metrics that depends on
+ * it.
+ * @param clusterResource overall cluster resource
+ * @param resource resource am
[hadoop] branch trunk updated: YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 379baa5 YARN-10947. Simplify AbstractCSQueue#initializeQueueState.
Contributed by Andras Gyori
379baa5 is described below
commit 379baa5eb65a65df50461f337b88ff13f2134aeb
Author: Szilard Nemeth
AuthorDate: Thu Mar 3 16:44:12 2022 +0100
YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by
Andras Gyori
---
.../scheduler/capacity/AbstractCSQueue.java| 56 +
.../scheduler/capacity/CSQueueUtils.java | 14 +---
.../scheduler/capacity/QueueStateHelper.java | 98 ++
3 files changed, 101 insertions(+), 67 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index e924932..b0ab336 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -349,7 +349,7 @@ public abstract class AbstractCSQueue implements CSQueue {
// Initialize the queue state based on previous state, configured state
// and its parent state
- initializeQueueState();
+ QueueStateHelper.setQueueState(this);
authorizer = YarnAuthorizationProvider.getInstance(configuration);
@@ -553,60 +553,6 @@ public abstract class AbstractCSQueue implements CSQueue {
return configuredCapacityVectors.get(label);
}
- private void initializeQueueState() {
-QueueState previousState = getState();
-QueueState configuredState = queueContext.getConfiguration()
-.getConfiguredState(getQueuePath());
-QueueState parentState = (parent == null) ? null : parent.getState();
-
-// verify that we can not any value for State other than RUNNING/STOPPED
-if (configuredState != null && configuredState != QueueState.RUNNING
-&& configuredState != QueueState.STOPPED) {
- throw new IllegalArgumentException("Invalid queue state configuration."
- + " We can only use RUNNING or STOPPED.");
-}
-// If we did not set state in configuration, use Running as default state
-QueueState defaultState = QueueState.RUNNING;
-
-if (previousState == null) {
- // If current state of the queue is null, we would inherit the state
- // from its parent. If this queue does not has parent, such as root
queue,
- // we would use the configured state.
- if (parentState == null) {
-updateQueueState((configuredState == null) ? defaultState
-: configuredState);
- } else {
-if (configuredState == null) {
- updateQueueState((parentState == QueueState.DRAINING) ?
- QueueState.STOPPED : parentState);
-} else if (configuredState == QueueState.RUNNING
-&& parentState != QueueState.RUNNING) {
- throw new IllegalArgumentException(
- "The parent queue:" + parent.getQueuePath()
- + " cannot be STOPPED as the child queue:" + getQueuePath()
- + " is in RUNNING state.");
-} else {
- updateQueueState(configuredState);
-}
- }
-} else {
- // when we get a refreshQueue request from AdminService,
- if (previousState == QueueState.RUNNING) {
-if (configuredState == QueueState.STOPPED) {
- stopQueue();
-}
- } else {
-if (configuredState == QueueState.RUNNING) {
- try {
-activateQueue();
- } catch (YarnException ex) {
-throw new IllegalArgumentException(ex.getMessage());
- }
-}
- }
-}
- }
-
protected QueueInfo getQueueInfo() {
// Deliberately doesn't use lock here, because this method will be invoked
// from schedulerApplicationAttempt, to avoid deadlock, sacrifice
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils
[hadoop] branch trunk updated: Revert "YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori"
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1c27c69 Revert "YARN-10947. Simplify
AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori"
1c27c69 is described below
commit 1c27c69f44e6ad34ad53209e92d4231f00c3ca13
Author: Szilard Nemeth
AuthorDate: Thu Mar 3 16:38:56 2022 +0100
Revert "YARN-10947. Simplify AbstractCSQueue#initializeQueueState.
Contributed by Andras Gyori"
This reverts commit de526e166c349e49d657c5980901670c086d36b2.
With this commit, the patch of
https://issues.apache.org/jira/browse/YARN-10850 has been committed.
---
.../yarn/client/api/impl/AHSv2ClientImpl.java | 4 +-
.../yarn/client/api/impl/TestAHSv2ClientImpl.java | 70 --
2 files changed, 3 insertions(+), 71 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
index 5f90930..e3653ba 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
@@ -137,7 +137,9 @@ public class AHSv2ClientImpl extends AHSClient {
ApplicationId appId = applicationAttemptId.getApplicationId();
ApplicationReport appReport = getApplicationReport(appId);
Map filters = new HashMap<>();
-filters.put("appattemptId", applicationAttemptId.toString());
+filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" +
+applicationAttemptId.toString() +
+"\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}");
List entities = readerClient.getContainerEntities(
appId, "ALL", filters, 0, null);
List containers =
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
index 9664e40..f4d7a56 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
@@ -22,7 +22,6 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
@@ -46,11 +45,8 @@ import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -112,72 +108,6 @@ public class TestAHSv2ClientImpl {
}
@Test
- public void testGetContainerByAppAttempt() throws IOException, YarnException
{
-int applicationId = 1;
-ApplicationId appId = ApplicationId.newInstance(0, applicationId);
-
-ApplicationAttemptId appAttemptId =
-ApplicationAttemptId.newInstance(appId, 1);
-
-ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
-
-when(spyTimelineReaderClient.getContainerEntities(
-appId, "ALL",
-ImmutableMap.of("appattemptId", appAttemptId.toString()),
-0, null))
-.thenReturn(Arrays.asList(createContainerEntity(containerId)));
-
-when(spyTimelineReaderClient.getApplicationEntity(appId, "ALL", null))
-.thenReturn(createApplicationTimelineEntity(appId, true,
-false));
-
-List containerList = client.getContainers(appAttemptId);
-
-assertThat(containerList.size()).isEqualTo(1);
-
-assertThat(containerList.get(0).getContainerId().getApplicationAttemptId()
-.getApplicationId().getId()).isEqualTo(applicationId);
- }
-
- @Test
- public void testGetMultipleContainersByAppAttempt() throws IOException,
YarnException {
-int numContainers = 4;
-int applicationId = 3;
-int applicationAttemptId = 3;
-ApplicationId appMultiple = ApplicationId.newInstance(0, applicationId);
-ApplicationAttem
[hadoop] branch trunk updated: YARN-11042. Fix testQueueSubmitWithACLsEnabledWithQueueMapping in TestAppManager. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 356d337 YARN-11042. Fix
testQueueSubmitWithACLsEnabledWithQueueMapping in TestAppManager. Contributed
by Tamas Domok
356d337 is described below
commit 356d337d1e5a97dc72ae209df919cfb99e0dd841
Author: Szilard Nemeth
AuthorDate: Wed Mar 2 16:19:17 2022 +0100
YARN-11042. Fix testQueueSubmitWithACLsEnabledWithQueueMapping in
TestAppManager. Contributed by Tamas Domok
---
.../org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 87147ce..8c73a0d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -331,7 +331,7 @@ public class TestAppManager extends AppManagerTestBase{
csConf.set(PREFIX + "root.test.acl_submit_applications", "test");
csConf.set(PREFIX + "root.test.acl_administer_queue", "test");
-asContext.setQueue("test");
+asContext.setQueue("oldQueue");
MockRM newMockRM = new MockRM(csConf);
RMContext newMockRMContext = newMockRM.getRMContext();
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 856e483 YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok 856e483 is described below commit 856e483592040ae8df25f8978aead93004262843 Author: Szilard Nemeth AuthorDate: Wed Mar 2 16:13:35 2022 +0100 YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok --- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 9cdb579..277e29a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -142,7 +142,7 @@ Configuration |: |: | | `yarn.scheduler.capacity.maximum-applications` / `yarn.scheduler.capacity..maximum-applications` | Maximum number of applications in the system which can be concurrently active both running and pending. Limits on each queue are directly proportional to their queue capacities and user limits. This is a hard limit and any applications submitted when this limit is reached will be rejected. Default is 1. This can be set for all queues with `yarn.scheduler.capacity.maximum-a [...] | `yarn.scheduler.capacity.maximum-am-resource-percent` / `yarn.scheduler.capacity..maximum-am-resource-percent` | Maximum percent of resources in the cluster which can be used to run application masters - controls number of concurrent active applications. Limits on each queue are directly proportional to their queue capacities and user limits. Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for all queues with `yarn.scheduler.capacity.maximum-am-resource [...] -| `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity..max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.sch [...] +| `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity..max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.sch [...] You can also limit the number of parallel applications on a per user basis. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-10894. Follow up YARN-10237: fix the new test case in TestRMWebServicesCapacitySched. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 192f532 YARN-10894. Follow up YARN-10237: fix the new test case in
TestRMWebServicesCapacitySched. Contributed by Tamas Domok
192f532 is described below
commit 192f53283b8cbd44c9d7780cc65e68143e0cae43
Author: Szilard Nemeth
AuthorDate: Wed Mar 2 16:04:38 2022 +0100
YARN-10894. Follow up YARN-10237: fix the new test case in
TestRMWebServicesCapacitySched. Contributed by Tamas Domok
---
.../resourcemanager/webapp/TestRMWebServicesCapacitySched.java| 8 +---
.../webapp/TestRMWebServicesSchedulerActivities.java | 4 ++--
2 files changed, 7 insertions(+), 5 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index b24b7c0..2038433 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -130,7 +130,7 @@ public class TestRMWebServicesCapacitySched extends
JerseyTestBase {
// Define top-level queues
config.setQueues(CapacitySchedulerConfiguration.ROOT,
-new String[] {"a", "b"});
+new String[] {"a", "b", "c"});
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
config.setCapacity(A, 10.5f);
@@ -391,7 +391,7 @@ public class TestRMWebServicesCapacitySched extends
JerseyTestBase {
assertEquals("utilization", info.getString("orderingPolicyInfo"));
JSONArray arr = info.getJSONObject("queues").getJSONArray("queue");
-assertEquals("incorrect number of elements in: " + arr, 2, arr.length());
+assertEquals("incorrect number of elements in: " + arr, 3, arr.length());
// test subqueues
for (int i = 0; i < arr.length(); i++) {
@@ -433,6 +433,7 @@ public class TestRMWebServicesCapacitySched extends
JerseyTestBase {
qi.numApplications = info.getInt("numApplications");
qi.queueName = info.getString("queueName");
qi.state = info.getString("state");
+qi.isAbsoluteResource = info.getBoolean("isAbsoluteResource");
verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity);
@@ -506,7 +507,8 @@ public class TestRMWebServicesCapacitySched extends
JerseyTestBase {
+ " expected: " + q, qshortName.matches(info.queueName));
assertTrue("state doesn't match",
(csConf.getState(q).toString()).matches(info.state));
-if (q.equals("c")) {
+
+if (q.equals("root.c")) {
assertTrue("c queue is not configured in Absolute resource",
info.isAbsoluteResource);
} else {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
index 1dd8020..3aea113 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
@@ -154,7 +154,7 @@ public class TestRMWebServicesSchedulerActivities
verifyStateOfAllocations(allocation,
FN_ACT_FINAL_ALLOCATION_STATE, "ALLOCATED");
verifyQueueOrder(allocation,
- "root-root.a-root.b-root.b.b2-root.b.b3-root.b.b1");
+ "root-root.a-root.c-root.b-root.b.b2-root.b.b3-root.b.b1");
} finally {
rm.stop();
}
@@ -378,7 +378,7 @@ public class TestRMWebServicesSchedulerActivities
JSONObject allocations = getFirstSubNodeFromJson(json,
FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS);
[hadoop] branch branch-3.3 updated: YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 3ef3c5a YARN-11033. isAbsoluteResource is not correct for dynamically
created queues. Contributed by Tamas Domok
3ef3c5a is described below
commit 3ef3c5a05be2df7ec3dcb6e546387a8b9caf057a
Author: Szilard Nemeth
AuthorDate: Wed Mar 2 14:45:31 2022 +0100
YARN-11033. isAbsoluteResource is not correct for dynamically created
queues. Contributed by Tamas Domok
---
.../resourcemanager/scheduler/capacity/AbstractCSQueue.java | 2 +-
.../resourcemanager/webapp/dao/CapacitySchedulerInfo.java| 5 +
.../webapp/dao/CapacitySchedulerQueueInfo.java | 12 +++-
.../webapp/TestRMWebServicesCapacitySched.java | 4 +++-
.../webapp/TestRMWebServicesForCSWithPartitions.java | 2 +-
5 files changed, 13 insertions(+), 12 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index e7d9fa5..3870d42 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -128,7 +128,7 @@ public abstract class AbstractCSQueue implements CSQueue {
// either at this level or anywhere in the queue's hierarchy.
private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false;
- protected enum CapacityConfigType {
+ public enum CapacityConfigType {
NONE, PERCENTAGE, ABSOLUTE_RESOURCE
};
protected CapacityConfigType capacityConfigType =
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index 8de7529..9b8773e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlType;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.security.AccessType;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@@ -46,6 +47,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected float usedCapacity;
protected float maxCapacity;
protected String queueName;
+ private boolean isAbsoluteResource;
protected CapacitySchedulerQueueInfoList queues;
protected QueueCapacitiesInfo capacities;
protected CapacitySchedulerHealthInfo health;
@@ -75,6 +77,9 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
health = new CapacitySchedulerHealthInfo(cs);
maximumAllocation = new ResourceInfo(parent.getMaximumAllocation());
+isAbsoluteResource = parent.getCapacityConfigType() ==
+AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
+
CapacitySchedulerConfiguration conf = cs.getConfiguration();
queueAcls = new QueueAclsInfo();
for (Map.Entry e : conf
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index e38ab0d..56ac29b25 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java
[hadoop] branch branch-3.3 updated: YARN-11014. YARN incorrectly validates maximum capacity resources on the validation API. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new f06f44b YARN-11014. YARN incorrectly validates maximum capacity
resources on the validation API. Contributed by Benjamin Teke
f06f44b is described below
commit f06f44b1c2f6e7d752fa43953dd0fd600680d4ea
Author: Szilard Nemeth
AuthorDate: Wed Mar 2 14:23:00 2022 +0100
YARN-11014. YARN incorrectly validates maximum capacity resources on the
validation API. Contributed by Benjamin Teke
---
.../scheduler/capacity/CapacityScheduler.java | 16 ++
.../capacity/CapacitySchedulerConfigValidator.java | 2 +
.../TestCapacitySchedulerConfigValidator.java | 270 -
3 files changed, 284 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 69e775f..d0d95c3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2056,6 +2056,22 @@ public class CapacityScheduler extends
}
}
+ /**
+ * Add node to nodeTracker. Used when validating CS configuration by
instantiating a new
+ * CS instance.
+ * @param nodesToAdd node to be added
+ */
+ public void addNodes(List nodesToAdd) {
+writeLock.lock();
+try {
+ for (FiCaSchedulerNode node : nodesToAdd) {
+nodeTracker.addNode(node);
+ }
+} finally {
+ writeLock.unlock();
+}
+ }
+
private void addNode(RMNode nodeManager) {
writeLock.lock();
try {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
index c3b4df4..d180ffb 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
@@ -42,6 +42,7 @@ public final class CapacitySchedulerConfigValidator {
public static boolean validateCSConfiguration(
final Configuration oldConf, final Configuration newConf,
final RMContext rmContext) throws IOException {
+CapacityScheduler liveScheduler = (CapacityScheduler)
rmContext.getScheduler();
CapacityScheduler newCs = new CapacityScheduler();
try {
//TODO: extract all the validation steps and replace reinitialize with
@@ -49,6 +50,7 @@ public final class CapacitySchedulerConfigValidator {
newCs.setConf(oldConf);
newCs.setRMContext(rmContext);
newCs.init(oldConf);
+ newCs.addNodes(liveScheduler.getAllNodes());
newCs.reinitialize(newConf, rmContext, true);
return true;
} finally {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
index 04f4349..ad114d9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
@@ -19,13 +19,23 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.conf.Configuration;
+import
[hadoop] branch branch-3.2 updated: YARN-11014. YARN incorrectly validates maximum capacity resources on the validation API. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 9e2e1c79 YARN-11014. YARN incorrectly validates maximum capacity
resources on the validation API. Contributed by Benjamin Teke
9e2e1c79 is described below
commit 9e2e1c795f192789fc8393c0935f45145f455a86
Author: Szilard Nemeth
AuthorDate: Wed Mar 2 14:21:36 2022 +0100
YARN-11014. YARN incorrectly validates maximum capacity resources on the
validation API. Contributed by Benjamin Teke
---
.../scheduler/capacity/CapacityScheduler.java | 16 ++
.../capacity/CapacitySchedulerConfigValidator.java | 4 +-
.../TestCapacitySchedulerConfigValidator.java | 270 -
3 files changed, 285 insertions(+), 5 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 1455728..70cce75 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1994,6 +1994,22 @@ public class CapacityScheduler extends
}
}
+ /**
+ * Add node to nodeTracker. Used when validating CS configuration by
instantiating a new
+ * CS instance.
+ * @param nodesToAdd node to be added
+ */
+ public void addNodes(List nodesToAdd) {
+writeLock.lock();
+try {
+ for (FiCaSchedulerNode node : nodesToAdd) {
+nodeTracker.addNode(node);
+ }
+} finally {
+ writeLock.unlock();
+}
+ }
+
private void addNode(RMNode nodeManager) {
try {
writeLock.lock();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
index d8e1285..a162e76 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
@@ -42,7 +42,8 @@ public final class CapacitySchedulerConfigValidator {
public static boolean validateCSConfiguration(
final Configuration oldConf, final Configuration newConf,
- final RMContext rmContext) throws IOException {
+ final RMContext rmContext) throws IOException {
+CapacityScheduler liveScheduler = (CapacityScheduler)
rmContext.getScheduler();
CapacityScheduler newCs = new CapacityScheduler();
try {
//TODO: extract all the validation steps and replace reinitialize with
@@ -50,6 +51,7 @@ public final class CapacitySchedulerConfigValidator {
newCs.setConf(oldConf);
newCs.setRMContext(rmContext);
newCs.init(oldConf);
+ newCs.addNodes(liveScheduler.getAllNodes());
newCs.reinitialize(newConf, rmContext, true);
return true;
} finally {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
index 2b35a7c..e64a241 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
@@ -18,14 +18,24 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
[hadoop] branch branch-3.1 updated: YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new a3d8749 YARN-11075. Explicitly declare serialVersionUID in
LogMutation class. Contributed by Benjamin Teke
a3d8749 is described below
commit a3d8749b8ca5646bd7f6b272673452d3427ad9d7
Author: Szilard Nemeth
AuthorDate: Tue Mar 1 18:08:38 2022 +0100
YARN-11075. Explicitly declare serialVersionUID in LogMutation class.
Contributed by Benjamin Teke
---
.../resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 1cbe5df..abb7c3d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -53,6 +53,7 @@ public abstract class YarnConfigurationStore {
* audit logging and recovery.
*/
static class LogMutation implements Serializable {
+private static final long serialVersionUID = 7754046036718906356L;
private Map updates;
private String user;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 4bd67d7 YARN-11075. Explicitly declare serialVersionUID in
LogMutation class. Contributed by Benjamin Teke
4bd67d7 is described below
commit 4bd67d751663cc76d68d08af69bd22ac64900a92
Author: Szilard Nemeth
AuthorDate: Tue Mar 1 18:06:39 2022 +0100
YARN-11075. Explicitly declare serialVersionUID in LogMutation class.
Contributed by Benjamin Teke
---
.../resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 1d5a6ac..2e29f28 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -53,6 +53,7 @@ public abstract class YarnConfigurationStore {
* audit logging and recovery.
*/
static class LogMutation implements Serializable {
+private static final long serialVersionUID = 7754046036718906356L;
private Map updates;
private String user;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 935619a YARN-11075. Explicitly declare serialVersionUID in
LogMutation class. Contributed by Benjamin Teke
935619a is described below
commit 935619a28c0a81d99de7e46f2b8ccf5d552b0453
Author: Szilard Nemeth
AuthorDate: Tue Mar 1 18:05:04 2022 +0100
YARN-11075. Explicitly declare serialVersionUID in LogMutation class.
Contributed by Benjamin Teke
---
.../resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 4480bc3..425d63f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -53,6 +53,7 @@ public abstract class YarnConfigurationStore {
* audit logging and recovery.
*/
public static class LogMutation implements Serializable {
+private static final long serialVersionUID = 7754046036718906356L;
private Map updates;
private String user;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10590. Consider legacy auto queue creation absolute resource template to avoid rounding errors. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3653754 YARN-10590. Consider legacy auto queue creation absolute
resource template to avoid rounding errors. Contributed by Andras Gyori
3653754 is described below
commit 365375412fe5eea82549630ee8c5598502b95caf
Author: Szilard Nemeth
AuthorDate: Tue Feb 22 12:26:35 2022 +0100
YARN-10590. Consider legacy auto queue creation absolute resource template
to avoid rounding errors. Contributed by Andras Gyori
---
.../scheduler/capacity/AutoCreatedLeafQueue.java | 23 +---
.../capacity/AutoCreatedLeafQueueConfig.java | 15
.../scheduler/capacity/ManagedParentQueue.java | 41 ++
.../GuaranteedOrZeroCapacityOverTimePolicy.java| 10 +++---
.../TestAbsoluteResourceWithAutoQueue.java | 8 ++---
.../TestCapacitySchedulerAutoCreatedQueueBase.java | 28 +--
6 files changed, 83 insertions(+), 42 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
index 384a652..910d8de 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.yarn.api.records.Resource;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
@@ -30,6 +31,8 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
+import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
+
/**
* Leaf queues which are auto created by an underlying implementation of
* AbstractManagedParentQueue. Eg: PlanQueue for reservations or
@@ -81,14 +84,14 @@ public class AutoCreatedLeafQueue extends
AbstractAutoCreatedLeafQueue {
QueueCapacities capacities = leafQueueTemplate.getQueueCapacities();
//reset capacities for the leaf queue
- mergeCapacities(capacities);
+ mergeCapacities(capacities, leafQueueTemplate.getResourceQuotas());
} finally {
writeLock.unlock();
}
}
- public void mergeCapacities(QueueCapacities capacities) {
+ public void mergeCapacities(QueueCapacities capacities, QueueResourceQuotas
resourceQuotas) {
for ( String nodeLabel : capacities.getExistingNodeLabels()) {
queueCapacities.setCapacity(nodeLabel,
capacities.getCapacity(nodeLabel));
@@ -101,9 +104,19 @@ public class AutoCreatedLeafQueue extends
AbstractAutoCreatedLeafQueue {
Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel,
queueContext.getClusterResource());
- getQueueResourceQuotas().setEffectiveMinResource(nodeLabel,
- Resources.multiply(resourceByLabel,
- queueCapacities.getAbsoluteCapacity(nodeLabel)));
+ // Update effective resource from template due to rounding errors.
+ // However, we need to consider deactivation as well, in which case we
fall back to
+ // Percentage calculation (as absolute capacity will be 0, resource will
be zero as well).
+ if (getCapacityConfigType().equals(ABSOLUTE_RESOURCE)
+ && queueCapacities.getAbsoluteCapacity(nodeLabel) > 0) {
+getQueueResourceQuotas().setEffectiveMinResource(nodeLabel,
+resourceQuotas.getConfiguredMinResource(nodeLabel));
+ } else {
+getQueueResourceQuotas().setEffectiveMinResource(nodeLabel,
+Resources.multiply(resourceByLabel,
+queueCapacities.getAbsoluteCapacity(nodeLabel)));
+ }
+
getQueueResourceQuotas().setEffectiveMaxResource(nodeLabel,
Resources.multiply(resourceByLabel, queueCapacities
.getAbsoluteMaximumCapacity(nodeLabel)));
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/schedul
[hadoop] branch trunk updated: YARN-10944. AbstractCSQueue: Eliminate code duplication in overloaded versions of setMaxCapacity. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 0463498 YARN-10944. AbstractCSQueue: Eliminate code duplication in
overloaded versions of setMaxCapacity. Contributed by Andras Gyori
0463498 is described below
commit 0463498adcd3b62b83e6383cae556093d4270d66
Author: Szilard Nemeth
AuthorDate: Tue Feb 22 11:55:34 2022 +0100
YARN-10944. AbstractCSQueue: Eliminate code duplication in overloaded
versions of setMaxCapacity. Contributed by Andras Gyori
---
.../scheduler/capacity/AbstractCSQueue.java| 32 --
1 file changed, 11 insertions(+), 21 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 4eb7b68..020c601 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -74,6 +74,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import static
org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NO_LABEL;
import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
public abstract class AbstractCSQueue implements CSQueue {
@@ -279,33 +280,23 @@ public abstract class AbstractCSQueue implements CSQueue {
}
/**
- * Set maximum capacity - used only for testing.
+ * Set maximum capacity for empty node label.
* @param maximumCapacity new max capacity
*/
@VisibleForTesting
void setMaxCapacity(float maximumCapacity) {
-writeLock.lock();
-try {
- // Sanity check
- CSQueueUtils.checkMaxCapacity(this.queuePath,
- queueCapacities.getCapacity(), maximumCapacity);
- float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity(
- maximumCapacity, parent);
- CSQueueUtils.checkAbsoluteCapacity(this.queuePath,
- queueCapacities.getAbsoluteCapacity(), absMaxCapacity);
-
- queueCapacities.setMaximumCapacity(maximumCapacity);
- queueCapacities.setAbsoluteMaximumCapacity(absMaxCapacity);
-} finally {
- writeLock.unlock();
-}
+internalSetMaximumCapacity(maximumCapacity, NO_LABEL);
}
/**
- * Set maximum capacity
+ * Set maximum capacity.
* @param maximumCapacity new max capacity
*/
void setMaxCapacity(String nodeLabel, float maximumCapacity) {
+internalSetMaximumCapacity(maximumCapacity, nodeLabel);
+ }
+
+ private void internalSetMaximumCapacity(float maximumCapacity, String
nodeLabel) {
writeLock.lock();
try {
// Sanity check
@@ -323,7 +314,6 @@ public abstract class AbstractCSQueue implements CSQueue {
}
}
-
@Override
public String getDefaultNodeLabelExpression() {
return this.queueNodeLabelsSettings.getDefaultLabelExpression();
@@ -917,7 +907,7 @@ public abstract class AbstractCSQueue implements CSQueue {
}
private static String ensurePartition(String partition) {
-return Optional.ofNullable(partition).orElse(RMNodeLabelsManager.NO_LABEL);
+return Optional.ofNullable(partition).orElse(NO_LABEL);
}
@FunctionalInterface
@@ -1016,8 +1006,8 @@ public abstract class AbstractCSQueue implements CSQueue {
// Add NO_LABEL also to this list as NO_LABEL also can be granted with
// resource in many general cases.
-if (!nodeLabels.contains(RMNodeLabelsManager.NO_LABEL)) {
- nodeLabels.add(RMNodeLabelsManager.NO_LABEL);
+if (!nodeLabels.contains(NO_LABEL)) {
+ nodeLabels.add(NO_LABEL);
}
return nodeLabels;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new de526e1 YARN-10947. Simplify AbstractCSQueue#initializeQueueState.
Contributed by Andras Gyori
de526e1 is described below
commit de526e166c349e49d657c5980901670c086d36b2
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 21:05:01 2022 +0100
YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by
Andras Gyori
---
.../yarn/client/api/impl/AHSv2ClientImpl.java | 4 +-
.../yarn/client/api/impl/TestAHSv2ClientImpl.java | 70 ++
2 files changed, 71 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
index e3653ba..5f90930 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
@@ -137,9 +137,7 @@ public class AHSv2ClientImpl extends AHSClient {
ApplicationId appId = applicationAttemptId.getApplicationId();
ApplicationReport appReport = getApplicationReport(appId);
Map filters = new HashMap<>();
-filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" +
-applicationAttemptId.toString() +
-"\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}");
+filters.put("appattemptId", applicationAttemptId.toString());
List entities = readerClient.getContainerEntities(
appId, "ALL", filters, 0, null);
List containers =
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
index f4d7a56..9664e40 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
@@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
@@ -45,8 +46,11 @@ import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -108,6 +112,72 @@ public class TestAHSv2ClientImpl {
}
@Test
+ public void testGetContainerByAppAttempt() throws IOException, YarnException
{
+int applicationId = 1;
+ApplicationId appId = ApplicationId.newInstance(0, applicationId);
+
+ApplicationAttemptId appAttemptId =
+ApplicationAttemptId.newInstance(appId, 1);
+
+ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
+
+when(spyTimelineReaderClient.getContainerEntities(
+appId, "ALL",
+ImmutableMap.of("appattemptId", appAttemptId.toString()),
+0, null))
+.thenReturn(Arrays.asList(createContainerEntity(containerId)));
+
+when(spyTimelineReaderClient.getApplicationEntity(appId, "ALL", null))
+.thenReturn(createApplicationTimelineEntity(appId, true,
+false));
+
+List containerList = client.getContainers(appAttemptId);
+
+assertThat(containerList.size()).isEqualTo(1);
+
+assertThat(containerList.get(0).getContainerId().getApplicationAttemptId()
+.getApplicationId().getId()).isEqualTo(applicationId);
+ }
+
+ @Test
+ public void testGetMultipleContainersByAppAttempt() throws IOException,
YarnException {
+int numContainers = 4;
+int applicationId = 3;
+int applicationAttemptId = 3;
+ApplicationId appMultiple = ApplicationId.newInstance(0, applicationId);
+ApplicationAttemptId appAttemptIdMultiple =
+ApplicationAttemptId.newInstance(appMultiple,
applicationAttemptId);
+
+List containerEntities = new ArrayList<>();
+for (int containe
[hadoop] branch trunk updated: YARN-11070. Minimum resource ratio is overridden by subsequent labels. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7dd288c YARN-11070. Minimum resource ratio is overridden by
subsequent labels. Contributed by Andras Gyori
7dd288c is described below
commit 7dd288ce8ccc93acd4f182e3db4b7936402f1592
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 19:58:14 2022 +0100
YARN-11070. Minimum resource ratio is overridden by subsequent labels.
Contributed by Andras Gyori
---
.../scheduler/capacity/AbstractCSQueue.java| 2 +-
.../scheduler/capacity/ParentQueue.java| 22
.../TestAbsoluteResourceConfiguration.java | 61 ++
3 files changed, 73 insertions(+), 12 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 809a860..4eb7b68 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -1298,7 +1298,7 @@ public abstract class AbstractCSQueue implements CSQueue {
CapacityConfigType.ABSOLUTE_RESOURCE)) {
newEffectiveMinResource = createNormalizedMinResource(
usageTracker.getQueueResourceQuotas().getConfiguredMinResource(label),
-((ParentQueue) parent).getEffectiveMinRatioPerResource());
+((ParentQueue) parent).getEffectiveMinRatio(label));
// Max resource of a queue should be the minimum of {parent's
maxResources,
// this queue's maxResources}. Both parent's maxResources and this
queue's
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index b77a90a..c624aab 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
@@ -101,11 +102,11 @@ public class ParentQueue extends AbstractCSQueue {
private AutoCreatedQueueTemplate autoCreatedQueueTemplate;
- // effective min ratio per resource, it is used during updateClusterResource,
- // leaf queue can use this to calculate effective resources.
- // This field will not be edited, reference will point to a new immutable map
- // after every time recalculation
- private volatile Map effectiveMinRatioPerResource;
+ // A ratio of the queue's effective minimum resource and the summary of the
configured
+ // minimum resource of its children grouped by labels and calculated for
each resource names
+ // distinctively.
+ private final Map> effectiveMinResourceRatio =
+ new ConcurrentHashMap<>();
public ParentQueue(CapacitySchedulerQueueContext queueContext,
String queueName, CSQueue parent, CSQueue old) throws IOException {
@@ -1328,8 +1329,8 @@ public class ParentQueue extends AbstractCSQueue {
}
}
-effectiveMinRatioPerResource = getEffectiveMinRatioPerResource(
-configuredMinResources, numeratorForMinRatio);
+effectiveMinResourceRatio.put(label, getEffectiveMinRatio(
+configuredMinResources, numeratorForMinRatio));
// Update effective resources for my self;
if (rootQueue) {
@@ -1340,7 +1341,7 @@ public class ParentQueue extends AbstractCSQueue {
}
}
- private Map getEffectiveMinRatioPerResource(
+ private Map getEffectiveMinRatio(
Resource configuredMinResources, Resource numeratorForMinRatio) {
Map effectiveMinRatioPerResource = ne
[hadoop] branch trunk updated: YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new ad0a1dc YARN-11075. Explicitly declare serialVersionUID in
LogMutation class. Contributed by Benjamin Teke
ad0a1dc is described below
commit ad0a1dc89723ed1630399203da614d6a338b1920
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 19:48:42 2022 +0100
YARN-11075. Explicitly declare serialVersionUID in LogMutation class.
Contributed by Benjamin Teke
---
.../resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 4480bc3..425d63f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -53,6 +53,7 @@ public abstract class YarnConfigurationStore {
* audit logging and recovery.
*/
public static class LogMutation implements Serializable {
+private static final long serialVersionUID = 7754046036718906356L;
private Map updates;
private String user;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10995. Move PendingApplicationComparator from GuaranteedOrZeroCapacityOverTimePolicy. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4c05d25 YARN-10995. Move PendingApplicationComparator from
GuaranteedOrZeroCapacityOverTimePolicy. Contributed by Benjamin Teke
4c05d25 is described below
commit 4c05d257ba3f3311b5bbc993f6e5e35637487d88
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 19:43:37 2022 +0100
YARN-10995. Move PendingApplicationComparator from
GuaranteedOrZeroCapacityOverTimePolicy. Contributed by Benjamin Teke
---
.../scheduler/capacity/CapacityScheduler.java | 31 +
.../capacity/CapacitySchedulerContext.java | 3 +-
.../capacity/CapacitySchedulerQueueContext.java| 7 ++---
.../GuaranteedOrZeroCapacityOverTimePolicy.java| 32 +-
4 files changed, 35 insertions(+), 38 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 16c18bc..44e80a6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -21,6 +21,7 @@ package
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -201,6 +202,9 @@ public class CapacityScheduler extends
private int threadNum = 0;
+ private final PendingApplicationComparator applicationComparator =
+ new PendingApplicationComparator();
+
@Override
public void setConf(Configuration conf) {
yarnConf = conf;
@@ -3555,4 +3559,31 @@ public class CapacityScheduler extends
return asyncSchedulerThreads;
}
}
+
+ @Override
+ public PendingApplicationComparator getPendingApplicationComparator(){
+return applicationComparator;
+ }
+
+ /**
+ * Comparator that orders applications by their submit time.
+ */
+ class PendingApplicationComparator
+ implements Comparator {
+
+@Override
+public int compare(FiCaSchedulerApp app1, FiCaSchedulerApp app2) {
+ RMApp rmApp1 = rmContext.getRMApps().get(app1.getApplicationId());
+ RMApp rmApp2 = rmContext.getRMApps().get(app2.getApplicationId());
+ if (rmApp1 != null && rmApp2 != null) {
+return Long.compare(rmApp1.getSubmitTime(), rmApp2.getSubmitTime());
+ } else if (rmApp1 != null) {
+return -1;
+ } else if (rmApp2 != null) {
+return 1;
+ } else{
+return 0;
+ }
+}
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
index 1d0600f..5795a7e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import java.util.Comparator;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -104,4 +102,5 @@ public interface CapacitySchedulerContext {
*/
Clock getClock();
+ CapacityScheduler.PendingApplicationComparator
getPendingApplicationComparator();
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/s
[hadoop] branch trunk updated: YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 51001fc YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok 51001fc is described below commit 51001fc7bf3bf2b83aaa5f4d2b4af0397ad5dc4c Author: Szilard Nemeth AuthorDate: Thu Feb 17 19:38:18 2022 +0100 YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok --- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index a945e14..9173cb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -151,7 +151,7 @@ Configuration |: |: | | `yarn.scheduler.capacity.maximum-applications` / `yarn.scheduler.capacity..maximum-applications` | Maximum number of applications in the system which can be concurrently active both running and pending. Limits on each queue are directly proportional to their queue capacities and user limits. This is a hard limit and any applications submitted when this limit is reached will be rejected. Default is 1. This can be set for all queues with `yarn.scheduler.capacity.maximum-a [...] | `yarn.scheduler.capacity.maximum-am-resource-percent` / `yarn.scheduler.capacity..maximum-am-resource-percent` | Maximum percent of resources in the cluster which can be used to run application masters - controls number of concurrent active applications. Limits on each queue are directly proportional to their queue capacities and user limits. Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for all queues with `yarn.scheduler.capacity.maximum-am-resource [...] -| `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity..max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.sch [...] +| `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity..max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.sch [...] You can also limit the number of parallel applications on a per user basis. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10580. Fix some issues in TestRMWebServicesCapacitySchedDynamicConfig. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 48e02f7 YARN-10580. Fix some issues in
TestRMWebServicesCapacitySchedDynamicConfig. Contributed by Tamas Domok
48e02f7 is described below
commit 48e02f79983b32b64d1ae10e8ddf162c06f700d6
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 19:17:31 2022 +0100
YARN-10580. Fix some issues in TestRMWebServicesCapacitySchedDynamicConfig.
Contributed by Tamas Domok
---
.../TestRMWebServicesCapacitySchedDynamicConfig.java | 16
1 file changed, 4 insertions(+), 12 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
index df4f18e..0e2ecd0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
@@ -102,8 +102,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
throws Exception {
Configuration config = CSConfigGenerator
.createPercentageConfig();
-config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-YarnConfiguration.MEMORY_CONFIGURATION_STORE);
initResourceManager(config);
@@ -121,8 +119,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
throws Exception {
Configuration config = CSConfigGenerator
.createPercentageConfigLegacyAutoCreation();
-config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-YarnConfiguration.MEMORY_CONFIGURATION_STORE);
initResourceManager(config);
@@ -141,8 +137,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
throws Exception {
Configuration config = CSConfigGenerator
.createAbsoluteConfigLegacyAutoCreation();
-config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-YarnConfiguration.MEMORY_CONFIGURATION_STORE);
initResourceManager(config);
initAutoQueueHandler(8192 * GB);
@@ -157,8 +151,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
throws Exception {
Configuration config = CSConfigGenerator
.createAbsoluteConfig();
-config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-YarnConfiguration.MEMORY_CONFIGURATION_STORE);
initResourceManager(config);
@@ -176,8 +168,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
throws Exception {
Configuration config = CSConfigGenerator
.createWeightConfig();
-config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-YarnConfiguration.MEMORY_CONFIGURATION_STORE);
initResourceManager(config);
@@ -197,8 +187,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
throws Exception {
Configuration config = CSConfigGenerator
.createWeightConfigWithAutoQueueCreationEnabled();
-config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-YarnConfiguration.MEMORY_CONFIGURATION_STORE);
config.setInt(CapacitySchedulerConfiguration
.getQueuePrefix("root.autoParent1") +
AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX +
@@ -332,6 +320,10 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
for (Map.Entry entry : configs.entrySet()) {
config.set(entry.getKey(), entry.getValue());
}
+
+ config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
+
return config;
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10894. Follow up YARN-10237: fix the new test case in TestRMWebServicesCapacitySched. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new bb1135c YARN-10894. Follow up YARN-10237: fix the new test case in
TestRMWebServicesCapacitySched. Contributed by Tamas Domok
bb1135c is described below
commit bb1135c77c34ca5dcd8d43fca31c635b65bf3638
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 19:12:51 2022 +0100
YARN-10894. Follow up YARN-10237: fix the new test case in
TestRMWebServicesCapacitySched. Contributed by Tamas Domok
---
.../webapp/TestRMWebServicesCapacitySched.java | 2 +-
.../TestRMWebServicesSchedulerActivities.java | 4 +-
.../scheduler-response-NodeLabelDefaultAPI.xml | 491 -
.../scheduler-response-PerUserResources.json | 487 +++-
.../webapp/scheduler-response-PerUserResources.xml | 490 +++-
.../test/resources/webapp/scheduler-response.json | 487 +++-
.../test/resources/webapp/scheduler-response.xml | 490 +++-
7 files changed, 2443 insertions(+), 8 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 1f73870..2be9b54 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -120,7 +120,7 @@ public class TestRMWebServicesCapacitySched extends
JerseyTestBase {
// Define top-level queues
config.setQueues(CapacitySchedulerConfiguration.ROOT,
-new String[] {"a", "b"});
+new String[] {"a", "b", "c"});
final String a = CapacitySchedulerConfiguration.ROOT + ".a";
config.setCapacity(a, 10.5f);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
index f864794..3fc3c26 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
@@ -156,7 +156,7 @@ public class TestRMWebServicesSchedulerActivities
verifyStateOfAllocations(allocation,
FN_ACT_FINAL_ALLOCATION_STATE, "ALLOCATED");
verifyQueueOrder(allocation,
- "root-root.a-root.b-root.b.b2-root.b.b3-root.b.b1");
+ "root-root.a-root.c-root.b-root.b.b2-root.b.b3-root.b.b1");
} finally {
rm.stop();
}
@@ -380,7 +380,7 @@ public class TestRMWebServicesSchedulerActivities
JSONObject allocations = getFirstSubNodeFromJson(json,
FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS);
verifyQueueOrder(allocations,
- "root-root.a-root.b-root.b.b3-root.b.b1");
+ "root-root.c-root.a-root.b-root.b.b3-root.b.b1");
verifyStateOfAllocations(allocations, FN_ACT_FINAL_ALLOCATION_STATE,
"RESERVED");
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml
index 48b6893..d49491f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml
@@ -10,6 +10,495 @@
2147483647
false
+
+root.c
+0.0
+0.0
+100.0
+
[hadoop] branch trunk updated: YARN-11071. AutoCreatedQueueTemplate incorrect wildcard level. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d58c10c YARN-11071. AutoCreatedQueueTemplate incorrect wildcard
level. Contributed by Tamas Domok
d58c10c is described below
commit d58c10c3a5f0d6c923d8a49457ce5a73a0ec7011
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 19:03:44 2022 +0100
YARN-11071. AutoCreatedQueueTemplate incorrect wildcard level. Contributed
by Tamas Domok
---
.../scheduler/capacity/AutoCreatedQueueTemplate.java | 3 +--
.../scheduler/capacity/TestAutoCreatedQueueTemplate.java | 11 +++
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
index 1603b19..8fa1e13 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
@@ -170,8 +170,7 @@ public class AutoCreatedQueueTemplate {
int wildcardLevel = 0;
// root can not be wildcarded
// MAX_WILDCARD_LEVEL will be configurable in the future
-int supportedWildcardLevel = Math.min(queuePathMaxIndex - 1,
-MAX_WILDCARD_LEVEL);
+int supportedWildcardLevel = Math.min(queuePathMaxIndex,
MAX_WILDCARD_LEVEL);
// Allow root to have template properties
if (queuePath.isRoot()) {
supportedWildcardLevel = 0;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
index 5b58feb..e73001b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
@@ -89,6 +89,17 @@ public class TestAutoCreatedQueueTemplate {
}
@Test
+ public void testWildcardAfterRoot() {
+conf.set(getTemplateKey("root.*", "acl_submit_applications"), "user");
+AutoCreatedQueueTemplate template =
+new AutoCreatedQueueTemplate(conf, new QueuePath("root.a"));
+template.setTemplateEntriesForChild(conf, "root.a");
+
+Assert.assertEquals("acl_submit_applications is set", "user",
+template.getTemplateProperties().get("acl_submit_applications"));
+ }
+
+ @Test
public void testTemplatePrecedence() {
conf.set(getTemplateKey("root.a.b", "capacity"), "6w");
conf.set(getTemplateKey("root.a.*", "capacity"), "4w");
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: YARN-10503. Support queue capacity in terms of absolute resources with custom resourceType. Contributed by Qi Zhu.
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 020acb8 YARN-10503. Support queue capacity in terms of absolute
resources with custom resourceType. Contributed by Qi Zhu.
020acb8 is described below
commit 020acb8d39fbc4800e4c6714195c6ec7733ec90c
Author: Szilard Nemeth
AuthorDate: Mon Dec 20 17:59:06 2021 +0100
YARN-10503. Support queue capacity in terms of absolute resources with
custom resourceType. Contributed by Qi Zhu.
---
.../hadoop/yarn/util/resource/ResourceUtils.java | 15
.../capacity/CapacitySchedulerConfiguration.java | 25 +--
.../capacity/TestCSAllocateCustomResource.java | 82 ++
.../src/test/resources/resource-types-test.xml | 2 +-
4 files changed, 118 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 5061604..790e754 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -766,4 +766,19 @@ public class ResourceUtils {
return info;
}
+
+ public static StringBuilder
+ getCustomResourcesStrings(Resource resource) {
+StringBuilder res = new StringBuilder();
+if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
+ ResourceInformation[] resources =
+ resource.getResources();
+ for (int i = 2; i < resources.length; i++) {
+ResourceInformation resInfo = resources[i];
+res.append(","
++ resInfo.getName() + "=" + resInfo.getValue());
+ }
+}
+return res;
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 06b56b1..59d6767 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -2096,11 +2096,14 @@ public class CapacitySchedulerConfiguration extends
ReservationSchedulerConfigur
}
StringBuilder resourceString = new StringBuilder();
+
resourceString
.append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() +
"="
+ resource.getMemorySize() + ","
+ AbsoluteResourceType.VCORES.toString().toLowerCase() + "="
-+ resource.getVirtualCores() + "]");
++ resource.getVirtualCores()
++ ResourceUtils.
+getCustomResourcesStrings(resource) + "]");
String prefix = getQueuePrefix(queue) + type;
if (!label.isEmpty()) {
@@ -2160,8 +2163,12 @@ public class CapacitySchedulerConfiguration extends
ReservationSchedulerConfigur
private void updateResourceValuesFromConfig(Set resourceTypes,
Resource resource, String[] splits) {
+String resourceName = splits[0].trim();
+
// If key is not a valid type, skip it.
-if (!resourceTypes.contains(splits[0])) {
+if (!resourceTypes.contains(resourceName)
+&& !ResourceUtils.getResourceTypes().containsKey(resourceName)) {
+ LOG.error(resourceName + " not supported.");
return;
}
@@ -2174,9 +2181,17 @@ public class CapacitySchedulerConfiguration extends
ReservationSchedulerConfigur
resourceValue = UnitsConversionUtil.convert(units, "Mi", resourceValue);
}
+// Custom resource type defined by user.
+// Such as GPU FPGA etc.
+if (!resourceTypes.contains(resourceName)) {
+ resource.setResourceInformation(resourceName, ResourceInformation
+ .newInstance(resourceName, units, resourceValue));
+ return;
+}
+
// map it based on key.
AbsoluteResourceType resType = AbsoluteResourceType
-.valueOf(StringUtils.toUpperCase(splits[0].trim()));
+.valueOf(StringUtils.toUpperCase(resourceName));
swit
[hadoop] branch branch-3.2 updated: YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 01de8fd YARN-6862. Nodemanager resource usage metrics sometimes are
negative. Contributed by Benjamin Teke
01de8fd is described below
commit 01de8fd9d650a41aba850043f78e565de759a83e
Author: Szilard Nemeth
AuthorDate: Fri Dec 17 14:53:06 2021 +0100
YARN-6862. Nodemanager resource usage metrics sometimes are negative.
Contributed by Benjamin Teke
---
.../monitor/ContainersMonitorImpl.java | 8
.../MockCPUResourceCalculatorProcessTree.java | 10 +
...> MockMemoryResourceCalculatorProcessTree.java} | 47 +++---
.../monitor/MockResourceCalculatorProcessTree.java | 6 +++
.../TestContainersMonitorResourceChange.java | 24 ---
5 files changed, 75 insertions(+), 20 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index a7bf73f..d7da7d7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -540,6 +540,14 @@ public class ContainersMonitorImpl extends AbstractService
implements
pTree.updateProcessTree();// update process-tree
long currentVmemUsage = pTree.getVirtualMemorySize();
long currentPmemUsage = pTree.getRssMemorySize();
+if (currentVmemUsage < 0 || currentPmemUsage < 0) {
+ // YARN-6862/YARN-5021 If the container just exited or for
+ // another reason the physical/virtual memory is UNAVAILABLE (-1)
+ // the values shouldn't be aggregated.
+ LOG.info("Skipping monitoring container {} because "
+ + "memory usage is not available.", containerId);
+ continue;
+}
// if machine has 6 cores and 3 are used,
// cpuUsagePercentPerCore should be 300%
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
index eb35c91..49161f3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
@@ -57,6 +57,16 @@ public class MockCPUResourceCalculatorProcessTree
}
@Override
+ public long getVirtualMemorySize(int olderThanAge) {
+return 0;
+ }
+
+ @Override
+ public long getRssMemorySize(int olderThanAge) {
+return 0;
+ }
+
+ @Override
public float getCpuUsagePercent() {
long cpu = this.cpuPercentage;
// First getter call will be returned with -1, and other calls will
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java
similarity index 50%
copy from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
copy to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java
index eb35c91..ea45ac4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-ser
[hadoop] branch branch-3.3 updated: YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 357423b YARN-6862. Nodemanager resource usage metrics sometimes are
negative. Contributed by Benjamin Teke
357423b is described below
commit 357423b57a93249aebd635eb461ba1cff63c7353
Author: Szilard Nemeth
AuthorDate: Fri Dec 17 14:51:41 2021 +0100
YARN-6862. Nodemanager resource usage metrics sometimes are negative.
Contributed by Benjamin Teke
---
.../monitor/ContainersMonitorImpl.java | 8
.../MockCPUResourceCalculatorProcessTree.java | 10 +
...> MockMemoryResourceCalculatorProcessTree.java} | 47 +++---
.../monitor/MockResourceCalculatorProcessTree.java | 6 +++
.../TestContainersMonitorResourceChange.java | 24 ---
5 files changed, 75 insertions(+), 20 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index a83ae3a..deccded 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -537,6 +537,14 @@ public class ContainersMonitorImpl extends AbstractService
implements
pTree.updateProcessTree();// update process-tree
long currentVmemUsage = pTree.getVirtualMemorySize();
long currentPmemUsage = pTree.getRssMemorySize();
+if (currentVmemUsage < 0 || currentPmemUsage < 0) {
+ // YARN-6862/YARN-5021 If the container just exited or for
+ // another reason the physical/virtual memory is UNAVAILABLE (-1)
+ // the values shouldn't be aggregated.
+ LOG.info("Skipping monitoring container {} because "
+ + "memory usage is not available.", containerId);
+ continue;
+}
// if machine has 6 cores and 3 are used,
// cpuUsagePercentPerCore should be 300%
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
index eb35c91..49161f3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
@@ -57,6 +57,16 @@ public class MockCPUResourceCalculatorProcessTree
}
@Override
+ public long getVirtualMemorySize(int olderThanAge) {
+return 0;
+ }
+
+ @Override
+ public long getRssMemorySize(int olderThanAge) {
+return 0;
+ }
+
+ @Override
public float getCpuUsagePercent() {
long cpu = this.cpuPercentage;
// First getter call will be returned with -1, and other calls will
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java
similarity index 50%
copy from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
copy to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java
index eb35c91..ea45ac4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-ser
[hadoop] branch trunk updated: YARN-11044. Fix TestApplicationLimits.testLimitsComputation() ineffective asserts. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6ea1685 YARN-11044. Fix TestApplicationLimits.testLimitsComputation()
ineffective asserts. Contributed by Benjamin Teke
6ea1685 is described below
commit 6ea1685abd879301ee5aa13942028e55c5ec90f8
Author: Szilard Nemeth
AuthorDate: Thu Dec 16 00:01:09 2021 +0100
YARN-11044. Fix TestApplicationLimits.testLimitsComputation() ineffective
asserts. Contributed by Benjamin Teke
---
.../scheduler/capacity/TestApplicationLimits.java | 15 ---
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 7cb0ccd..f06631d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -267,6 +267,8 @@ public class TestApplicationLimits {
@Test
public void testLimitsComputation() throws Exception {
+final float epsilon = 1e-5f;
+
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
setupQueueConfiguration(csConf);
@@ -340,10 +342,9 @@ public class TestApplicationLimits {
assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
// should default to global setting if per queue setting not set
-assertEquals((long)
-
CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
-(long)csConf.getMaximumApplicationMasterResourcePerQueuePercent(
-queue.getQueuePath()));
+
assertEquals(CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
+csConf.getMaximumApplicationMasterResourcePerQueuePercent(
+queue.getQueuePath()), epsilon);
// Change the per-queue max AM resources percentage.
csConf.setFloat(PREFIX + queue.getQueuePath()
@@ -361,9 +362,9 @@ public class TestApplicationLimits {
queue = (LeafQueue)queues.get(A);
-assertEquals((long) 0.5,
-(long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(
- queue.getQueuePath()));
+assertEquals(0.5f,
+csConf.getMaximumApplicationMasterResourcePerQueuePercent(
+ queue.getQueuePath()), epsilon);
assertThat(queue.calculateAndGetAMResourceLimit()).isEqualTo(
Resource.newInstance(800 * GB, 1));
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: Clean up checkstyle warnings from YARN-11024/10907/10929. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 843f66f Clean up checkstyle warnings from YARN-11024/10907/10929.
Contributed by Benjamin Teke
843f66f is described below
commit 843f66f4dc012dff402bfebc183d46673cd47419
Author: Szilard Nemeth
AuthorDate: Tue Dec 14 22:00:43 2021 +0100
Clean up checkstyle warnings from YARN-11024/10907/10929. Contributed by
Benjamin Teke
---
.../scheduler/capacity/AbstractCSQueue.java| 3 +-
.../scheduler/capacity/AbstractLeafQueue.java | 38 +-
.../GuaranteedOrZeroCapacityOverTimePolicy.java| 1 -
.../scheduler/capacity/TestApplicationLimits.java | 35 +---
.../capacity/TestApplicationLimitsByPartition.java | 1 -
.../scheduler/capacity/TestReservations.java | 1 -
6 files changed, 33 insertions(+), 46 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 3a0e2ae..809a860 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -145,7 +145,8 @@ public abstract class AbstractCSQueue implements CSQueue {
CSQueueMetrics metrics = old != null ?
(CSQueueMetrics) old.getMetrics() :
CSQueueMetrics.forQueue(getQueuePath(), parent,
-queueContext.getConfiguration().getEnableUserMetrics(),
queueContext.getConfiguration());
+queueContext.getConfiguration().getEnableUserMetrics(),
+queueContext.getConfiguration());
this.usageTracker = new CSQueueUsageTracker(metrics);
this.queueCapacities = new QueueCapacities(parent == null);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
index 8b31241..e194800 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
@@ -453,8 +453,7 @@ public class AbstractLeafQueue extends AbstractCSQueue {
}
@Override
- public List
- getQueueUserAclInfo(UserGroupInformation user) {
+ public List getQueueUserAclInfo(UserGroupInformation user)
{
readLock.lock();
try {
QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance(
@@ -527,8 +526,8 @@ public class AbstractLeafQueue extends AbstractCSQueue {
}
@Override
- public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
throws
- IOException {
+ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
+ throws IOException {
writeLock.lock();
try {
@@ -634,7 +633,8 @@ public class AbstractLeafQueue extends AbstractCSQueue {
// Check submission limits for queues
//TODO recalculate max applications because they can depend on capacity
- if (getNumApplications() >= getMaxApplications() && !(this instanceof
AutoCreatedLeafQueue)) {
+ if (getNumApplications() >= getMaxApplications() &&
+ !(this instanceof AutoCreatedLeafQueue)) {
String msg =
"Queue " + getQueuePath() + " already has " + getNumApplications()
+ " applications,"
@@ -646,7 +646,8 @@ public class AbstractLeafQueue extends AbstractCSQueue {
// Check submission limits for the user on this queue
User user = usersManager.getUserAndAddIfAbsent(userName);
//TODO recalculate max applications because they can depend on capacity
- if (user.getTotalApplications() >= getMaxApplicationsPerUser() &&
!(this instanceof AutoCreatedLeafQueue)) {
+ if (user.getTotalApplications() >= getMaxApplicat
[hadoop] branch trunk updated: YARN-10929. Do not use a separate config in legacy CS AQC. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 364d38f YARN-10929. Do not use a separate config in legacy CS AQC.
Contributed by Benjamin Teke
364d38f is described below
commit 364d38f00d75e06099ef030030fca7ec90a8a817
Author: Szilard Nemeth
AuthorDate: Tue Dec 14 13:41:22 2021 +0100
YARN-10929. Do not use a separate config in legacy CS AQC. Contributed by
Benjamin Teke
---
.../scheduler/capacity/AbstractCSQueue.java| 44 ++
.../scheduler/capacity/AbstractLeafQueue.java | 69 +-
.../capacity/AbstractManagedParentQueue.java | 17 +-
.../scheduler/capacity/AutoCreatedLeafQueue.java | 10 ++--
.../capacity/CSQueuePreemptionSettings.java| 19 +++---
.../capacity/CapacitySchedulerQueueContext.java| 4 ++
.../scheduler/capacity/LeafQueue.java | 2 +-
.../scheduler/capacity/ManagedParentQueue.java | 28 +++--
.../scheduler/capacity/ParentQueue.java| 10 ++--
.../scheduler/capacity/PlanQueue.java | 2 +-
.../capacity/QueueAllocationSettings.java | 11 +---
.../scheduler/capacity/ReservationQueue.java | 6 +-
.../capacity/TestCapacitySchedulerPerf.java| 2 +-
13 files changed, 84 insertions(+), 140 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 5040b02..3a0e2ae 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -173,10 +173,9 @@ public abstract class AbstractCSQueue implements CSQueue {
return new QueuePath(parent.getQueuePath(), queueName);
}
- protected void setupConfigurableCapacities(
- CapacitySchedulerConfiguration configuration) {
+ protected void setupConfigurableCapacities() {
CSQueueUtils.loadCapacitiesByLabelsFromConf(queuePath, queueCapacities,
-configuration, this.queueNodeLabelsSettings.getConfiguredNodeLabels());
+queueContext.getConfiguration(),
this.queueNodeLabelsSettings.getConfiguredNodeLabels());
}
@Override
@@ -329,14 +328,14 @@ public abstract class AbstractCSQueue implements CSQueue {
return this.queueNodeLabelsSettings.getDefaultLabelExpression();
}
- protected void setupQueueConfigs(Resource clusterResource,
- CapacitySchedulerConfiguration configuration) throws
+ protected void setupQueueConfigs(Resource clusterResource) throws
IOException {
writeLock.lock();
try {
+ CapacitySchedulerConfiguration configuration =
queueContext.getConfiguration();
if (isDynamicQueue() || this instanceof AbstractAutoCreatedLeafQueue) {
-setDynamicQueueProperties(configuration);
+setDynamicQueueProperties();
}
// Collect and set the Node label configuration
@@ -344,7 +343,7 @@ public abstract class AbstractCSQueue implements CSQueue {
getQueuePath(),
queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues());
// Initialize the queue capacities
- setupConfigurableCapacities(configuration);
+ setupConfigurableCapacities();
updateAbsoluteCapacities();
updateCapacityConfigType();
@@ -354,26 +353,23 @@ public abstract class AbstractCSQueue implements CSQueue {
// Setup queue's maximumAllocation respecting the global
// and the queue settings
- // TODO remove the getConfiguration() param after the AQC configuration
duplication
- // removal is resolved
- this.queueAllocationSettings.setupMaximumAllocation(configuration,
- queueContext.getConfiguration(), getQueuePath(),
+ this.queueAllocationSettings.setupMaximumAllocation(configuration,
getQueuePath(),
parent);
// Initialize the queue state based on previous state, configured state
// and its parent state
- initializeQueueState(configuration);
+ initializeQueueState();
authorizer = YarnAuthorizationProvider.getInstance(configuration);
this.acls = configuration.getAcls(getQueuePath());
- this.userWeights = getUserWeightsFromHierarchy(configuration);
+ this.userWeights = getUserWeightsFromHierarchy
[hadoop] branch trunk updated: YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new f544ef4 YARN-11033. isAbsoluteResource is not correct for dynamically
created queues. Contributed by Tamas Domok
f544ef4 is described below
commit f544ef4a51007be24592b9b3dc91bfd73c2f7a88
Author: Szilard Nemeth
AuthorDate: Fri Dec 10 15:09:53 2021 +0100
YARN-11033. isAbsoluteResource is not correct for dynamically created
queues. Contributed by Tamas Domok
---
.../webapp/dao/CapacitySchedulerInfo.java | 5 +
.../webapp/dao/CapacitySchedulerQueueInfo.java | 12 +-
...estRMWebServicesCapacitySchedDynamicConfig.java | 35 +-
.../TestRMWebServicesForCSWithPartitions.java | 2 +-
.../webapp/scheduler-response-AbsoluteMode.json| 3 +-
...r-response-AbsoluteModeLegacyAutoCreation.json} | 825 +++--
.../scheduler-response-NodeLabelDefaultAPI.xml | 3 +-
.../scheduler-response-PerUserResources.json | 3 +-
.../webapp/scheduler-response-PerUserResources.xml | 3 +-
.../webapp/scheduler-response-PercentageMode.json | 3 +-
...-response-PercentageModeLegacyAutoCreation.json | 3 +-
.../webapp/scheduler-response-WeightMode.json | 3 +-
...onse-WeightModeWithAutoCreatedQueues-After.json | 3 +-
...nse-WeightModeWithAutoCreatedQueues-Before.json | 3 +-
.../test/resources/webapp/scheduler-response.json | 3 +-
.../test/resources/webapp/scheduler-response.xml | 3 +-
16 files changed, 477 insertions(+), 435 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index c09ab54..74c7c20 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlType;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@@ -49,6 +50,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected String queueName;
private String queuePath;
protected int maxParallelApps;
+ private boolean isAbsoluteResource;
protected CapacitySchedulerQueueInfoList queues;
protected QueueCapacitiesInfo capacities;
protected CapacitySchedulerHealthInfo health;
@@ -90,6 +92,9 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
health = new CapacitySchedulerHealthInfo(cs);
maximumAllocation = new ResourceInfo(parent.getMaximumAllocation());
+isAbsoluteResource = parent.getCapacityConfigType() ==
+AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
+
CapacitySchedulerConfiguration conf = cs.getConfiguration();
queueAcls = new QueueAclsInfo();
queueAcls.addAll(getSortedQueueAclInfoList(queueName, conf));
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index e27054d..78b5392 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.security.AccessType;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+import
[hadoop] branch trunk updated: YARN-10982. Replace all occurences of queuePath with the new QueuePath class. Contributed by Tibor Kovacs
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a0d8cde YARN-10982. Replace all occurences of queuePath with the new
QueuePath class. Contributed by Tibor Kovacs
a0d8cde is described below
commit a0d8cde133a7d8719217567efaa4e66c583f790d
Author: Szilard Nemeth
AuthorDate: Thu Dec 9 17:51:44 2021 +0100
YARN-10982. Replace all occurences of queuePath with the new QueuePath
class. Contributed by Tibor Kovacs
---
.../scheduler/capacity/AbstractCSQueue.java| 25 +++---
.../capacity/AbstractManagedParentQueue.java | 2 +-
.../capacity/AutoCreatedQueueTemplate.java | 14 ++--
.../scheduler/capacity/CSQueue.java| 6 ++
.../scheduler/capacity/CSQueueUtils.java | 8 +-
.../capacity/CapacitySchedulerConfiguration.java | 57 +++---
.../scheduler/capacity/ManagedParentQueue.java | 9 +--
.../scheduler/capacity/ParentQueue.java| 22 +++---
.../scheduler/capacity/QueuePath.java | 23 ++
.../TestAbsoluteResourceConfiguration.java | 90 +++---
.../TestAbsoluteResourceWithAutoQueue.java | 18 ++---
.../capacity/TestAutoCreatedQueueTemplate.java | 57 +++---
.../capacity/TestCSAllocateCustomResource.java | 4 +-
.../scheduler/capacity/TestCapacityScheduler.java | 9 ++-
.../TestCapacitySchedulerConfigValidator.java | 35 +
.../scheduler/capacity/TestParentQueue.java| 4 +-
.../scheduler/capacity/TestQueuePath.java | 10 +++
.../TestRMWebServicesConfigurationMutation.java| 15 ++--
18 files changed, 233 insertions(+), 175 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index efdfa8e..097a9df 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -82,7 +82,6 @@ public abstract class AbstractCSQueue implements CSQueue {
protected final QueueAllocationSettings queueAllocationSettings;
volatile CSQueue parent;
protected final QueuePath queuePath;
- final String queueName;
protected QueueNodeLabelsSettings queueNodeLabelsSettings;
private volatile QueueAppLifetimeAndLimitSettings queueAppLifetimeSettings;
private CSQueuePreemptionSettings preemptionSettings;
@@ -143,7 +142,6 @@ public abstract class AbstractCSQueue implements CSQueue {
this.labelManager = cs.getRMContext().getNodeLabelManager();
this.parent = parent;
this.queuePath = createQueuePath(parent, queueName);
-this.queueName = queuePath.getLeafName();
this.resourceCalculator = cs.getResourceCalculator();
this.activitiesManager = cs.getActivitiesManager();
@@ -176,7 +174,7 @@ public abstract class AbstractCSQueue implements CSQueue {
protected void setupConfigurableCapacities(
CapacitySchedulerConfiguration configuration) {
-CSQueueUtils.loadCapacitiesByLabelsFromConf(getQueuePath(),
queueCapacities,
+CSQueueUtils.loadCapacitiesByLabelsFromConf(queuePath, queueCapacities,
configuration, this.queueNodeLabelsSettings.getConfiguredNodeLabels());
}
@@ -186,6 +184,11 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
+ public QueuePath getQueuePathObject() {
+return this.queuePath;
+ }
+
+ @Override
public float getCapacity() {
return queueCapacities.getCapacity();
}
@@ -241,7 +244,7 @@ public abstract class AbstractCSQueue implements CSQueue {
@Override
public String getQueueName() {
-return queueName;
+return this.queuePath.getLeafName();
}
@Override
@@ -279,11 +282,11 @@ public abstract class AbstractCSQueue implements CSQueue {
writeLock.lock();
try {
// Sanity check
- CSQueueUtils.checkMaxCapacity(getQueuePath(),
+ CSQueueUtils.checkMaxCapacity(this.queuePath,
queueCapacities.getCapacity(), maximumCapacity);
float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity(
maximumCapacity, parent);
- CSQueueUtils.checkAbsoluteCapacity(getQueuePath(),
+ CSQueueUtils.checkAbsoluteCapacity(this.queuePath,
queueCapacities.getAbsoluteCapacity(), absMaxCapacity
[hadoop] branch branch-3.3 updated: YARN-11020. [UI2] No container is found for an application attempt with a single AM container. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 8053b0f YARN-11020. [UI2] No container is found for an application
attempt with a single AM container. Contributed by Andras Gyori
8053b0f is described below
commit 8053b0f2059c3ddfb80ece12c08d545a3a24eba6
Author: Szilard Nemeth
AuthorDate: Thu Dec 9 13:02:55 2021 +0100
YARN-11020. [UI2] No container is found for an application attempt with a
single AM container. Contributed by Andras Gyori
---
.../src/main/webapp/app/serializers/yarn-jhs-container.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
index 599cf7f..8aebb71 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
@@ -45,8 +45,8 @@ export default DS.JSONAPISerializer.extend({
},
normalizeArrayResponse(store, primaryModelClass, payload/*, id,
requestType*/) {
-
-payload = payload["containerLogsInfo"]
+// Handling single container and multiple containers case at the same time
+payload = [].concat(payload["containerLogsInfo"]);
var normalizedArrayResponse = {
data: []
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11038. Fix testQueueSubmitWithACL* tests in TestAppManager. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 66a08e9 YARN-11038. Fix testQueueSubmitWithACL* tests in
TestAppManager. Contributed by Tamas Domok
66a08e9 is described below
commit 66a08e933ae8f1737f8771e1316bb44c97aa6500
Author: Szilard Nemeth
AuthorDate: Wed Dec 8 16:41:35 2021 +0100
YARN-11038. Fix testQueueSubmitWithACL* tests in TestAppManager.
Contributed by Tamas Domok
---
.../server/resourcemanager/TestAppManager.java | 129 ++---
1 file changed, 62 insertions(+), 67 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index f916264..4a393cc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -274,10 +274,41 @@ public class TestAppManager extends AppManagerTestBase{
setupDispatcher(rmContext, conf);
}
+ private static PlacementManager createMockPlacementManager(
+ String userRegex, String placementQueue, String placementParentQueue
+ ) throws YarnException {
+PlacementManager placementMgr = mock(PlacementManager.class);
+doAnswer(new Answer() {
+
+ @Override
+ public ApplicationPlacementContext answer(InvocationOnMock invocation)
+ throws Throwable {
+return new ApplicationPlacementContext(placementQueue,
placementParentQueue);
+ }
+
+}).when(placementMgr).placeApplication(
+any(ApplicationSubmissionContext.class),
+matches(userRegex),
+any(Boolean.class));
+
+return placementMgr;
+ }
+
+ private TestRMAppManager createAppManager(RMContext context, Configuration
configuration) {
+ApplicationMasterService masterService = new
ApplicationMasterService(context,
+context.getScheduler());
+
+return new TestRMAppManager(context,
+new ClientToAMTokenSecretManagerInRM(),
+context.getScheduler(), masterService,
+new ApplicationACLsManager(configuration), configuration);
+ }
+
@Test
public void testQueueSubmitWithACLsEnabledWithQueueMapping()
- throws IOException, YarnException, InterruptedException {
-YarnConfiguration conf = new YarnConfiguration();
+ throws YarnException {
+YarnConfiguration conf = new YarnConfiguration(new Configuration(false));
+conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
@@ -285,11 +316,11 @@ public class TestAppManager extends AppManagerTestBase{
CapacitySchedulerConfiguration(conf, false);
csConf.set(PREFIX + "root.queues", "default,test");
-csConf.setFloat(PREFIX + "root.default.capacity", 50.0f);
-csConf.setFloat(PREFIX + "root.default.maximum-capacity", 100.0f);
+csConf.setCapacity("root.default", 50.0f);
+csConf.setMaximumCapacity("root.default", 100.0f);
-csConf.setFloat(PREFIX + "root.test.capacity", 50.0f);
-csConf.setFloat(PREFIX + "root.test.maximum-capacity", 100.0f);
+csConf.setCapacity("root.test", 50.0f);
+csConf.setMaximumCapacity("root.test", 100.0f);
csConf.set(PREFIX + "root.acl_submit_applications", " ");
csConf.set(PREFIX + "root.acl_administer_queue", " ");
@@ -300,55 +331,30 @@ public class TestAppManager extends AppManagerTestBase{
csConf.set(PREFIX + "root.test.acl_submit_applications", "test");
csConf.set(PREFIX + "root.test.acl_administer_queue", "test");
-csConf.set(PREFIX + "root.test.acl_submit_applications", "test");
-csConf.set(PREFIX + "root.test.acl_administer_queue", "test");
-
-csConf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
-
-// Setup a PlacementManager returns a new queue
-PlacementManager placementMgr = mock(PlacementManager.class);
-doAnswer(new Answer() {
-
- @Override
- public ApplicationPlacementContext answer(InvocationOnMock invocation)
- throws Throwable {
-return new ApplicationPlacementContext("test");
- }
-
-}).when(placementMgr).placeAppli
[hadoop] branch trunk updated: YARN-11016. Queue weight is incorrectly reset to zero. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1260796 YARN-11016. Queue weight is incorrectly reset to zero.
Contributed by Andras Gyori
1260796 is described below
commit 126079612c8b567041ec3ea466e787a932bdf635
Author: Szilard Nemeth
AuthorDate: Tue Dec 7 15:55:16 2021 +0100
YARN-11016. Queue weight is incorrectly reset to zero. Contributed by
Andras Gyori
---
.../scheduler/capacity/QueueCapacities.java| 2 +-
.../capacity/TestCapacitySchedulerWeightMode.java | 46 ++
2 files changed, 47 insertions(+), 1 deletion(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
index 2d82966..0edf83f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
@@ -312,7 +312,7 @@ public class QueueCapacities {
_set(label, CapacityType.MAX_CAP, 0);
_set(label, CapacityType.ABS_CAP, 0);
_set(label, CapacityType.ABS_MAX_CAP, 0);
-_set(label, CapacityType.WEIGHT, 0);
+_set(label, CapacityType.WEIGHT, -1);
}
} finally {
writeLock.unlock();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
index a5d80dc..115813a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.util.Sets;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -38,6 +39,7 @@ import
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
@@ -74,6 +76,28 @@ public class TestCapacitySchedulerWeightMode {
return set;
}
+ public static CapacitySchedulerConfiguration
getConfigWithInheritedAccessibleNodeLabel(
+ Configuration config) {
+CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(
+config);
+
+// Define top-level queues
+conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+new String[] { "a"});
+
+conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 100f);
+conf.setCapacityByLabel(A, "newLabel", 100f);
+conf.setAccessibleNodeLabels(A, toSet("newLabel"));
+conf.setAllowZeroCapacitySum(A, true);
+
+// Define 2nd-level queues
+conf.setQueues(A, new String[] { "a1" });
+conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100f);
+
+return conf;
+ }
+
+
/*
* Queue structure:
* root (*)
@@ -368,6 +392,28 @@ public class TestCapacitySchedulerWeightMode {
rm.close();
}
+ /**
+ * Tests whether weight is correctly reset to
[hadoop] branch trunk updated: YARN-11006. Allow overriding user limit factor and maxAMResourcePercent with AQCv2 templates. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7284d23 YARN-11006. Allow overriding user limit factor and
maxAMResourcePercent with AQCv2 templates. Contributed by Benjamin Teke
7284d23 is described below
commit 7284d234765547072ba00477128d811aa302fc3f
Author: Szilard Nemeth
AuthorDate: Tue Dec 7 15:43:55 2021 +0100
YARN-11006. Allow overriding user limit factor and maxAMResourcePercent
with AQCv2 templates. Contributed by Benjamin Teke
---
.../resourcemanager/scheduler/capacity/LeafQueue.java| 2 +-
.../TestCapacitySchedulerNewQueueAutoCreation.java | 16
2 files changed, 17 insertions(+), 1 deletion(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index c7f42d1..4592f2a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -1707,12 +1707,12 @@ public class LeafQueue extends AbstractCSQueue {
@Override
protected void setDynamicQueueProperties(
CapacitySchedulerConfiguration configuration) {
-super.setDynamicQueueProperties(configuration);
// set to -1, to disable it
configuration.setUserLimitFactor(getQueuePath(), -1);
// Set Max AM percentage to a higher value
configuration.setMaximumApplicationMasterResourcePerQueuePercent(
getQueuePath(), 1f);
+super.setDynamicQueueProperties(configuration);
}
private void updateSchedulerHealthForCompletedContainer(
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index 9b23fdd..3d40ccf 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -688,11 +688,27 @@ public class TestCapacitySchedulerNewQueueAutoCreation
LeafQueue a2 = createQueue("root.a.a-auto.a2");
Assert.assertEquals("weight is not set by template", 6f,
a2.getQueueCapacities().getWeight(), 1e-6);
+Assert.assertEquals("user limit factor should be disabled with dynamic
queues",
+-1f, a2.getUserLimitFactor(), 1e-6);
+Assert.assertEquals("maximum AM resource percent should be 1 with dynamic
queues",
+1f, a2.getMaxAMResourcePerQueuePercent(), 1e-6);
+
+// Set the user-limit-factor and maximum-am-resource-percent via templates
to ensure their
+// modified defaults are indeed overridden
+csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix(
+"root.a.*") + "user-limit-factor", "10");
+csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix(
+"root.a.*") + "maximum-am-resource-percent", "0.8");
cs.reinitialize(csConf, mockRM.getRMContext());
a2 = (LeafQueue) cs.getQueue("root.a.a-auto.a2");
Assert.assertEquals("weight is overridden", 6f,
a2.getQueueCapacities().getWeight(), 1e-6);
+Assert.assertEquals("user limit factor should be modified by templates",
+10f, a2.getUserLimitFactor(), 1e-6);
+Assert.assertEquals("maximum AM resource percent should be modified by
templates",
+0.8f, a2.getMaxAMResourcePerQueuePercent(), 1e-6);
+
csConf.setNonLabeledQueueWeight("root.a.a-auto.a2", 4f);
cs.reinitialize(csConf, mockRM.getRMContext());
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11023. Extend the root QueueInfo with max-parallel-apps in CapacityScheduler. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 60440a8 YARN-11023. Extend the root QueueInfo with max-parallel-apps
in CapacityScheduler. Contributed by Tamas Domok
60440a8 is described below
commit 60440a836ab602b9c76efbc78c05c4621a8f7b45
Author: Szilard Nemeth
AuthorDate: Tue Dec 7 15:26:07 2021 +0100
YARN-11023. Extend the root QueueInfo with max-parallel-apps in
CapacityScheduler. Contributed by Tamas Domok
---
.../resourcemanager/webapp/dao/CapacitySchedulerInfo.java | 2 ++
.../webapp/TestRMWebServicesCapacitySched.java| 11 +++
.../webapp/TestRMWebServicesForCSWithPartitions.java | 2 +-
3 files changed, 10 insertions(+), 5 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index 79d11e2..abf53d6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -50,6 +50,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected float normalizedWeight;
protected String queueName;
private String queuePath;
+ protected int maxParallelApps;
protected CapacitySchedulerQueueInfoList queues;
protected QueueCapacitiesInfo capacities;
protected CapacitySchedulerHealthInfo health;
@@ -83,6 +84,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
this.maxCapacity = max * 100;
this.weight = parent.getQueueCapacities().getWeight();
this.normalizedWeight = parent.getQueueCapacities().getNormalizedWeight();
+this.maxParallelApps = parent.getMaxParallelApps();
capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
parent.getQueueResourceQuotas(), false);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index d4e7ca6..e9df9d3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -270,7 +270,8 @@ public class TestRMWebServicesCapacitySched extends
JerseyTestBase {
WebServicesTestUtils.getXmlFloat(element, "capacity"),
WebServicesTestUtils.getXmlFloat(element, "maxCapacity"),
WebServicesTestUtils.getXmlString(element, "queueName"),
- WebServicesTestUtils.getXmlString(element, "queuePath"));
+ WebServicesTestUtils.getXmlString(element, "queuePath"),
+ WebServicesTestUtils.getXmlInt(element, "maxParallelApps"));
NodeList children = element.getChildNodes();
for (int j = 0; j < children.getLength(); j++) {
@@ -374,13 +375,14 @@ public class TestRMWebServicesCapacitySched extends
JerseyTestBase {
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo");
-assertEquals("incorrect number of elements in: " + info, 22,
info.length());
+assertEquals("incorrect number of elements in: " + info, 23,
info.length());
verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"),
(float) info.getDouble("maxCapacity"),
info.getString("queueName"),
-info.getString("queuePath"));
+info.getString("queuePath"),
+info.getInt("maxParallelApps"));
JSONObject health = info.getJSONObject("health");
[hadoop] branch trunk updated: YARN-10757. jsonschema2pojo-maven-plugin version is not defined. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 79cc7b4 YARN-10757. jsonschema2pojo-maven-plugin version is not
defined. Contributed by Tamas Domok
79cc7b4 is described below
commit 79cc7b46aff8155b5501acdfd1c605aaae6a6a4d
Author: Szilard Nemeth
AuthorDate: Tue Dec 7 14:56:55 2021 +0100
YARN-10757. jsonschema2pojo-maven-plugin version is not defined.
Contributed by Tamas Domok
---
pom.xml | 6 ++
1 file changed, 6 insertions(+)
diff --git a/pom.xml b/pom.xml
index 5bdf9fd..09f1ccb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -116,6 +116,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
https://maven.apache.org/x
1.4.3
4.2.2
4.2.0
+
1.1.1
bash
@@ -472,6 +473,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
https://maven.apache.org/x
+
+ org.jsonschema2pojo
+ jsonschema2pojo-maven-plugin
+ ${jsonschema2pojo-maven-plugin.version}
+
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11020. [UI2] No container is found for an application attempt with a single AM container. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7a8c2e0 YARN-11020. [UI2] No container is found for an application
attempt with a single AM container. Contributed by Andras Gyori
7a8c2e0 is described below
commit 7a8c2e009761b8405406e185ab6302113be1f487
Author: Szilard Nemeth
AuthorDate: Tue Dec 7 08:51:03 2021 +0100
YARN-11020. [UI2] No container is found for an application attempt with a
single AM container. Contributed by Andras Gyori
---
.../src/main/webapp/app/serializers/yarn-jhs-container.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
index 599cf7f..8aebb71 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
@@ -45,8 +45,8 @@ export default DS.JSONAPISerializer.extend({
},
normalizeArrayResponse(store, primaryModelClass, payload/*, id,
requestType*/) {
-
-payload = payload["containerLogsInfo"]
+// Handling single container and multiple containers case at the same time
+payload = [].concat(payload["containerLogsInfo"]);
var normalizedArrayResponse = {
data: []
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11014. YARN incorrectly validates maximum capacity resources on the validation API. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d75f2db YARN-11014. YARN incorrectly validates maximum capacity
resources on the validation API. Contributed by Benjamin Teke
d75f2db is described below
commit d75f2db2f952c5cb785345d912f169d0d45a491b
Author: Szilard Nemeth
AuthorDate: Tue Dec 7 08:42:57 2021 +0100
YARN-11014. YARN incorrectly validates maximum capacity resources on the
validation API. Contributed by Benjamin Teke
---
.../scheduler/capacity/CapacityScheduler.java | 16 ++
.../capacity/CapacitySchedulerConfigValidator.java | 2 +
.../TestCapacitySchedulerConfigValidator.java | 270 -
3 files changed, 284 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index d2c84e7..bd1089b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2239,6 +2239,22 @@ public class CapacityScheduler extends
}
}
+ /**
+ * Add node to nodeTracker. Used when validating CS configuration by
instantiating a new
+ * CS instance.
+ * @param nodesToAdd node to be added
+ */
+ public void addNodes(List nodesToAdd) {
+writeLock.lock();
+try {
+ for (FiCaSchedulerNode node : nodesToAdd) {
+nodeTracker.addNode(node);
+ }
+} finally {
+ writeLock.unlock();
+}
+ }
+
private void addNode(RMNode nodeManager) {
writeLock.lock();
try {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
index ca0d586..fd601ac 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
@@ -42,6 +42,7 @@ public final class CapacitySchedulerConfigValidator {
public static boolean validateCSConfiguration(
final Configuration oldConf, final Configuration newConf,
final RMContext rmContext) throws IOException {
+CapacityScheduler liveScheduler = (CapacityScheduler)
rmContext.getScheduler();
CapacityScheduler newCs = new CapacityScheduler();
try {
//TODO: extract all the validation steps and replace reinitialize with
@@ -49,6 +50,7 @@ public final class CapacitySchedulerConfigValidator {
newCs.setConf(oldConf);
newCs.setRMContext(rmContext);
newCs.init(oldConf);
+ newCs.addNodes(liveScheduler.getAllNodes());
newCs.reinitialize(newConf, rmContext, true);
return true;
} finally {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
index 04f4349..ad114d9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
@@ -19,13 +19,23 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.conf.Configuration;
+import
[hadoop] branch trunk updated: YARN-11001. Add docs on removing node label mapping from a node. Contributed by Manu Zhang
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new b933f5f YARN-11001. Add docs on removing node label mapping from a node. Contributed by Manu Zhang b933f5f is described below commit b933f5f54b7cc6e46f33da3e15065ab9ebe52939 Author: Szilard Nemeth AuthorDate: Mon Nov 15 16:02:18 2021 +0100 YARN-11001. Add docs on removing node label mapping from a node. Contributed by Manu Zhang --- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md| 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md index 39d8311..32545f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md @@ -85,7 +85,8 @@ Notes: ###Add/modify node-to-labels mapping to YARN * Configuring nodes to labels mapping in **Centralized** NodeLabel setup -* Executing ```yarn rmadmin -replaceLabelsOnNode “node1[:port]=label1 node2=label2” [-failOnUnknownNodes]```. Added label1 to node1, label2 to node2. If user don’t specify port, it adds the label to all ```NodeManagers``` running on the node. If option ```-failOnUnknownNodes``` is set, this command will fail if specified nodes are unknown. +* Executing ```yarn rmadmin -replaceLabelsOnNode "node1[:port]=label1 node2=label2" [-failOnUnknownNodes]``` adds label1 to node1, label2 to node2. If user doesn’t specify port, it adds the label to all ```NodeManagers``` running on the node. If option ```-failOnUnknownNodes``` is set, this command will fail if specified nodes are unknown. +* To remove mapping from a node, use `-replaceLabelsOnNode` but do not specify any labels. For example, ```yarn rmadmin -replaceLabelsOnNode "node1"``` removes label1 from node1. * Configuring nodes to labels mapping in **Distributed** NodeLabel setup - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10997. Revisit allocation and reservation logging. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7cb887e YARN-10997. Revisit allocation and reservation logging.
Contributed by Andras Gyori
7cb887e is described below
commit 7cb887e6c2bfc058bb7b7d7bde2554d261432bdb
Author: Szilard Nemeth
AuthorDate: Fri Nov 12 15:42:14 2021 +0100
YARN-10997. Revisit allocation and reservation logging. Contributed by
Andras Gyori
---
.../scheduler/capacity/CapacityScheduler.java | 2 --
.../scheduler/common/fica/FiCaSchedulerApp.java | 13 +
2 files changed, 9 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index b8091c7..09ae99e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -3294,13 +3294,11 @@ public class CapacityScheduler extends
long commitSuccess = System.nanoTime() - commitStart;
CapacitySchedulerMetrics.getMetrics()
.addCommitSuccess(commitSuccess);
- LOG.info("Allocation proposal accepted");
isSuccess = true;
} else{
long commitFailed = System.nanoTime() - commitStart;
CapacitySchedulerMetrics.getMetrics()
.addCommitFailure(commitFailed);
- LOG.info("Failed to accept allocation proposal");
}
LOG.debug("Allocation proposal accepted={}, proposal={}", isSuccess,
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index a1b8205..011a254 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -357,6 +357,7 @@ public class FiCaSchedulerApp extends
SchedulerApplicationAttempt {
// adding NP check as this proposal could not be allocated from reserved
// container in async-scheduling mode
if (allocation.getAllocateFromReservedContainer() == null) {
+LOG.debug("Trying to allocate from reserved container in async
scheduling mode");
return false;
}
RMContainer fromReservedContainer =
@@ -572,6 +573,8 @@ public class FiCaSchedulerApp extends
SchedulerApplicationAttempt {
if (updatePending &&
getOutstandingAsksCount(schedulerContainer.getSchedulerRequestKey())
<= 0) {
+ LOG.debug("Rejecting appliance of allocation due to existing pending
allocation " +
+ "request for " + schedulerContainer);
return false;
}
@@ -670,10 +673,12 @@ public class FiCaSchedulerApp extends
SchedulerApplicationAttempt {
schedulerContainer.getRmContainer().getContainer(),
reReservation);
- LOG.info("Reserved container=" + rmContainer.getContainerId()
- + ", on node=" + schedulerContainer.getSchedulerNode()
- + " with resource=" + rmContainer
- .getAllocatedOrReservedResource());
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Reserved container=" + rmContainer.getContainerId()
++ ", on node=" + schedulerContainer.getSchedulerNode()
++ " with resource=" + rmContainer
+.getAllocatedOrReservedResource());
+ }
}
}
} finally {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10997. Revisit allocation and reservation logging. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2da7364 YARN-10997. Revisit allocation and reservation logging.
Contributed by Andras Gyori
2da7364 is described below
commit 2da7364c7e052bb1e1f2005b187efafa10b69c2f
Author: Szilard Nemeth
AuthorDate: Fri Nov 12 15:42:14 2021 +0100
YARN-10997. Revisit allocation and reservation logging. Contributed by
Andras Gyori
---
.../scheduler/capacity/CapacityScheduler.java | 2 --
.../scheduler/common/fica/FiCaSchedulerApp.java | 13 +
2 files changed, 9 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index b8091c7..09ae99e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -3294,13 +3294,11 @@ public class CapacityScheduler extends
long commitSuccess = System.nanoTime() - commitStart;
CapacitySchedulerMetrics.getMetrics()
.addCommitSuccess(commitSuccess);
- LOG.info("Allocation proposal accepted");
isSuccess = true;
} else{
long commitFailed = System.nanoTime() - commitStart;
CapacitySchedulerMetrics.getMetrics()
.addCommitFailure(commitFailed);
- LOG.info("Failed to accept allocation proposal");
}
LOG.debug("Allocation proposal accepted={}, proposal={}", isSuccess,
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index a1b8205..011a254 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -357,6 +357,7 @@ public class FiCaSchedulerApp extends
SchedulerApplicationAttempt {
// adding NP check as this proposal could not be allocated from reserved
// container in async-scheduling mode
if (allocation.getAllocateFromReservedContainer() == null) {
+LOG.debug("Trying to allocate from reserved container in async
scheduling mode");
return false;
}
RMContainer fromReservedContainer =
@@ -572,6 +573,8 @@ public class FiCaSchedulerApp extends
SchedulerApplicationAttempt {
if (updatePending &&
getOutstandingAsksCount(schedulerContainer.getSchedulerRequestKey())
<= 0) {
+ LOG.debug("Rejecting appliance of allocation due to existing pending
allocation " +
+ "request for " + schedulerContainer);
return false;
}
@@ -670,10 +673,12 @@ public class FiCaSchedulerApp extends
SchedulerApplicationAttempt {
schedulerContainer.getRmContainer().getContainer(),
reReservation);
- LOG.info("Reserved container=" + rmContainer.getContainerId()
- + ", on node=" + schedulerContainer.getSchedulerNode()
- + " with resource=" + rmContainer
- .getAllocatedOrReservedResource());
+ if (LOG.isDebugEnabled()) {
+LOG.debug("Reserved container=" + rmContainer.getContainerId()
++ ", on node=" + schedulerContainer.getSchedulerNode()
++ " with resource=" + rmContainer
+.getAllocatedOrReservedResource());
+ }
}
}
} finally {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10996. Fix race condition of User object acquisitions. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e220e88 YARN-10996. Fix race condition of User object acquisitions.
Contributed by Andras Gyori
e220e88 is described below
commit e220e88eca26311af707f7969da36d402f022a8d
Author: Szilard Nemeth
AuthorDate: Fri Nov 12 15:33:39 2021 +0100
YARN-10996. Fix race condition of User object acquisitions. Contributed by
Andras Gyori
---
.../capacity/FifoIntraQueuePreemptionPlugin.java| 10 ++
.../scheduler/capacity/LeafQueue.java | 21 ++---
.../scheduler/capacity/UsersManager.java| 3 ++-
.../capacity/mockframework/MockApplications.java| 1 +
4 files changed, 27 insertions(+), 8 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
index a9d3c05..ea17fed 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
@@ -30,6 +30,7 @@ import java.util.PriorityQueue;
import java.util.Set;
import java.util.TreeSet;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -419,8 +420,10 @@ public class FifoIntraQueuePreemptionPlugin
String userName = app.getUser();
TempUserPerPartition tmpUser = usersPerPartition.get(userName);
if (tmpUser == null) {
-ResourceUsage userResourceUsage = tq.leafQueue.getUser(userName)
-.getResourceUsage();
+// User might have already been removed, but preemption still accounts
for this app,
+// therefore reinserting the user will not cause a memory leak
+User user = tq.leafQueue.getOrCreateUser(userName);
+ResourceUsage userResourceUsage = user.getResourceUsage();
// perUserAMUsed was populated with running apps, now we are looping
// through both running and pending apps.
@@ -428,8 +431,7 @@ public class FifoIntraQueuePreemptionPlugin
amUsed = (userSpecificAmUsed == null)
? Resources.none() : userSpecificAmUsed;
-tmpUser = new TempUserPerPartition(
-tq.leafQueue.getUser(userName), tq.queueName,
+tmpUser = new TempUserPerPartition(user, tq.queueName,
Resources.clone(userResourceUsage.getUsed(partition)),
Resources.clone(amUsed),
Resources.clone(userResourceUsage.getReserved(partition)),
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index a4e2a82..c7f42d1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -519,6 +519,11 @@ public class LeafQueue extends AbstractCSQueue {
return usersManager.getUser(userName);
}
+ @VisibleForTesting
+ public User getOrCreateUser(String userName) {
+return usersManager.getUserAndAddIfAbsent(userName);
+ }
+
@Private
public List getPriorityACLs() {
readLock.lock();
@@ -2007,7 +2012,12 @@ public class LeafQueue extends AbstractCSQueue {
public void incAMUsedResource(String nodeLabel, Resource resourceToInc,
SchedulerApplicationAttempt application) {
-getUser(application.getUser()).getResourceUsage().incAMUsed(nodeLabel,
+User user = getUser(application.getUser());
+if (user == null) {
+ return;
+}
+
+user.getResourceUsage().incAMUsed(nodeLabel,
resourceToInc);
// ResourceUsage has its own lock, no addition lock needs here
[hadoop] branch trunk updated: YARN-10909. AbstractCSQueue: Annotate all methods with VisibleForTesting that are only used by test code. Contributed by JackWangCS, Szilard Nemeth
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9cfd8d0 YARN-10909. AbstractCSQueue: Annotate all methods with
VisibleForTesting that are only used by test code. Contributed by JackWangCS,
Szilard Nemeth
9cfd8d0 is described below
commit 9cfd8d0a83fac8fd54d9d744b79c8898d7adaebe
Author: Jack
AuthorDate: Sat Oct 23 14:47:00 2021 +0200
YARN-10909. AbstractCSQueue: Annotate all methods with VisibleForTesting
that are only used by test code. Contributed by JackWangCS, Szilard Nemeth
---
.../resourcemanager/scheduler/capacity/AbstractCSQueue.java | 11 +++
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index e3feb51..e89a547 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -287,6 +287,7 @@ public abstract class AbstractCSQueue implements CSQueue {
* Set maximum capacity - used only for testing.
* @param maximumCapacity new max capacity
*/
+ @VisibleForTesting
void setMaxCapacity(float maximumCapacity) {
writeLock.lock();
try {
@@ -1023,7 +1024,8 @@ public abstract class AbstractCSQueue implements CSQueue {
return getEffectiveMaxCapacity(nodePartition);
}
- public boolean hasChildQueues() {
+ @VisibleForTesting
+ boolean hasChildQueues() {
List childQueues = getChildQueues();
return childQueues != null && !childQueues.isEmpty();
}
@@ -1608,7 +1610,7 @@ public abstract class AbstractCSQueue implements CSQueue {
getAutoExpiredDeletionTime());
}
- public void updateLastSubmittedTimeStamp() {
+ void updateLastSubmittedTimeStamp() {
writeLock.lock();
try {
usageTracker.setLastSubmittedTimestamp(Time.monotonicNow());
@@ -1617,7 +1619,8 @@ public abstract class AbstractCSQueue implements CSQueue {
}
}
- public long getLastSubmittedTimestamp() {
+ @VisibleForTesting
+ long getLastSubmittedTimestamp() {
readLock.lock();
try {
@@ -1628,7 +1631,7 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@VisibleForTesting
- public void setLastSubmittedTimestamp(long lastSubmittedTimestamp) {
+ void setLastSubmittedTimestamp(long lastSubmittedTimestamp) {
writeLock.lock();
try {
usageTracker.setLastSubmittedTimestamp(lastSubmittedTimestamp);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10930. Introduce universal capacity resource vector. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 32ecaed YARN-10930. Introduce universal capacity resource vector.
Contributed by Andras Gyori
32ecaed is described below
commit 32ecaed9c3c06a48ef01d0437e62e8faccd3e9f3
Author: 9uapaw
AuthorDate: Fri Oct 22 17:32:33 2021 +0200
YARN-10930. Introduce universal capacity resource vector. Contributed by
Andras Gyori
---
.../scheduler/capacity/AbstractCSQueue.java| 12 +-
.../scheduler/capacity/CSQueue.java| 8 +
.../capacity/CapacitySchedulerConfiguration.java | 19 +-
.../scheduler/capacity/QueueCapacityVector.java| 258 +
.../scheduler/capacity/ResourceVector.java | 129 +++
.../capacity/conf/QueueCapacityConfigParser.java | 215 +
.../capacity/TestQueueCapacityVector.java | 111 +
.../scheduler/capacity/TestResourceVector.java | 118 ++
.../conf/TestQueueCapacityConfigParser.java| 241 +++
9 files changed, 1108 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 2d9bf85..e3feb51 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -131,6 +131,8 @@ public abstract class AbstractCSQueue implements CSQueue {
protected CapacityConfigType capacityConfigType =
CapacityConfigType.NONE;
+ protected Map configuredCapacityVectors;
+
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
protected CapacitySchedulerContext csContext;
@@ -374,6 +376,8 @@ public abstract class AbstractCSQueue implements CSQueue {
this.reservationsContinueLooking =
configuration.getReservationContinueLook();
+ this.configuredCapacityVectors = csContext.getConfiguration()
+ .parseConfiguredResourceVector(queuePath, configuredNodeLabels);
// Update metrics
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
@@ -688,6 +692,12 @@ public abstract class AbstractCSQueue implements CSQueue {
minimumAllocation);
}
+ @Override
+ public QueueCapacityVector getConfiguredCapacityVector(
+ String label) {
+return configuredCapacityVectors.get(label);
+ }
+
private void initializeQueueState(CapacitySchedulerConfiguration
configuration) {
QueueState previousState = getState();
QueueState configuredState = configuration
@@ -978,7 +988,7 @@ public abstract class AbstractCSQueue implements CSQueue {
"Default lifetime " + defaultAppLifetime
+ " can't exceed maximum lifetime " + myMaxAppLifetime);
}
-
+
if (defaultAppLifetime <= 0) {
defaultAppLifetime = myMaxAppLifetime;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 03a5afb..2acc1d4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -420,6 +420,14 @@ public interface CSQueue extends SchedulerQueue {
Resource getEffectiveCapacity(String label);
/**
+ * Get configured capacity resource vector parsed from the capacity config
+ * of the queue.
+ * @param label node label (partition)
+ * @return capacity resource vector
+ */
+ QueueCapacityVector getConfiguredCapacityVector(String label);
+
+ /**
* Get effective capacity of queue. If min/max resource is configured,
* preference will be given to absolute configuration over normal capacity.
* Also round down the result to normalizeDown.
di
[hadoop] branch trunk updated: YARN-10948. Rename SchedulerQueue#activeQueue to activateQueue. Contributed by Adam Antal
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 23772d9 YARN-10948. Rename SchedulerQueue#activeQueue to
activateQueue. Contributed by Adam Antal
23772d9 is described below
commit 23772d946bdf26a72fb611a9719afe743cdebd52
Author: Adam Antal
AuthorDate: Fri Oct 22 16:33:03 2021 +0200
YARN-10948. Rename SchedulerQueue#activeQueue to activateQueue. Contributed
by Adam Antal
---
.../yarn/server/resourcemanager/scheduler/QueueStateManager.java | 2 +-
.../hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueue.java | 4 ++--
.../server/resourcemanager/scheduler/capacity/AbstractCSQueue.java| 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java
index 0127e1e..ab0f445 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java
@@ -77,7 +77,7 @@ public class QueueStateManager
extends Queue {
void stopQueue();
/**
- * Active the queue.
+ * Activate the queue.
* @throws YarnException if the queue can not be activated.
*/
- void activeQueue() throws YarnException;
+ void activateQueue() throws YarnException;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index ca861fb..2d9bf85 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -733,7 +733,7 @@ public abstract class AbstractCSQueue implements CSQueue {
} else {
if (configuredState == QueueState.RUNNING) {
try {
-activeQueue();
+activateQueue();
} catch (YarnException ex) {
throw new IllegalArgumentException(ex.getMessage());
}
@@ -1322,7 +1322,7 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public void activeQueue() throws YarnException {
+ public void activateQueue() throws YarnException {
this.writeLock.lock();
try {
if (getState() == QueueState.RUNNING) {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10954. Remove commented code block from CSQueueUtils#loadCapacitiesByLabelsFromConf. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 616cea2 YARN-10954. Remove commented code block from
CSQueueUtils#loadCapacitiesByLabelsFromConf. Contributed by Andras Gyori
616cea2 is described below
commit 616cea2e8068e990d24057d2b0d6090f35e21371
Author: 9uapaw
AuthorDate: Tue Oct 19 13:06:35 2021 +0200
YARN-10954. Remove commented code block from
CSQueueUtils#loadCapacitiesByLabelsFromConf. Contributed by Andras Gyori
---
.../server/resourcemanager/scheduler/capacity/CSQueueUtils.java | 9 -
1 file changed, 9 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index f43ea30..410117a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -92,15 +92,6 @@ public class CSQueueUtils {
queueCapacities.setWeight(label,
csConf.getLabeledQueueWeight(queuePath, label));
}
-
- /*float absCapacity = queueCapacities.getCapacity(label);
- float absMaxCapacity = queueCapacities.getMaximumCapacity(label);
- if (absCapacity > absMaxCapacity) {
-throw new IllegalArgumentException("Illegal queue capacity setting "
-+ "(abs-capacity=" + absCapacity + ") > (abs-maximum-capacity="
-+ absMaxCapacity + ") for queue=["
-+ queuePath + "],label=[" + label + "]");
- }*/
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-10869. CS considers only the default maximum-allocation-mb/vcore property as a maximum when it creates dynamic queues (#3504)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 7000458 YARN-10869. CS considers only the default
maximum-allocation-mb/vcore property as a maximum when it creates dynamic
queues (#3504)
7000458 is described below
commit 700045896c6175913d9071c132b533871dcbc20e
Author: Benjamin Teke
AuthorDate: Tue Oct 12 18:05:50 2021 +0200
YARN-10869. CS considers only the default maximum-allocation-mb/vcore
property as a maximum when it creates dynamic queues (#3504)
Co-authored-by: Benjamin Teke
---
.../scheduler/capacity/AbstractCSQueue.java| 50 --
.../capacity/CapacitySchedulerConfiguration.java | 14 ++
.../TestCapacitySchedulerAutoCreatedQueueBase.java | 32 --
.../TestCapacitySchedulerAutoQueueCreation.java| 33 +-
4 files changed, 102 insertions(+), 27 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index fda5e16..e7d9fa5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -100,7 +100,7 @@ public abstract class AbstractCSQueue implements CSQueue {
String defaultLabelExpression;
private String multiNodeSortingPolicyName = null;
- Map acls =
+ Map acls =
new HashMap();
volatile boolean reservationsContinueLooking;
private volatile boolean preemptionDisabled;
@@ -112,7 +112,7 @@ public abstract class AbstractCSQueue implements CSQueue {
volatile ResourceUsage queueUsage;
private final boolean fullPathQueueNamingPolicy = false;
-
+
// Track capacities like
used-capcity/abs-used-capacity/capacity/abs-capacity,
// etc.
QueueCapacities queueCapacities;
@@ -134,7 +134,7 @@ public abstract class AbstractCSQueue implements CSQueue {
protected CapacityConfigType capacityConfigType =
CapacityConfigType.NONE;
- private final RecordFactory recordFactory =
+ private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
protected CapacitySchedulerContext csContext;
protected YarnAuthorizationProvider authorizer = null;
@@ -250,12 +250,12 @@ public abstract class AbstractCSQueue implements CSQueue {
public QueueState getState() {
return state;
}
-
+
@Override
public CSQueueMetrics getMetrics() {
return metrics;
}
-
+
@Override
public String getQueueShortName() {
return queueName;
@@ -283,7 +283,7 @@ public abstract class AbstractCSQueue implements CSQueue {
public void setParent(CSQueue newParentQueue) {
this.parent = newParentQueue;
}
-
+
public Set getAccessibleNodeLabels() {
return accessibleLabels;
}
@@ -344,7 +344,7 @@ public abstract class AbstractCSQueue implements CSQueue {
public String getDefaultNodeLabelExpression() {
return defaultLabelExpression;
}
-
+
void setupQueueConfigs(Resource clusterResource)
throws IOException {
setupQueueConfigs(clusterResource, csContext.getConfiguration());
@@ -471,8 +471,14 @@ public abstract class AbstractCSQueue implements CSQueue {
private void setupMaximumAllocation(CapacitySchedulerConfiguration csConf) {
String myQueuePath = getQueuePath();
+/* YARN-10869: When using AutoCreatedLeafQueues, the passed configuration
+* object is a cloned one containing only the template configs
+* (see ManagedParentQueue#getLeafQueueConfigs). To ensure that the actual
+* cluster maximum allocation is fetched the original config object should
+* be used.
+*/
Resource clusterMax = ResourceUtils
-.fetchMaximumAllocationFromConfig(csConf);
+.fetchMaximumAllocationFromConfig(this.csContext.getConfiguration());
Resource queueMax = csConf.getQueueMaximumAllocation(myQueuePath);
maximumAllocation = Resources.clone(
@@ -740,7 +746,7 @@ public abstract class AbstractCSQueue implements CSQueue {
stats.setReservedContainers(getMetrics().getReservedContainers());
return stats;
}
-
+
public Map getQueueConfigurations() {
Map queueConfigurations = new HashMap<>();
Set nodeLabels = getNodeLabelsForQueue();
@@ -776,12 +782,12 @@ public abstract
[hadoop] branch trunk updated: YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8c271ca YARN-6862. Nodemanager resource usage metrics sometimes are
negative. Contributed by Benjamin Teke
8c271ca is described below
commit 8c271caa81db683fe4b92853bd8ca5da060e6593
Author: Szilard Nemeth
AuthorDate: Tue Oct 12 17:32:40 2021 +0200
YARN-6862. Nodemanager resource usage metrics sometimes are negative.
Contributed by Benjamin Teke
---
.../monitor/ContainersMonitorImpl.java | 8
.../MockCPUResourceCalculatorProcessTree.java | 10 +
...> MockMemoryResourceCalculatorProcessTree.java} | 47 +++---
.../monitor/MockResourceCalculatorProcessTree.java | 6 +++
.../TestContainersMonitorResourceChange.java | 24 ---
5 files changed, 75 insertions(+), 20 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index a83ae3a..deccded 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -537,6 +537,14 @@ public class ContainersMonitorImpl extends AbstractService
implements
pTree.updateProcessTree();// update process-tree
long currentVmemUsage = pTree.getVirtualMemorySize();
long currentPmemUsage = pTree.getRssMemorySize();
+if (currentVmemUsage < 0 || currentPmemUsage < 0) {
+ // YARN-6862/YARN-5021 If the container just exited or for
+ // another reason the physical/virtual memory is UNAVAILABLE (-1)
+ // the values shouldn't be aggregated.
+ LOG.info("Skipping monitoring container {} because "
+ + "memory usage is not available.", containerId);
+ continue;
+}
// if machine has 6 cores and 3 are used,
// cpuUsagePercentPerCore should be 300%
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
index eb35c91..49161f3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
@@ -57,6 +57,16 @@ public class MockCPUResourceCalculatorProcessTree
}
@Override
+ public long getVirtualMemorySize(int olderThanAge) {
+return 0;
+ }
+
+ @Override
+ public long getRssMemorySize(int olderThanAge) {
+return 0;
+ }
+
+ @Override
public float getCpuUsagePercent() {
long cpu = this.cpuPercentage;
// First getter call will be returned with -1, and other calls will
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java
similarity index 50%
copy from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java
copy to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java
index eb35c91..ea45ac4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodeman
[hadoop] branch trunk updated: YARN-10950. Code cleanup in QueueCapacities (#3454)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a9b2469 YARN-10950. Code cleanup in QueueCapacities (#3454)
a9b2469 is described below
commit a9b2469a534c5bc554c09aaf2d460a5a00922aca
Author: Adam Antal
AuthorDate: Sun Sep 19 14:42:02 2021 +0200
YARN-10950. Code cleanup in QueueCapacities (#3454)
---
.../scheduler/capacity/QueueCapacities.java| 51 --
1 file changed, 18 insertions(+), 33 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
index 86d35d6..2d82966 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
@@ -33,17 +33,16 @@ import
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana
public class QueueCapacities {
private static final String NL = CommonNodeLabelsManager.NO_LABEL;
private static final float LABEL_DOESNT_EXIST_CAP = 0f;
- private Map capacitiesMap;
- private ReadLock readLock;
- private WriteLock writeLock;
+ private final Map capacitiesMap;
+ private final ReadLock readLock;
+ private final WriteLock writeLock;
private final boolean isRoot;
public QueueCapacities(boolean isRoot) {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
readLock = lock.readLock();
writeLock = lock.writeLock();
-
-capacitiesMap = new HashMap();
+capacitiesMap = new HashMap<>();
this.isRoot = isRoot;
}
@@ -52,15 +51,15 @@ public class QueueCapacities {
USED_CAP(0), ABS_USED_CAP(1), MAX_CAP(2), ABS_MAX_CAP(3), CAP(4),
ABS_CAP(5),
MAX_AM_PERC(6), RESERVED_CAP(7), ABS_RESERVED_CAP(8), WEIGHT(9),
NORMALIZED_WEIGHT(10);
-private int idx;
+private final int idx;
-private CapacityType(int idx) {
+CapacityType(int idx) {
this.idx = idx;
}
}
private static class Capacities {
-private float[] capacitiesArr;
+private final float[] capacitiesArr;
public Capacities() {
capacitiesArr = new float[CapacityType.values().length];
@@ -71,19 +70,17 @@ public class QueueCapacities {
@Override
public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("{used=" + capacitiesArr[0] + "%, ")
- .append("abs_used=" + capacitiesArr[1] + "%, ")
- .append("max_cap=" + capacitiesArr[2] + "%, ")
- .append("abs_max_cap=" + capacitiesArr[3] + "%, ")
- .append("cap=" + capacitiesArr[4] + "%, ")
- .append("abs_cap=" + capacitiesArr[5] + "%, ")
- .append("max_am_perc=" + capacitiesArr[6] + "%, ")
- .append("reserved_cap=" + capacitiesArr[7] + "%, ")
- .append("abs_reserved_cap=" + capacitiesArr[8] + "%, ")
- .append("weight=" + capacitiesArr[9] + "w, ")
- .append("normalized_weight=" + capacitiesArr[10] + "w}");
- return sb.toString();
+ return "{used=" + capacitiesArr[0] + "%, " +
+ "abs_used=" + capacitiesArr[1] + "%, " +
+ "max_cap=" + capacitiesArr[2] + "%, " +
+ "abs_max_cap=" + capacitiesArr[3] + "%, " +
+ "cap=" + capacitiesArr[4] + "%, " +
+ "abs_cap=" + capacitiesArr[5] + "%, " +
+ "max_am_perc=" + capacitiesArr[6] + "%, " +
+ "reserved_cap=" + capacitiesArr[7] + "%, " +
+ "abs_reserved_cap=" + capacitiesArr[8] + "%, " +
+ "weight=" + capacitiesArr[9] + "w, " +
+ "normalized_weight=" + capacitiesArr[10] + "w}";
}
}
@@ -118,7 +115,6 @@ public class QueueCapacities {
}
}
- /* Used Capacity Getter and Setter */
public float getUsedCapacity() {
return _get(NL, CapacityType.USED_CAP);
}
@@ -135,7 +131,6 @@ public class QueueCapacities {
_set(la
[hadoop] branch trunk updated: YARN-10937. Fix log message arguments in LogAggregationFileController (#3450)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5dcb23c YARN-10937. Fix log message arguments in
LogAggregationFileController (#3450)
5dcb23c is described below
commit 5dcb23c1f005dfa8203c156abfc3d538a67028dc
Author: TiborKovacsCloudera
<90398134+tiborkovacscloud...@users.noreply.github.com>
AuthorDate: Sun Sep 19 14:35:34 2021 +0200
YARN-10937. Fix log message arguments in LogAggregationFileController
(#3450)
---
.../logaggregation/filecontroller/LogAggregationFileController.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index bd159ee..3c3380c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -390,7 +390,7 @@ public abstract class LogAggregationFileController {
remoteFS.setPermission(qualified, new
FsPermission(TLDIR_PERMISSIONS));
} catch ( UnsupportedOperationException use) {
LOG.info("Unable to set permissions for configured filesystem since"
- + " it does not support this", remoteFS.getScheme());
+ + " it does not support this {}", remoteFS.getScheme());
fsSupportsChmod = false;
}
@@ -437,7 +437,7 @@ public abstract class LogAggregationFileController {
remoteFS.setPermission(permissionCheckFile, new
FsPermission(TLDIR_PERMISSIONS));
} catch (UnsupportedOperationException use) {
LOG.info("Unable to set permissions for configured filesystem since"
-+ " it does not support this", remoteFS.getScheme());
++ " it does not support this {}", remoteFS.getScheme());
fsSupportsChmod = false;
} catch (IOException e) {
LOG.warn("Failed to check if FileSystem supports permissions on "
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10913. AbstractCSQueue: Group preemption methods and fields into a separate class (#3420)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new aa74a30 YARN-10913. AbstractCSQueue: Group preemption methods and
fields into a separate class (#3420)
aa74a30 is described below
commit aa74a303ed30a057893c6d2d9fb1e07e7d1f4a7d
Author: Szilard Nemeth <954799+szilard-nem...@users.noreply.github.com>
AuthorDate: Sun Sep 19 13:11:56 2021 +0200
YARN-10913. AbstractCSQueue: Group preemption methods and fields into a
separate class (#3420)
---
.../scheduler/capacity/AbstractCSQueue.java| 100 ++---
.../scheduler/capacity/CSQueuePreemption.java | 119 +
2 files changed, 129 insertions(+), 90 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 75aebfe..f0d0639 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueStatistics;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -106,10 +105,6 @@ public abstract class AbstractCSQueue implements CSQueue {
Map acls =
new HashMap();
volatile boolean reservationsContinueLooking;
- private volatile boolean preemptionDisabled;
- // Indicates if the in-queue preemption setting is ever disabled within the
- // hierarchy of this queue.
- private boolean intraQueuePreemptionDisabledInHierarchy;
// Track resource usage-by-label like used-resource/pending-resource, etc.
volatile ResourceUsage queueUsage;
@@ -129,6 +124,7 @@ public abstract class AbstractCSQueue implements CSQueue {
// Indicates if this queue's default lifetime was set by a config property,
// either at this level or anywhere in the queue's hierarchy.
private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false;
+ private CSQueuePreemption preemptionSettings;
public enum CapacityConfigType {
// FIXME, from what I can see, Percentage mode can almost apply to weighted
@@ -401,10 +397,7 @@ public abstract class AbstractCSQueue implements CSQueue {
this, labelManager, null);
// Store preemption settings
- this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(this,
- configuration);
- this.intraQueuePreemptionDisabledInHierarchy =
- isIntraQueueHierarchyPreemptionDisabled(this, configuration);
+ this.preemptionSettings = new CSQueuePreemption(this, csContext,
configuration);
this.priority = configuration.getQueuePriority(
getQueuePath());
@@ -777,7 +770,7 @@ public abstract class AbstractCSQueue implements CSQueue {
queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression);
queueInfo.setCurrentCapacity(getUsedCapacity());
queueInfo.setQueueStatistics(getQueueStatistics());
-queueInfo.setPreemptionDisabled(preemptionDisabled);
+queueInfo.setPreemptionDisabled(getPreemptionDisabled());
queueInfo.setIntraQueuePreemptionDisabled(
getIntraQueuePreemptionDisabled());
queueInfo.setQueueConfigurations(getQueueConfigurations());
@@ -902,17 +895,18 @@ public abstract class AbstractCSQueue implements CSQueue {
@Private
public boolean getPreemptionDisabled() {
-return preemptionDisabled;
+return preemptionSettings.isPreemptionDisabled();
}
@Private
public boolean getIntraQueuePreemptionDisabled() {
-return intraQueuePreemptionDisabledInHierarchy || preemptionDisabled;
+return preemptionSettings.isIntraQueuePreemptionDisabledInHierarchy() ||
+preemptionSettings.isPreemptionDisabled();
}
@Private
public boolean getIntraQueuePreemptionDisabledInHierarchy() {
-return intraQueuePreemptionDisabledInHierarchy;
+return preemptionSettings.isIntraQueuePreemptionDisabledInHierarchy();
}
@Private
@@ -935,43 +929,6 @@ public abstract
[hadoop] branch branch-3.2 updated: YARN-10870. Missing user filtering check -> yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by Gergely Pollak
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 4066cb0 YARN-10870. Missing user filtering check ->
yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by
Gergely Pollak
4066cb0 is described below
commit 4066cb0793b2aa49a585384dc798934ef50f8ba5
Author: Szilard Nemeth
AuthorDate: Tue Sep 14 18:09:35 2021 +0200
YARN-10870. Missing user filtering check ->
yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by
Gergely Pollak
---
.../webapp/FairSchedulerAppsBlock.java | 70 +++---
1 file changed, 62 insertions(+), 8 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index 14ad277..0ba8f1a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -23,18 +23,21 @@ import static
org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Map;
+import java.security.Principal;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.text.StringEscapeUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -49,6 +52,8 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
+import javax.servlet.http.HttpServletRequest;
+
/**
* Shows application information specific to the fair
* scheduler as part of the fair scheduler page.
@@ -58,10 +63,19 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
final FairSchedulerInfo fsinfo;
final Configuration conf;
final ResourceManager rm;
+ final boolean filterAppsByUser;
+
@Inject
public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx,
Configuration conf) {
super(ctx);
+this.conf = conf;
+this.rm = rm;
+
+this.filterAppsByUser = conf.getBoolean(
+YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
+YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
+
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
fsinfo = new FairSchedulerInfo(scheduler);
apps = new ConcurrentHashMap();
@@ -70,13 +84,53 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
if (!(RMAppState.NEW.equals(entry.getValue().getState())
|| RMAppState.NEW_SAVING.equals(entry.getValue().getState())
|| RMAppState.SUBMITTED.equals(entry.getValue().getState( {
-apps.put(entry.getKey(), entry.getValue());
+if (!filterAppsByUser || hasAccess(entry.getValue(),
+ctx.requestContext().getRequest())) {
+ apps.put(entry.getKey(), entry.getValue());
+}
}
}
-this.conf = conf;
-this.rm = rm;
}
-
+
+ private UserGroupInformation getCallerUserGroupInformation(
+ HttpServletRequest hsr, boolean usePrincipal) {
+
+String remoteUser = hsr.getRemoteUser();
+if (usePrincipal) {
+ Principal princ = hsr.getUserPrincipal();
+ remoteUser = princ == null ? null : princ.getName();
+}
+
+UserGroupInformation callerUGI = null;
+if (remoteUser != null) {
+ callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+}
+
+return callerUGI;
+ }
+
+ protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) {
+//
[hadoop] branch branch-3.3 updated: YARN-10870. Missing user filtering check -> yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by Gergely Pollak
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 6c68211 YARN-10870. Missing user filtering check ->
yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by
Gergely Pollak
6c68211 is described below
commit 6c6821106221e95e08fbacefb4c4ef74dc56b8e7
Author: Szilard Nemeth
AuthorDate: Tue Sep 14 18:08:34 2021 +0200
YARN-10870. Missing user filtering check ->
yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by
Gergely Pollak
---
.../webapp/FairSchedulerAppsBlock.java | 70 +++---
1 file changed, 62 insertions(+), 8 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index 14ad277..0ba8f1a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -23,18 +23,21 @@ import static
org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Map;
+import java.security.Principal;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.text.StringEscapeUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -49,6 +52,8 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
+import javax.servlet.http.HttpServletRequest;
+
/**
* Shows application information specific to the fair
* scheduler as part of the fair scheduler page.
@@ -58,10 +63,19 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
final FairSchedulerInfo fsinfo;
final Configuration conf;
final ResourceManager rm;
+ final boolean filterAppsByUser;
+
@Inject
public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx,
Configuration conf) {
super(ctx);
+this.conf = conf;
+this.rm = rm;
+
+this.filterAppsByUser = conf.getBoolean(
+YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
+YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
+
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
fsinfo = new FairSchedulerInfo(scheduler);
apps = new ConcurrentHashMap();
@@ -70,13 +84,53 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
if (!(RMAppState.NEW.equals(entry.getValue().getState())
|| RMAppState.NEW_SAVING.equals(entry.getValue().getState())
|| RMAppState.SUBMITTED.equals(entry.getValue().getState( {
-apps.put(entry.getKey(), entry.getValue());
+if (!filterAppsByUser || hasAccess(entry.getValue(),
+ctx.requestContext().getRequest())) {
+ apps.put(entry.getKey(), entry.getValue());
+}
}
}
-this.conf = conf;
-this.rm = rm;
}
-
+
+ private UserGroupInformation getCallerUserGroupInformation(
+ HttpServletRequest hsr, boolean usePrincipal) {
+
+String remoteUser = hsr.getRemoteUser();
+if (usePrincipal) {
+ Principal princ = hsr.getUserPrincipal();
+ remoteUser = princ == null ? null : princ.getName();
+}
+
+UserGroupInformation callerUGI = null;
+if (remoteUser != null) {
+ callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+}
+
+return callerUGI;
+ }
+
+ protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) {
+//
[hadoop] branch trunk updated: YARN-10915. AbstractCSQueue: Simplify complex logic in methods: deriveCapacityFromAbsoluteConfigurations and updateEffectiveResources (#3418)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5dc2f7b YARN-10915. AbstractCSQueue: Simplify complex logic in
methods: deriveCapacityFromAbsoluteConfigurations and updateEffectiveResources
(#3418)
5dc2f7b is described below
commit 5dc2f7b1376cdc392a5d204fb0ff3b9dec0ba36c
Author: Benjamin Teke
AuthorDate: Tue Sep 14 18:05:40 2021 +0200
YARN-10915. AbstractCSQueue: Simplify complex logic in methods:
deriveCapacityFromAbsoluteConfigurations and updateEffectiveResources (#3418)
Co-authored-by: Benjamin Teke
---
.../scheduler/capacity/AbstractCSQueue.java| 163 +++--
.../scheduler/capacity/ParentQueue.java| 6 +-
2 files changed, 91 insertions(+), 78 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index f97460c..75aebfe 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -1516,8 +1516,8 @@ public abstract class AbstractCSQueue implements CSQueue {
parentQueueCapacities, queueCapacities.getExistingNodeLabels());
}
- private Resource getMinResourceNormalized(String name,
- Map effectiveMinRatio, Resource minResource) {
+ private Resource createNormalizedMinResource(Resource minResource,
+ Map effectiveMinRatio) {
Resource ret = Resource.newInstance(minResource);
int maxLength = ResourceUtils.getNumberOfCountableResourceTypes();
for (int i = 0; i < maxLength; i++) {
@@ -1527,18 +1527,35 @@ public abstract class AbstractCSQueue implements
CSQueue {
Float ratio = effectiveMinRatio.get(nResourceInformation.getName());
if (ratio != null) {
ret.setResourceValue(i,
-(long) (nResourceInformation.getValue() * ratio.floatValue()));
+(long) (nResourceInformation.getValue() * ratio));
if (LOG.isDebugEnabled()) {
- LOG.debug("Updating min resource for Queue: " + name + " as " + ret
+ LOG.debug("Updating min resource for Queue: " + queuePath + " as " +
ret
.getResourceInformation(i) + ", Actual resource: "
- + nResourceInformation.getValue() + ", ratio: " + ratio
- .floatValue());
+ + nResourceInformation.getValue() + ", ratio: " + ratio);
}
}
}
return ret;
}
+ private Resource getOrInheritMaxResource(Resource resourceByLabel, String
label) {
+Resource parentMaxResource =
+parent.getQueueResourceQuotas().getConfiguredMaxResource(label);
+if (parentMaxResource.equals(Resources.none())) {
+ parentMaxResource =
+ parent.getQueueResourceQuotas().getEffectiveMaxResource(label);
+}
+
+Resource configuredMaxResource =
+getQueueResourceQuotas().getConfiguredMaxResource(label);
+if (configuredMaxResource.equals(Resources.none())) {
+ return Resources.clone(parentMaxResource);
+}
+
+return Resources.clone(Resources.min(resourceCalculator, resourceByLabel,
+configuredMaxResource, parentMaxResource));
+ }
+
void updateMaxAppRelatedField(CapacitySchedulerConfiguration conf,
LeafQueue leafQueue) {
int maxApplications = conf.getMaximumApplicationsPerQueue(queuePath);
@@ -1586,99 +1603,97 @@ public abstract class AbstractCSQueue implements
CSQueue {
.getMaximumCapacity(maxLabel));
}
- private void deriveCapacityFromAbsoluteConfigurations(String label,
- Resource clusterResource, ResourceCalculator rc) {
-
-/*
- * In case when queues are configured with absolute resources, it is better
- * to update capacity/max-capacity etc w.r.t absolute resource as well. In
- * case of computation, these values wont be used any more. However for
- * metrics and UI, its better these values are pre-computed here itself.
- */
-
-// 1. Update capacity as a float based on parent's minResource
-float f = rc.divide(clusterResource,
+ void deriveCapacityFromAbsoluteConfigurations(String label,
+ Resource clusterResource) {
+// Update capacity with a float calculated from the parent's minResour
[hadoop] branch trunk updated: YARN-10917. Investigate and simplify CapacitySchedulerConfigValidator#validateQueueHierarchy (#3403)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 783d94f YARN-10917. Investigate and simplify
CapacitySchedulerConfigValidator#validateQueueHierarchy (#3403)
783d94f is described below
commit 783d94f5cdf2f3b03aee5ae5a1bcd4cc14dcb292
Author: Tamas Domok
AuthorDate: Tue Sep 14 17:54:25 2021 +0200
YARN-10917. Investigate and simplify
CapacitySchedulerConfigValidator#validateQueueHierarchy (#3403)
* YARN-10917. Investigate and simplify
CapacitySchedulerConfigValidator#validateQueueHierarchy.
Co-authored-by: Tamas Domok
---
.../capacity/CapacitySchedulerConfigValidator.java | 152 -
1 file changed, 88 insertions(+), 64 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
index ef9f97a..ca0d586 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
@@ -106,12 +106,8 @@ public final class CapacitySchedulerConfigValidator {
}
}
- private static boolean isDynamicQueue(CSQueue csQueue) {
-return ((AbstractCSQueue)csQueue).isDynamicQueue();
- }
-
/**
- * Ensure all existing queues are present. Queues cannot be deleted if its
not
+ * Ensure all existing queues are present. Queues cannot be deleted if it's
not
* in Stopped state, Queue's cannot be moved from one hierarchy to other
also.
* Previous child queue could be converted into parent queue if it is in
* STOPPED state.
@@ -125,78 +121,106 @@ public final class CapacitySchedulerConfigValidator {
CapacitySchedulerConfiguration newConf) throws IOException {
// check that all static queues are included in the newQueues list
for (CSQueue oldQueue : queues.getQueues()) {
- if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(
- oldQueue.getClass( {
-String queuePath = oldQueue.getQueuePath();
-CSQueue newQueue = newQueues.get(queuePath);
-String configPrefix = newConf.getQueuePrefix(
-oldQueue.getQueuePath());
-String state = newConf.get(configPrefix + "state");
-QueueState newQueueState = null;
-if (state != null) {
- try {
-newQueueState = QueueState.valueOf(state);
- } catch (Exception ex) {
-LOG.warn("Not a valid queue state for queue "
-+ oldQueue.getQueuePath());
+ if
(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(oldQueue.getClass())) {
+continue;
+ }
+
+ final String queuePath = oldQueue.getQueuePath();
+ final String configPrefix =
CapacitySchedulerConfiguration.getQueuePrefix(
+ oldQueue.getQueuePath());
+ final QueueState newQueueState =
createQueueState(newConf.get(configPrefix + "state"),
+ queuePath);
+ final CSQueue newQueue = newQueues.get(queuePath);
+
+ if (null == newQueue) {
+// old queue doesn't exist in the new XML
+if (isEitherQueueStopped(oldQueue.getState(), newQueueState)) {
+ LOG.info("Deleting Queue {}, as it is not present in the modified
capacity " +
+ "configuration xml", queuePath);
+} else {
+ if (!isDynamicQueue(oldQueue)) {
+throw new IOException(oldQueue.getQueuePath() + " cannot be"
++ " deleted from the capacity scheduler configuration, as the"
++ " queue is not yet in stopped state. Current State : "
++ oldQueue.getState());
}
}
-if (null == newQueue) {
- // old queue doesn't exist in the new XML
- if (oldQueue.getState() == QueueState.STOPPED ||
- newQueueState == QueueState.STOPPED) {
-LOG.info("Deleting Queue " + queuePath + ", as it is not"
-+ " present in the modified capacity configuration xml");
- } else {
-if (!isDynamicQueue(oldQueue)) {
- throw new IOException(oldQueue.getQueuePath() + " cannot be"
-
[hadoop] branch branch-3.2 updated: YARN-10901. Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir (#3410)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 478d2f2 YARN-10901. Permission checking error on an existing
directory in LogAggregationFileController#verifyAndCreateRemoteLogDir (#3410)
478d2f2 is described below
commit 478d2f2897abba5bad2f5204b0d4ec02cbb7ba30
Author: Tamas Domok
AuthorDate: Tue Sep 14 17:35:07 2021 +0200
YARN-10901. Permission checking error on an existing directory in
LogAggregationFileController#verifyAndCreateRemoteLogDir (#3410)
Co-authored-by: Tamas Domok
---
.../LogAggregationFileController.java | 13 +++-
.../TestLogAggregationFileController.java | 75 ++
2 files changed, 86 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index 35f2d45..697ecbc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.logaggregation.filecontroller;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.RandomStringUtils;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -368,10 +369,13 @@ public abstract class LogAggregationFileController {
throw new YarnRuntimeException("Failed to create remoteLogDir ["
+ remoteRootLogDir + "]", e);
}
-} else{
+} else {
//Check if FS has capability to set/modify permissions
+ Path permissionCheckFile = new Path(qualified,
String.format("%s.permission_check",
+ RandomStringUtils.randomAlphanumeric(8)));
try {
-remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS));
+remoteFS.createNewFile(permissionCheckFile);
+remoteFS.setPermission(permissionCheckFile, new
FsPermission(TLDIR_PERMISSIONS));
} catch (UnsupportedOperationException use) {
LOG.info("Unable to set permissions for configured filesystem since"
+ " it does not support this", remoteFS.getScheme());
@@ -379,6 +383,11 @@ public abstract class LogAggregationFileController {
} catch (IOException e) {
LOG.warn("Failed to check if FileSystem suppports permissions on "
+ "remoteLogDir [" + remoteRootLogDir + "]", e);
+ } finally {
+try {
+ remoteFS.delete(permissionCheckFile, false);
+} catch (IOException ignored) {
+}
}
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java
new file mode 100644
index 000..3566884
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.logaggregation.filecontroller;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.A
[hadoop] branch branch-3.3 updated: YARN-10901. Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir (#3409)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 8e4ac01 YARN-10901. Permission checking error on an existing
directory in LogAggregationFileController#verifyAndCreateRemoteLogDir (#3409)
8e4ac01 is described below
commit 8e4ac01135e386fb5ed09846c0f7bb9deffdadde
Author: Tamas Domok
AuthorDate: Tue Sep 14 17:34:32 2021 +0200
YARN-10901. Permission checking error on an existing directory in
LogAggregationFileController#verifyAndCreateRemoteLogDir (#3409)
Co-authored-by: Tamas Domok
---
.../LogAggregationFileController.java | 13 +-
.../TestLogAggregationFileController.java | 53 ++
2 files changed, 64 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index cf305ba..4844ae2 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.logaggregation.filecontroller;
+import org.apache.commons.lang3.RandomStringUtils;
import
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import java.io.FileNotFoundException;
@@ -373,10 +374,13 @@ public abstract class LogAggregationFileController {
throw new YarnRuntimeException("Failed to create remoteLogDir ["
+ remoteRootLogDir + "]", e);
}
-} else{
+} else {
//Check if FS has capability to set/modify permissions
+ Path permissionCheckFile = new Path(qualified,
String.format("%s.permission_check",
+ RandomStringUtils.randomAlphanumeric(8)));
try {
-remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS));
+remoteFS.createNewFile(permissionCheckFile);
+remoteFS.setPermission(permissionCheckFile, new
FsPermission(TLDIR_PERMISSIONS));
} catch (UnsupportedOperationException use) {
LOG.info("Unable to set permissions for configured filesystem since"
+ " it does not support this", remoteFS.getScheme());
@@ -384,6 +388,11 @@ public abstract class LogAggregationFileController {
} catch (IOException e) {
LOG.warn("Failed to check if FileSystem suppports permissions on "
+ "remoteLogDir [" + remoteRootLogDir + "]", e);
+ } finally {
+try {
+ remoteFS.delete(permissionCheckFile, false);
+} catch (IOException ignored) {
+}
}
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java
index 5ade0fa..818e011 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java
@@ -19,17 +19,23 @@
package org.apache.hadoop.yarn.logaggregation.filecontroller;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
import org.junit.Test;
+import org.mockito.ArgumentMatcher;
import org.mockito.Mockito;
import java.io.FileNotFoundException;
import java.net.URI;
+import static
org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController.TLDIR_PERMISSIONS;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
@@ -88,4 +94,51 @@ public class TestLogAggregationFileController {
verify(fs).setOwner(any(), eq("yarn_user"), eq(tes
[hadoop] branch trunk updated (4f563ff -> 63c8922)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 4f563ff HDFS-16225. Fix typo for FederationTestUtils (#3428). Contributed by tomscut. add 63c8922 YARN-10912. AbstractCSQueue#updateConfigurableResourceRequirement: Separate validation logic from initialization logic (#3390) No new revisions were added by this update. Summary of changes: .../scheduler/capacity/AbstractCSQueue.java| 70 -- 1 file changed, 37 insertions(+), 33 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10872. Replace getPropsWithPrefix calls in AutoCreatedQueueTemplate (#3396)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 971f1b8 YARN-10872. Replace getPropsWithPrefix calls in
AutoCreatedQueueTemplate (#3396)
971f1b8 is described below
commit 971f1b8b0a37b4d3eafdecf705afa4f684253e26
Author: Benjamin Teke
AuthorDate: Fri Sep 10 17:32:42 2021 +0200
YARN-10872. Replace getPropsWithPrefix calls in AutoCreatedQueueTemplate
(#3396)
Co-authored-by: Benjamin Teke
---
.../capacity/AbstractManagedParentQueue.java | 37 --
.../capacity/AutoCreatedQueueTemplate.java | 24 --
.../capacity/CapacitySchedulerConfiguration.java | 5 +--
.../capacity/ConfigurationProperties.java | 12 ++-
.../capacity/TestConfigurationProperties.java | 11 +++
5 files changed, 46 insertions(+), 43 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
index a9e82a6..9c16de0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
@@ -28,11 +28,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.Comparator;
import java.util.Iterator;
import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
/**
* A container class for automatically created child leaf queues.
@@ -173,44 +170,22 @@ public abstract class AbstractManagedParentQueue extends
ParentQueue {
return queueManagementPolicy;
}
- protected SortedMap getConfigurationsWithPrefix
- (SortedMap sortedConfigs, String prefix) {
-return sortedConfigs.subMap( prefix, prefix + Character.MAX_VALUE );
- }
-
- protected SortedMap sortCSConfigurations() {
-SortedMap sortedConfigs = new TreeMap(
-new Comparator() {
- public int compare(String s1, String s2) {
-return s1.compareToIgnoreCase(s2);
- }
-
-});
-
-for (final Iterator> iterator =
- csContext.getConfiguration().iterator(); iterator.hasNext(); ) {
- final Map.Entry confKeyValuePair = iterator.next();
- sortedConfigs.put(confKeyValuePair.getKey(),
confKeyValuePair.getValue());
-}
-return sortedConfigs;
- }
-
protected CapacitySchedulerConfiguration initializeLeafQueueConfigs(String
configPrefix) {
CapacitySchedulerConfiguration leafQueueConfigs = new
CapacitySchedulerConfiguration(new Configuration(false), false);
-String prefix = YarnConfiguration.RESOURCE_TYPES + ".";
Map rtProps = csContext
-.getConfiguration().getPropsWithPrefix(prefix);
+.getConfiguration().getConfigurationProperties()
+.getPropertiesWithPrefix(YarnConfiguration.RESOURCE_TYPES + ".", true);
for (Map.Entry entry : rtProps.entrySet()) {
- leafQueueConfigs.set(prefix + entry.getKey(), entry.getValue());
+ leafQueueConfigs.set(entry.getKey(), entry.getValue());
}
-SortedMap sortedConfigs = sortCSConfigurations();
-SortedMap templateConfigs = getConfigurationsWithPrefix
-(sortedConfigs, configPrefix);
+Map templateConfigs = csContext
+.getConfiguration().getConfigurationProperties()
+.getPropertiesWithPrefix(configPrefix, true);
for (final Iterator> iterator =
templateConfigs.entrySet().iterator(); iterator.hasNext(); ) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
index cb0f789..eff8c4e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resource
[hadoop] branch trunk updated (b229e5a -> 811fd23)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from b229e5a YARN-10910. AbstractCSQueue#setupQueueConfigs: Separate validation logic from initialization logic (#3407) add 811fd23 YARN-10852. Optimise CSConfiguration getAllUserWeightsForQueue (#3392) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/conf/Configuration.java | 2 +- .../capacity/CapacitySchedulerConfiguration.java | 34 +++--- .../scheduler/capacity/TestLeafQueue.java | 1 + 3 files changed, 25 insertions(+), 12 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (29a6f14 -> b229e5a)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 29a6f14 YARN-10914. Simplify duplicated code for tracking ResourceUsage in AbstractCSQueue (#3402) add b229e5a YARN-10910. AbstractCSQueue#setupQueueConfigs: Separate validation logic from initialization logic (#3407) No new revisions were added by this update. Summary of changes: .../scheduler/capacity/AbstractCSQueue.java| 215 +++-- 1 file changed, 113 insertions(+), 102 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10914. Simplify duplicated code for tracking ResourceUsage in AbstractCSQueue (#3402)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 29a6f14 YARN-10914. Simplify duplicated code for tracking
ResourceUsage in AbstractCSQueue (#3402)
29a6f14 is described below
commit 29a6f141d4bcc5307d2ac72c742f87611d7092d5
Author: Tamas Domok
AuthorDate: Fri Sep 10 15:57:46 2021 +0200
YARN-10914. Simplify duplicated code for tracking ResourceUsage in
AbstractCSQueue (#3402)
Co-authored-by: Tamas Domok
---
.../scheduler/capacity/AbstractCSQueue.java| 97 ++
1 file changed, 43 insertions(+), 54 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 1f5820d..2da52d5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -73,6 +73,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -1189,84 +1190,72 @@ public abstract class AbstractCSQueue implements
CSQueue {
}
+ private static String ensurePartition(String partition) {
+return Optional.ofNullable(partition).orElse(RMNodeLabelsManager.NO_LABEL);
+ }
+
+ @FunctionalInterface
+ interface Counter {
+void count(String partition, Resource resource);
+ }
+
+ @FunctionalInterface
+ interface CounterWithApp {
+void count(String partition, Resource reservedRes,
SchedulerApplicationAttempt application);
+ }
+
+ private void count(String partition, Resource resource, Counter counter,
Counter parentCounter) {
+final String checkedPartition = ensurePartition(partition);
+counter.count(checkedPartition, resource);
+Optional.ofNullable(parentCounter).ifPresent(c ->
c.count(checkedPartition, resource));
+ }
+
+ private void countAndUpdate(String partition, Resource resource,
+ Counter counter, CounterWithApp parentCounter) {
+final String checkedPartition = ensurePartition(partition);
+counter.count(checkedPartition, resource);
+CSQueueUtils.updateUsedCapacity(resourceCalculator,
+labelManager.getResourceByLabel(checkedPartition, Resources.none()),
+checkedPartition, this);
+Optional.ofNullable(parentCounter).ifPresent(c ->
c.count(checkedPartition, resource, null));
+ }
+
@Override
public void incReservedResource(String partition, Resource reservedRes) {
-if (partition == null) {
- partition = RMNodeLabelsManager.NO_LABEL;
-}
-
-queueUsage.incReserved(partition, reservedRes);
-if(null != parent){
- parent.incReservedResource(partition, reservedRes);
-}
+count(partition, reservedRes, queueUsage::incReserved,
+parent == null ? null : parent::incReservedResource);
}
@Override
public void decReservedResource(String partition, Resource reservedRes) {
-if (partition == null) {
- partition = RMNodeLabelsManager.NO_LABEL;
-}
-
-queueUsage.decReserved(partition, reservedRes);
-if(null != parent){
- parent.decReservedResource(partition, reservedRes);
-}
+count(partition, reservedRes, queueUsage::decReserved,
+parent == null ? null : parent::decReservedResource);
}
@Override
public void incPendingResource(String nodeLabel, Resource resourceToInc) {
-if (nodeLabel == null) {
- nodeLabel = RMNodeLabelsManager.NO_LABEL;
-}
-// ResourceUsage has its own lock, no addition lock needs here.
-queueUsage.incPending(nodeLabel, resourceToInc);
-if (null != parent) {
- parent.incPendingResource(nodeLabel, resourceToInc);
-}
+count(nodeLabel, resourceToInc, queueUsage::incPending,
+parent == null ? null : parent::incPendingResource);
}
@Override
public void decPendingResource(String nodeLabel, Resource resourceToDec) {
-if (nodeLabel == null) {
- nodeLabel = RMNodeLabelsManager.NO_LABEL;
-}
-// ResourceUsage has its own lock, no addition lock needs here.
-queueUsage.decPending(nodeLabel, resourceToDec);
-if (null != parent) {
- parent.decPendingResource(nodeLabel, resourceToDec);
-}
+
[hadoop] branch trunk updated: YARN-10870. Missing user filtering check -> yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by Gergely Pollak
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2ff3fc5 YARN-10870. Missing user filtering check ->
yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by
Gergely Pollak
2ff3fc5 is described below
commit 2ff3fc50e4a9bc60a1ca968bd495a18728084eaa
Author: Szilard Nemeth
AuthorDate: Wed Sep 8 18:01:39 2021 +0200
YARN-10870. Missing user filtering check ->
yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by
Gergely Pollak
---
.../webapp/FairSchedulerAppsBlock.java | 69 +++---
1 file changed, 61 insertions(+), 8 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index 14ad277..f6202cb 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -23,18 +23,21 @@ import static
org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Map;
+import java.security.Principal;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.text.StringEscapeUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -49,6 +52,8 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
+import javax.servlet.http.HttpServletRequest;
+
/**
* Shows application information specific to the fair
* scheduler as part of the fair scheduler page.
@@ -58,10 +63,19 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
final FairSchedulerInfo fsinfo;
final Configuration conf;
final ResourceManager rm;
+ final boolean filterAppsByUser;
+
@Inject
public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx,
Configuration conf) {
super(ctx);
+this.conf = conf;
+this.rm = rm;
+
+this.filterAppsByUser = conf.getBoolean(
+YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
+YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
+
FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
fsinfo = new FairSchedulerInfo(scheduler);
apps = new ConcurrentHashMap();
@@ -70,13 +84,52 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
if (!(RMAppState.NEW.equals(entry.getValue().getState())
|| RMAppState.NEW_SAVING.equals(entry.getValue().getState())
|| RMAppState.SUBMITTED.equals(entry.getValue().getState( {
-apps.put(entry.getKey(), entry.getValue());
+if (!filterAppsByUser || hasAccess(entry.getValue(),
+ctx.requestContext().getRequest())) {
+ apps.put(entry.getKey(), entry.getValue());
+}
}
}
-this.conf = conf;
-this.rm = rm;
}
-
+
+ private UserGroupInformation getCallerUserGroupInformation(
+ HttpServletRequest hsr, boolean usePrincipal) {
+String remoteUser = hsr.getRemoteUser();
+if (usePrincipal) {
+ Principal princ = hsr.getUserPrincipal();
+ remoteUser = princ == null ? null : princ.getName();
+}
+
+UserGroupInformation callerUGI = null;
+if (remoteUser != null) {
+ callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+}
+
+return callerUGI;
+ }
+
+ protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) {
+//
[hadoop] branch trunk updated: HADOOP-17857. Check real user ACLs in addition to proxied user ACLs. Contributed by Eric Payne
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5428d36 HADOOP-17857. Check real user ACLs in addition to proxied
user ACLs. Contributed by Eric Payne
5428d36 is described below
commit 5428d36b56fab319ab68258139d6133ded9bbafc
Author: Szilard Nemeth
AuthorDate: Wed Sep 8 17:27:22 2021 +0200
HADOOP-17857. Check real user ACLs in addition to proxied user ACLs.
Contributed by Eric Payne
---
.../hadoop/security/authorize/AccessControlList.java | 12 +---
.../security/authorize/TestAccessControlList.java | 18 ++
2 files changed, 27 insertions(+), 3 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
index e86d918..aa5b01f 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
@@ -56,6 +56,7 @@ public class AccessControlList implements Writable {
// Indicates an ACL string that represents access to all users
public static final String WILDCARD_ACL_VALUE = "*";
private static final int INITIAL_CAPACITY = 256;
+ public static final String USE_REAL_ACLS = "~";
// Set of users who are granted access.
private Collection users;
@@ -224,9 +225,12 @@ public class AccessControlList implements Writable {
/**
* Checks if a user represented by the provided {@link UserGroupInformation}
- * is a member of the Access Control List
+ * is a member of the Access Control List. If user was proxied and
+ * USE_REAL_ACLS + the real user name is in the control list, then treat this
+ * case as if user were in the ACL list.
* @param ugi UserGroupInformation to check if contained in the ACL
- * @return true if ugi is member of the list
+ * @return true if ugi is member of the list or if USE_REAL_ACLS + real user
+ * is in the list
*/
public final boolean isUserInList(UserGroupInformation ugi) {
if (allAllowed || users.contains(ugi.getShortUserName())) {
@@ -239,7 +243,9 @@ public class AccessControlList implements Writable {
}
}
}
-return false;
+UserGroupInformation realUgi = ugi.getRealUser();
+return realUgi != null &&
+ users.contains(USE_REAL_ACLS + realUgi.getShortUserName());
}
public boolean isUserAllowed(UserGroupInformation ugi) {
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestAccessControlList.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestAccessControlList.java
index 8e1b82b..53ab275 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestAccessControlList.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestAccessControlList.java
@@ -471,4 +471,22 @@ public class TestAccessControlList {
+ " is incorrectly granted the access-control!!",
acl.isUserAllowed(ugi));
}
+
+ @Test
+ public void testUseRealUserAclsForProxiedUser() {
+String realUser = "realUser";
+AccessControlList acl = new AccessControlList(realUser);
+UserGroupInformation realUserUgi =
+UserGroupInformation.createRemoteUser(realUser);
+UserGroupInformation user1 =
+UserGroupInformation.createProxyUserForTesting("regularJane",
+realUserUgi, new String [] {"group1"});
+assertFalse("User " + user1 + " should not have been granted access.",
+acl.isUserAllowed(user1));
+
+acl = new AccessControlList(AccessControlList.USE_REAL_ACLS + realUser);
+
+assertTrue("User " + user1 + " should have access but was denied.",
+acl.isUserAllowed(user1));
+ }
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (4e209a3 -> 5e16689)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 4e209a3 YARN-10919. Remove LeafQueue#scheduler field (#3382) add 5e16689 YARN-10901. Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir (#3355) No new revisions were added by this update. Summary of changes: .../LogAggregationFileController.java | 13 +- .../TestLogAggregationFileController.java | 53 ++ 2 files changed, 64 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (40e639a -> 4e209a3)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 40e639a YARN-10646. TestCapacitySchedulerWeightMode test descriptor comments doesnt reflect the correct scenario (#3339) add 4e209a3 YARN-10919. Remove LeafQueue#scheduler field (#3382) No new revisions were added by this update. Summary of changes: .../resourcemanager/scheduler/capacity/LeafQueue.java | 13 + 1 file changed, 5 insertions(+), 8 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10646. TestCapacitySchedulerWeightMode test descriptor comments doesnt reflect the correct scenario (#3339)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 40e639a YARN-10646. TestCapacitySchedulerWeightMode test descriptor
comments doesnt reflect the correct scenario (#3339)
40e639a is described below
commit 40e639ad078a5f6f0503f206c4a0c4c012793349
Author: Benjamin Teke
AuthorDate: Wed Sep 8 16:11:04 2021 +0200
YARN-10646. TestCapacitySchedulerWeightMode test descriptor comments doesnt
reflect the correct scenario (#3339)
Co-authored-by: Benjamin Teke
---
.../capacity/TestCapacitySchedulerWeightMode.java | 24 +++---
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
index 171123a..a5d80dc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
@@ -198,9 +198,9 @@ public class TestCapacitySchedulerWeightMode {
* a x(=100%), y(50%) b y(=50%), z(=100%)
* __
* / / \
- * a1 ([x,y]: w=100)b1(no) b2([y,z]: w=100)
+ * a1 ([x,y]: w=1)b1(no) b2([y,z]: w=1)
*
- * Parent uses weight, child uses percentage
+ * Parent uses percentages, child uses weights
*/
public static Configuration getCSConfWithLabelsParentUsePctChildUseWeight(
Configuration config) {
@@ -210,9 +210,9 @@ public class TestCapacitySchedulerWeightMode {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT,
new String[] { "a", "b" });
-conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "x", 100);
-conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100);
-conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100);
+conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
+conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100);
conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 10);
conf.setMaximumCapacity(A, 10);
@@ -228,23 +228,23 @@ public class TestCapacitySchedulerWeightMode {
// Define 2nd-level queues
conf.setQueues(A, new String[] { "a1" });
-conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100);
+conf.setLabeledQueueWeight(A1, RMNodeLabelsManager.NO_LABEL, 1);
conf.setMaximumCapacity(A1, 100);
conf.setAccessibleNodeLabels(A1, toSet("x", "y"));
conf.setDefaultNodeLabelExpression(A1, "x");
-conf.setCapacityByLabel(A1, "x", 100);
-conf.setCapacityByLabel(A1, "y", 100);
+conf.setLabeledQueueWeight(A1, "x", 1);
+conf.setLabeledQueueWeight(A1, "y", 1);
conf.setQueues(B, new String[] { "b1", "b2" });
-conf.setCapacityByLabel(B1, RMNodeLabelsManager.NO_LABEL, 50);
+conf.setLabeledQueueWeight(B1, RMNodeLabelsManager.NO_LABEL, 1);
conf.setMaximumCapacity(B1, 50);
conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET);
-conf.setCapacityByLabel(B2, RMNodeLabelsManager.NO_LABEL, 50);
+conf.setLabeledQueueWeight(B2, RMNodeLabelsManager.NO_LABEL, 1);
conf.setMaximumCapacity(B2, 50);
conf.setAccessibleNodeLabels(B2, toSet("y", "z"));
-conf.setCapacityByLabel(B2, "y", 100);
-conf.setCapacityByLabel(B2, "z", 100);
+conf.setLabeledQueueWeight(B2, "y", 1);
+conf.setLabeledQueueWeight(B2, "z", 1);
return conf;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
