[hadoop] branch trunk updated: YARN.11590. RM process stuck after calling confStore.format() when ZK SSL/TLS is enabled, as netty thread waits indefinitely. Contributed by Ferenc Erdelyi
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 2736f8856144 YARN.11590. RM process stuck after calling confStore.format() when ZK SSL/TLS is enabled, as netty thread waits indefinitely. Contributed by Ferenc Erdelyi 2736f8856144 is described below commit 2736f8856144c8361be3821cd1c158ef2d9a2dec Author: Szilard Nemeth AuthorDate: Mon Oct 16 15:17:58 2023 -0400 YARN.11590. RM process stuck after calling confStore.format() when ZK SSL/TLS is enabled, as netty thread waits indefinitely. Contributed by Ferenc Erdelyi --- .../hadoop/yarn/server/resourcemanager/ResourceManager.java | 9 + .../scheduler/capacity/conf/YarnConfigurationStore.java | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 90eaed3d8a02..f358a22cee71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -1903,10 +1903,11 @@ public class ResourceManager extends CompositeService } if (scheduler instanceof MutableConfScheduler && isConfigurationMutable) { - YarnConfigurationStore confStore = YarnConfigurationStoreFactory - .getStore(conf); - confStore.initialize(conf, conf, rmContext); - confStore.format(); + try (YarnConfigurationStore confStore = YarnConfigurationStoreFactory + .getStore(conf)) { +confStore.initialize(conf, conf, rmContext); +confStore.format(); + } } else { System.out.println(String.format("Scheduler Configuration format only " + "supported by %s.", MutableConfScheduler.class.getSimpleName())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java index 0f44f439056b..2ec2ea442a64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java @@ -44,7 +44,7 @@ import java.util.Map; * {@code getPendingMutations}, and replay/confirm them via * {@code confirmMutation} as in the normal case. */ -public abstract class YarnConfigurationStore { +public abstract class YarnConfigurationStore implements AutoCloseable { public static final Logger LOG = LoggerFactory.getLogger(YarnConfigurationStore.class); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: MAPREDUCE-7456. Extend add-opens flag to container launch commands on JDK17 nodes. Contributed by Peter Szucs
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 2d871fab7888 MAPREDUCE-7456. Extend add-opens flag to container launch commands on JDK17 nodes. Contributed by Peter Szucs 2d871fab7888 is described below commit 2d871fab7888aa531a199aa52bc1e0875f6a85f3 Author: Szilard Nemeth AuthorDate: Wed Sep 27 22:33:21 2023 -0400 MAPREDUCE-7456. Extend add-opens flag to container launch commands on JDK17 nodes. Contributed by Peter Szucs --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 8 + .../src/main/resources/yarn-default.xml| 11 +++ .../containermanager/launcher/ContainerLaunch.java | 8 +++-- .../localizer/ContainerLocalizer.java | 15 ++ .../launcher/TestContainerLaunch.java | 8 +++-- .../localizer/TestContainerLocalizer.java | 35 ++ 6 files changed, 81 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 13d4c209110e..90a8978a228b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2232,6 +2232,14 @@ public class YarnConfiguration extends Configuration { public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT = "-Xmx256m"; + /* + * Flag to indicate whether JDK17's required add-exports flags should be added to + * container localizers regardless of the user specified JAVA_OPTS. + */ + public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_KEY = + NM_PREFIX + "container-localizer.java.opts.add-exports-as-default"; + public static final boolean NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_DEFAULT = true; + /** The log level of container localizer process. */ public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL= NM_PREFIX + "container-localizer.log.level"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 2259b73fb651..9991e841d74b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1484,6 +1484,17 @@ -Xmx256m + + yarn.nodemanager.container-localizer.java.opts.add-exports-as-default +true +Since on JDK17 it is no longer possible to use the reflection API to + access public fields and methods, add-exports flags should be added to container + localizers regardless of the user specified JAVA_OPTS. Setting this to true will + add the flags to the container localizer on nodes with JDK17 or higher. + Defaults to true, but the setting has no effect on nodes using JDK16 and before. + + + The log level for container localizer while it is an independent process. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 143086db2aec..5466182b715d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -123,6 +123,11 @@ public class ContainerLaunch implements Callable { private static final String PID_FILE_NAME_FMT = "%s.pid"; static final String EXIT_CODE_FILE_SUFFIX = ".exitcode"; + private static final String ADDITIONAL_JDK17_PLUS_OPTIONS = + "--add-opens=java.base/java.lang=ALL-UNNAMED " + + "--add-exports=java.base/sun.net.dns=ALL-UNNAMED " + + "--add-exports=java.base/sun.net.util=ALL-UNNAMED"; + protected final Dispatcher dispatcher; protected final ContainerExecutor exec; protected final Application app; @@ -171,8 +176,7 @@ public class ContainerLaunch implements Callable { File.
[hadoop] branch trunk updated: YARN-11468. Zookeeper SSL/TLS support. Contributed by Ferenc Erdelyi
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new d9cb76ac985a YARN-11468. Zookeeper SSL/TLS support. Contributed by Ferenc Erdelyi d9cb76ac985a is described below commit d9cb76ac985a1f28208e29a6f1efa6d161720fb0 Author: Szilard Nemeth AuthorDate: Wed Sep 27 18:21:45 2023 -0400 YARN-11468. Zookeeper SSL/TLS support. Contributed by Ferenc Erdelyi --- .../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java| 4 .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 6 ++ .../apache/hadoop/yarn/server/resourcemanager/ResourceManager.java | 3 ++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index bbb1ed6f8a7d..13d4c209110e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -857,6 +857,10 @@ public class YarnConfiguration extends Configuration { /** Zookeeper interaction configs */ public static final String RM_ZK_PREFIX = RM_PREFIX + "zk-"; + /** Enable Zookeeper SSL/TLS communication. */ + public static final String RM_ZK_CLIENT_SSL_ENABLED = RM_ZK_PREFIX + "client-ssl.enabled"; + public static final boolean DEFAULT_RM_ZK_CLIENT_SSL_ENABLED = false; + public static final String RM_ZK_ADDRESS = RM_ZK_PREFIX + "address"; public static final String RM_ZK_NUM_RETRIES = RM_ZK_PREFIX + "num-retries"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 9fa600db4b03..2259b73fb651 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -741,6 +741,12 @@ 1048576 + +Enable SSL/TLS encryption for the ZooKeeper communication. +yarn.resourcemanager.zk-client-ssl.enabled +false + + Name of the cluster. In a HA setting, this is used to ensure the RM participates in leader diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 2730dde72fc1..90eaed3d8a02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -427,7 +427,8 @@ public class ResourceManager extends CompositeService authInfos.add(authInfo); } -manager.start(authInfos); +manager.start(authInfos, config.getBoolean(YarnConfiguration.RM_ZK_CLIENT_SSL_ENABLED, +YarnConfiguration.DEFAULT_RM_ZK_CLIENT_SSL_ENABLED)); return manager; } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-18870. CURATOR-599 change broke functionality introduced in HADOOP-18139 and HADOOP-18709. Contributed by Ferenc Erdelyi
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 9342ecf6ccd HADOOP-18870. CURATOR-599 change broke functionality introduced in HADOOP-18139 and HADOOP-18709. Contributed by Ferenc Erdelyi 9342ecf6ccd is described below commit 9342ecf6ccd5c7ef443a0eb722852d2addc1d5db Author: Szilard Nemeth AuthorDate: Wed Sep 6 21:32:36 2023 -0400 HADOOP-18870. CURATOR-599 change broke functionality introduced in HADOOP-18139 and HADOOP-18709. Contributed by Ferenc Erdelyi --- .../hadoop/util/curator/ZKCuratorManager.java | 12 +++- .../hadoop/util/curator/TestZKCuratorManager.java | 35 ++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java index 81ee4663e1c..4df79774329 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java @@ -549,7 +549,17 @@ public final class ZKCuratorManager { public ZooKeeper newZooKeeper(String connectString, int sessionTimeout, Watcher watcher, boolean canBeReadOnly ) throws Exception { - ZKClientConfig zkClientConfig = new ZKClientConfig(); + return this.newZooKeeper(connectString, sessionTimeout, + watcher, canBeReadOnly, new ZKClientConfig()); +} + +@Override +public ZooKeeper newZooKeeper(String connectString, int sessionTimeout, +Watcher watcher, boolean canBeReadOnly, ZKClientConfig zkClientConfig +) throws Exception { + if (zkClientConfig == null) { +zkClientConfig = new ZKClientConfig(); + } if (zkPrincipal != null) { LOG.info("Configuring zookeeper to use {} as the server principal", zkPrincipal); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java index fd15a0c2b1b..4365e43e491 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java @@ -22,10 +22,15 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import javax.security.auth.login.AppConfigurationEntry; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryNTimes; import org.apache.curator.test.TestingServer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -193,6 +198,36 @@ public class TestZKCuratorManager { } } + @Test + public void testCuratorFrameworkFactory() throws Exception{ +// By not explicitly calling the NewZooKeeper method validate that the Curator override works. +ZKClientConfig zkClientConfig = new ZKClientConfig(); +Configuration conf = new Configuration(); +conf.set(CommonConfigurationKeys.ZK_ADDRESS, this.server.getConnectString()); +int numRetries = conf.getInt(CommonConfigurationKeys.ZK_NUM_RETRIES, +CommonConfigurationKeys.ZK_NUM_RETRIES_DEFAULT); +int zkSessionTimeout = conf.getInt(CommonConfigurationKeys.ZK_TIMEOUT_MS, +CommonConfigurationKeys.ZK_TIMEOUT_MS_DEFAULT); +int zkRetryInterval = conf.getInt( +CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS, +CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS_DEFAULT); +RetryNTimes retryPolicy = new RetryNTimes(numRetries, zkRetryInterval); + +CuratorFramework client = CuratorFrameworkFactory.builder() +.connectString(conf.get(CommonConfigurationKeys.ZK_ADDRESS)) +.zkClientConfig(zkClientConfig) +.sessionTimeoutMs(zkSessionTimeout).retryPolicy(retryPolicy) +.authorization(new ArrayList<>()) +.zookeeperFactory(new ZKCuratorManager.HadoopZookeeperFactory( +"foo1", "bar1", "bar1.keytab", false, +new ZKCuratorManager.TruststoreKeystore(conf)) + +).build(); +client.start(); + validateJaasConfiguration(ZKCuratorManager.HadoopZookeeperFactory.JAAS_CLIENT_ENTRY, +"bar1", "bar1.keytab", client.getZookeeperClient().getZooKeeper()); + } + private voi
[hadoop] branch trunk updated: MAPREDUCE-7441. Fix race condition in closing FadvisedFileRegion. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 1c15987ee38 MAPREDUCE-7441. Fix race condition in closing FadvisedFileRegion. Contributed by Benjamin Teke 1c15987ee38 is described below commit 1c15987ee38b96cf2ef4ff383f3f86e082c50fab Author: Szilard Nemeth AuthorDate: Fri Jun 23 14:40:03 2023 -0400 MAPREDUCE-7441. Fix race condition in closing FadvisedFileRegion. Contributed by Benjamin Teke --- .../apache/hadoop/mapred/FadvisedFileRegion.java | 102 - 1 file changed, 58 insertions(+), 44 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java index 9290a282e39..184b58e6c76 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java @@ -41,6 +41,7 @@ public class FadvisedFileRegion extends DefaultFileRegion { private static final Logger LOG = LoggerFactory.getLogger(FadvisedFileRegion.class); + private final Object closeLock = new Object(); private final boolean manageOsCache; private final int readaheadLength; private final ReadaheadPool readaheadPool; @@ -51,12 +52,12 @@ public class FadvisedFileRegion extends DefaultFileRegion { private final int shuffleBufferSize; private final boolean shuffleTransferToAllowed; private final FileChannel fileChannel; - - private ReadaheadRequest readaheadRequest; + + private volatile ReadaheadRequest readaheadRequest; public FadvisedFileRegion(RandomAccessFile file, long position, long count, boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool, - String identifier, int shuffleBufferSize, + String identifier, int shuffleBufferSize, boolean shuffleTransferToAllowed) throws IOException { super(file.getChannel(), position, count); this.manageOsCache = manageOsCache; @@ -73,97 +74,110 @@ public class FadvisedFileRegion extends DefaultFileRegion { @Override public long transferTo(WritableByteChannel target, long position) - throws IOException { -if (readaheadPool != null && readaheadLength > 0) { - readaheadRequest = readaheadPool.readaheadStream(identifier, fd, - position() + position, readaheadLength, - position() + count(), readaheadRequest); + throws IOException { +synchronized (closeLock) { + if (fd.valid()) { +if (readaheadPool != null && readaheadLength > 0) { + readaheadRequest = readaheadPool.readaheadStream(identifier, fd, + position() + position, readaheadLength, + position() + count(), readaheadRequest); +} + +if(this.shuffleTransferToAllowed) { + return super.transferTo(target, position); +} else { + return customShuffleTransfer(target, position); +} + } else { +return 0L; + } } - -if(this.shuffleTransferToAllowed) { - return super.transferTo(target, position); -} else { - return customShuffleTransfer(target, position); -} + } /** - * This method transfers data using local buffer. It transfers data from - * a disk to a local buffer in memory, and then it transfers data from the + * This method transfers data using local buffer. It transfers data from + * a disk to a local buffer in memory, and then it transfers data from the * buffer to the target. This is used only if transferTo is disallowed in - * the configuration file. super.TransferTo does not perform well on Windows - * due to a small IO request generated. customShuffleTransfer can control - * the size of the IO requests by changing the size of the intermediate + * the configuration file. super.TransferTo does not perform well on Windows + * due to a small IO request generated. customShuffleTransfer can control + * the size of the IO requests by changing the size of the intermediate * buffer. */ @VisibleForTesting long customShuffleTransfer(WritableByteChannel target, long position) - throws IOException { + throws IOException { long actualCount = this.count - position; if (actualCount < 0 || position < 0) { throw new IllegalArgumentException( - "position out of range: " + position + - " (expected: 0 - " + (this.count -
[hadoop] branch trunk updated: HADOOP-18709. Add curator based ZooKeeper communication support over SSL/TLS into the common library. Contributed by Ferenc Erdelyi
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new e0a339223a8 HADOOP-18709. Add curator based ZooKeeper communication support over SSL/TLS into the common library. Contributed by Ferenc Erdelyi e0a339223a8 is described below commit e0a339223a89367d9391758ff62c99923369b5ee Author: Szilard Nemeth AuthorDate: Sun Jun 4 14:40:41 2023 -0400 HADOOP-18709. Add curator based ZooKeeper communication support over SSL/TLS into the common library. Contributed by Ferenc Erdelyi --- hadoop-common-project/hadoop-common/pom.xml| 8 + .../apache/hadoop/fs/CommonConfigurationKeys.java | 8 + .../hadoop/util/curator/ZKCuratorManager.java | 222 ++--- .../src/main/resources/core-default.xml| 29 +++ .../util/curator/TestSecureZKCuratorManager.java | 214 .../util/curator/resources/data/ssl/keystore.jks | Bin 0 -> 4160 bytes .../util/curator/resources/data/ssl/truststore.jks | Bin 0 -> 994 bytes 7 files changed, 458 insertions(+), 23 deletions(-) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 58006c011da..426f7a4af41 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -340,6 +340,14 @@ + + io.netty + netty-handler + + + io.netty + netty-transport-native-epoll + io.dropwizard.metrics metrics-core diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index 9d6224366d1..63f494a20ac 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -417,6 +417,14 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { /** How often to retry a ZooKeeper operation in milliseconds. */ public static final String ZK_RETRY_INTERVAL_MS = ZK_PREFIX + "retry-interval-ms"; + /** Keystore location for ZooKeeper client connection over SSL. */ + public static final String ZK_SSL_KEYSTORE_LOCATION = ZK_PREFIX + "ssl.keystore.location"; + /** Keystore password for ZooKeeper client connection over SSL. */ + public static final String ZK_SSL_KEYSTORE_PASSWORD = ZK_PREFIX + "ssl.keystore.password"; + /** Truststore location for ZooKeeper client connection over SSL. */ + public static final String ZK_SSL_TRUSTSTORE_LOCATION = ZK_PREFIX + "ssl.truststore.location"; + /** Truststore password for ZooKeeper client connection over SSL. */ + public static final String ZK_SSL_TRUSTSTORE_PASSWORD = ZK_PREFIX + "ssl.truststore.password"; public static final intZK_RETRY_INTERVAL_MS_DEFAULT = 1000; /** Default domain name resolver for hadoop to use. */ public static final String HADOOP_DOMAINNAME_RESOLVER_IMPL = diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java index c11b8683867..81ee4663e1c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.AuthInfo; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -39,13 +40,17 @@ import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.client.ZKClientConfig; +import org.apache.zookeeper.common.ClientX509Util; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.util.Preconditions; +import javax.naming.ConfigurationException; + /** * Helper class that provides utility methods specific to ZK operations. */ @@ -122,7 +127,7 @@ public final class ZKCuratorManager { * Start the connection to the ZooKeeper ensemble. * @throws IOException If the connection cannot be started. */ - public void start() throws IOException { + public void start() throws IOException{ this.start(new ArrayList<>());
[hadoop] branch trunk updated: YARN-11079. Make an AbstractParentQueue to store common ParentQueue and ManagedParentQueue functionality. Contributed by Susheel Gupta
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new c7699d3dcd4 YARN-11079. Make an AbstractParentQueue to store common ParentQueue and ManagedParentQueue functionality. Contributed by Susheel Gupta c7699d3dcd4 is described below commit c7699d3dcd4f8feaf2c5ae5943b8a4cec738e95d Author: Szilard Nemeth AuthorDate: Thu May 4 22:16:18 2023 -0400 YARN-11079. Make an AbstractParentQueue to store common ParentQueue and ManagedParentQueue functionality. Contributed by Susheel Gupta --- .../ProportionalCapacityPreemptionPolicy.java |6 +- .../monitor/capacity/TempQueuePerPartition.java|3 +- .../csmappingrule/MappingRuleValidationHelper.java |5 +- .../scheduler/capacity/AbstractCSQueue.java|6 +- .../capacity/AbstractManagedParentQueue.java |2 +- .../{ParentQueue.java => AbstractParentQueue.java} | 233 +-- .../capacity/AutoCreatedQueueManagementPolicy.java |4 +- .../capacity/CSMaxRunningAppsEnforcer.java | 12 +- .../scheduler/capacity/CapacityScheduler.java |2 +- .../capacity/CapacitySchedulerConfigValidator.java |4 +- .../capacity/CapacitySchedulerQueueManager.java|9 +- .../scheduler/capacity/ManagedParentQueue.java |2 +- .../scheduler/capacity/ParentQueue.java| 1572 +--- .../scheduler/capacity/PlanQueue.java |1 + .../GuaranteedOrZeroCapacityOverTimePolicy.java|8 +- .../event/QueueManagementChangeEvent.java |9 +- .../webapp/dao/CapacitySchedulerInfo.java |6 +- .../webapp/dao/CapacitySchedulerQueueInfo.java |6 +- .../dao/helper/CapacitySchedulerInfoHelper.java|3 +- .../TestCapacitySchedulerNewQueueAutoCreation.java |4 +- .../capacity/TestCapacitySchedulerQueues.java |2 +- .../scheduler/capacity/TestChildQueueOrder.java|4 +- .../scheduler/capacity/TestParentQueue.java|4 +- 23 files changed, 133 insertions(+), 1774 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index 1c4d60962ee..443241a664a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -21,6 +21,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractParentQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -41,7 +42,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .ManagedParentQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptableQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent; @@ -630,9 +630,9 @@ public class ProportionalCapacityPreemptionPolicy partitionToLookAt, killable, absCap, absMaxCap, partitionResource, reserved, curQueue, effMinRes, effMaxRes); - if (curQueue instanceof ParentQueue) { + if (curQueue instanceof AbstractParentQueue) { String configuredOrderingPolicy = -((ParentQueue) curQueue).getQueueOrderingPolicy().getConfigName(); +((AbstractParentQueue) curQueue).getQueueOrderingPolicy().getConfigName(); // Recursively add children for (CSQueue c : curQueue.getChildQueues()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-y
[hadoop] branch trunk updated: MAPREDUCE-7434. Fix ShuffleHandler tests. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 8f6be3678d1 MAPREDUCE-7434. Fix ShuffleHandler tests. Contributed by Tamas Domok 8f6be3678d1 is described below commit 8f6be3678d1113e3e7f5477c357fc81f62d460b8 Author: Szilard Nemeth AuthorDate: Wed Mar 1 16:10:05 2023 +0100 MAPREDUCE-7434. Fix ShuffleHandler tests. Contributed by Tamas Domok --- .../hadoop/mapred/TestShuffleChannelHandler.java | 2 +- .../apache/hadoop/mapred/TestShuffleHandler.java | 44 +++--- .../hadoop/mapred/TestShuffleHandlerBase.java | 29 +++--- 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleChannelHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleChannelHandler.java index 7fedc7bb2dc..66fa3de94f8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleChannelHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleChannelHandler.java @@ -225,7 +225,7 @@ public class TestShuffleChannelHandler extends TestShuffleHandlerBase { final ShuffleTest t = createShuffleTest(); final EmbeddedChannel shuffle = t.createShuffleHandlerChannelFileRegion(); -String dataFile = getDataFile(tempDir.toAbsolutePath().toString(), TEST_ATTEMPT_2); +String dataFile = getDataFile(TEST_USER, tempDir.toAbsolutePath().toString(), TEST_ATTEMPT_2); assertTrue("should delete", new File(dataFile).delete()); FullHttpRequest req = t.createRequest(getUri(TEST_JOB_ID, 0, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java index 37a9210286c..cc46b49b113 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java @@ -29,6 +29,7 @@ import static org.apache.hadoop.test.MetricsAsserts.assertCounter; import static org.apache.hadoop.test.MetricsAsserts.assertGauge; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -41,6 +42,7 @@ import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.MalformedURLException; @@ -159,7 +161,7 @@ public class TestShuffleHandler extends TestShuffleHandlerBase { shuffleHandler.init(conf); shuffleHandler.start(); final String port = shuffleHandler.getConfig().get(SHUFFLE_PORT_CONFIG_KEY); -final SecretKey secretKey = shuffleHandler.addTestApp(); +final SecretKey secretKey = shuffleHandler.addTestApp(TEST_USER); // setup connections HttpURLConnection[] conns = new HttpURLConnection[connAttempts]; @@ -237,7 +239,7 @@ public class TestShuffleHandler extends TestShuffleHandlerBase { shuffleHandler.init(conf); shuffleHandler.start(); final String port = shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY); -final SecretKey secretKey = shuffleHandler.addTestApp(); +final SecretKey secretKey = shuffleHandler.addTestApp(TEST_USER); HttpURLConnection conn1 = createRequest( geURL(port, TEST_JOB_ID, 0, Collections.singletonList(TEST_ATTEMPT_1), true), @@ -278,18 +280,34 @@ public class TestShuffleHandler extends TestShuffleHandlerBase { conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); +final String randomUser = "randomUser"; +final String attempt = "attempt_1_0004_m_04_0"; +generateMapOutput(randomUser, tempDir.toAbsolutePath().toString(), attempt, +Arrays.asList(TEST_DATA_C, TEST_DATA_B, TEST_DATA_A)); + ShuffleHandlerMock shuffleHandler = new ShuffleHandlerMock();
[hadoop] branch trunk updated: HADOOP-18602. Remove netty3 dependency
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new b677d40ab5e HADOOP-18602. Remove netty3 dependency b677d40ab5e is described below commit b677d40ab5ee899a0264b1763d632221cefe44b4 Author: Szilard Nemeth AuthorDate: Fri Jan 27 16:32:50 2023 +0100 HADOOP-18602. Remove netty3 dependency --- LICENSE-binary | 1 - hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 - hadoop-project/pom.xml | 8 3 files changed, 14 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 95ceb0d97c1..432dc5d28f7 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -260,7 +260,6 @@ io.grpc:grpc-netty:1.26.0 io.grpc:grpc-protobuf:1.26.0 io.grpc:grpc-protobuf-lite:1.26.0 io.grpc:grpc-stub:1.26.0 -io.netty:netty:3.10.6.Final io.netty:netty-all:4.1.77.Final io.netty:netty-buffer:4.1.77.Final io.netty:netty-codec:4.1.77.Final diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index b51c7154f7b..4e7efab06e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -175,11 +175,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";> slf4j-log4j12 provided - - io.netty - netty - compile - io.netty netty-all diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 33350c929e8..aada4e9b6de 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -140,7 +140,6 @@ 2.2.21 2.9.0 3.2.4 -3.10.6.Final 4.1.77.Final 1.1.8.2 1.7.1 @@ -1000,13 +999,6 @@ - - -io.netty -netty -${netty3.version} - - io.netty netty-all - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-6971. Clean up different ways to create resources. Contributed by Riya Khandelwal
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 815cde98103 YARN-6971. Clean up different ways to create resources. Contributed by Riya Khandelwal 815cde98103 is described below commit 815cde98103fe57dde4f6686626bc4fcc1278a2d Author: Szilard Nemeth AuthorDate: Wed Jan 25 17:28:29 2023 +0100 YARN-6971. Clean up different ways to create resources. Contributed by Riya Khandelwal --- .../v2/app/rm/TestRMContainerAllocator.java| 29 .../server/uam/UnmanagedApplicationManager.java| 4 +- .../server/nodemanager/TestNodeStatusUpdater.java | 13 ++-- .../containermanager/BaseContainerManagerTest.java | 5 +- .../containermanager/container/TestContainer.java | 3 +- .../launcher/TestContainerLaunch.java | 3 +- .../logaggregation/TestLogAggregationService.java | 3 +- .../monitor/TestContainersMonitor.java | 3 +- ...estContainerSchedulerBehaviorCompatibility.java | 4 +- ...ContainerSchedulerOppContainersByResources.java | 8 +-- .../scheduler/TestContainerSchedulerQueuing.java | 78 +++--- .../TestNMContainerTokenSecretManager.java | 3 +- .../server/nodemanager/webapp/MockContainer.java | 3 +- .../server/nodemanager/webapp/TestNMWebServer.java | 4 +- .../server/resourcemanager/ClientRMService.java| 3 +- .../resourcemanager/DefaultAMSProcessor.java | 2 +- .../scheduler/fair/FairSchedulerConfiguration.java | 3 +- .../server/resourcemanager/QueueACLsTestBase.java | 4 +- .../resourcemanager/ReservationACLsTestBase.java | 4 +- .../resourcemanager/TestApplicationACLs.java | 6 +- .../resourcemanager/TestClientRMService.java | 8 +-- .../server/resourcemanager/TestClientRMTokens.java | 5 +- .../TestResourceTrackerService.java| 35 +- .../TestRMDelegatedNodeLabelsUpdater.java | 4 +- .../resourcetracker/TestNMExpiry.java | 4 +- .../resourcetracker/TestNMReconnect.java | 10 +-- .../resourcetracker/TestRMNMRPCResponseId.java | 4 +- .../rmapp/attempt/TestRMAppAttemptTransitions.java | 8 +-- .../rmcontainer/TestRMContainerImpl.java | 9 +-- .../scheduler/TestAbstractYarnScheduler.java | 5 +- .../capacity/TestApplicationPriorityACLs.java | 4 +- .../TestCapacitySchedulerAmbiguousLeafs.java | 4 +- .../scheduler/fair/FairSchedulerTestBase.java | 3 +- .../scheduler/fair/TestFairScheduler.java | 6 +- .../fair/TestFairSchedulerConfiguration.java | 19 +++--- .../TestDominantResourceFairnessPolicy.java| 5 +- .../scheduler/fifo/TestFifoScheduler.java | 10 +-- .../webapp/TestRMWebServicesNodes.java | 4 +- 38 files changed, 170 insertions(+), 162 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java index fd7a2f0ceda..fe2f3072141 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java @@ -148,6 +148,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.ControlledClock; import org.apache.hadoop.yarn.util.Records; +import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.SystemClock; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -502,8 +503,8 @@ public class TestRMContainerAllocator { 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob, SystemClock.getInstance()); -allocator.setMapResourceRequest(BuilderUtils.newResource(1024, 1)); -allocator.setReduceResourceRequest(BuilderUtils.newResource(1024, 1)); +allocator.setMapResourceRequest(Resources.createResource(1024)); +allocator.setReduceResourceRequest(Resources.createResource(1024)); RMContainerAllocator.AssignedRequests assignedRequests = allocator.getAssignedRequests(); RMContainerAllocator.ScheduledRequests scheduledRequests = @@ -557,8 +558,8 @@ public class TestRMContainerAllocator { clock.setTime(1); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttem
[hadoop] branch trunk updated: YARN-5607. Document TestContainerResourceUsage#waitForContainerCompletion. Contributed by Susheel Gupta
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 29f2230cb61 YARN-5607. Document TestContainerResourceUsage#waitForContainerCompletion. Contributed by Susheel Gupta 29f2230cb61 is described below commit 29f2230cb615b975950691044a621ff008088cf5 Author: Szilard Nemeth AuthorDate: Wed Jan 25 15:13:24 2023 +0100 YARN-5607. Document TestContainerResourceUsage#waitForContainerCompletion. Contributed by Susheel Gupta --- .../hadoop/yarn/server/resourcemanager/MockRM.java | 25 ++ .../TestContainerResourceUsage.java| 18 ++-- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 95db1171b35..316f8e06cb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -1011,6 +1011,31 @@ public class MockRM extends ResourceManager { LOG.info("app is removed from scheduler, " + appId); } + /** + * Wait until a container has reached a completion state. + * The timeout is 20 seconds. + * @param nm A mock nodemanager + * @param rm A mock resourcemanager + * @param amContainerId The id of an am container + * @param container A container + * @throws Exception + * if interrupted while waiting for the completion transition + * or an unexpected error while MockNM is hearbeating. + */ + public static void waitForContainerCompletion(MockRM rm, MockNM nm, +ContainerId amContainerId, RMContainer container) throws Exception { +ContainerId containerId = container.getContainerId(); +if (null != rm.scheduler.getRMContainer(containerId)) { + if (containerId.equals(amContainerId)) { +rm.waitForState(nm, containerId, RMContainerState.COMPLETED); + } else { +rm.waitForState(nm, containerId, RMContainerState.KILLED); + } +} else { + rm.drainEvents(); +} + } + private void drainEventsImplicitly() { if (!disableDrainEventsImplicitly) { drainEvents(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java index 9a01087ce3a..54fd6509647 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java @@ -348,7 +348,7 @@ public class TestContainerResourceUsage { // If keepRunningContainers is false, all live containers should now // be completed. Calculate the resource usage metrics for all of them. for (RMContainer c : rmContainers) { -waitforContainerCompletion(rm, nm, amContainerId, c); +MockRM.waitForContainerCompletion(rm, nm, amContainerId, c); AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); memorySeconds += ru.getMemorySeconds(); vcoreSeconds += ru.getVcoreSeconds(); @@ -400,7 +400,7 @@ public class TestContainerResourceUsage { // Calculate container usage metrics for second attempt. for (RMContainer c : rmContainers) { - waitforContainerCompletion(rm, nm, amContainerId, c); + MockRM.waitForContainerCompletion(rm, nm, amContainerId, c); AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); memorySeconds += ru.getMemorySeconds(); vcoreSeconds += ru.getVcoreSeconds(); @@ -417,20 +417,6 @@ public class TestContainerResourceUsage { return; } - private void waitforContainerCompletion(MockRM rm, MockNM nm, - ContainerId amContainerId, RMContainer container) throws Exception { -ContainerId containerId = container.getContainerId(); -if (null != rm.scheduler.getRMContainer(containerId)) { - if (containerId.equals(amContainerId)) { -rm.waitForState(nm, c
[hadoop] branch trunk updated: YARN-10946. AbstractCSQueue: Create separate class for constructing Queue API objects. Contributed by Peter Szucs
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 5440c75c4ab YARN-10946. AbstractCSQueue: Create separate class for constructing Queue API objects. Contributed by Peter Szucs 5440c75c4ab is described below commit 5440c75c4abee87aeff6da3c7e3e586978cd27dc Author: Szilard Nemeth AuthorDate: Thu Dec 1 15:11:58 2022 +0100 YARN-10946. AbstractCSQueue: Create separate class for constructing Queue API objects. Contributed by Peter Szucs --- .../scheduler/capacity/AbstractCSQueue.java| 80 +- .../scheduler/capacity/CSQueueInfoProvider.java| 117 + 2 files changed, 118 insertions(+), 79 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 1a5a1ce0fd4..7a3ec4f6f4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -29,10 +29,8 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; -import org.apache.hadoop.yarn.api.records.QueueConfigurations; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueState; -import org.apache.hadoop.yarn.api.records.QueueStatistics; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -584,83 +582,7 @@ public abstract class AbstractCSQueue implements CSQueue { // from schedulerApplicationAttempt, to avoid deadlock, sacrifice // consistency here. // TODO, improve this -QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); -queueInfo.setQueueName(queuePath.getLeafName()); -queueInfo.setQueuePath(queuePath.getFullPath()); - queueInfo.setAccessibleNodeLabels(queueNodeLabelsSettings.getAccessibleNodeLabels()); -queueInfo.setCapacity(queueCapacities.getCapacity()); -queueInfo.setMaximumCapacity(queueCapacities.getMaximumCapacity()); -queueInfo.setQueueState(getState()); - queueInfo.setDefaultNodeLabelExpression(queueNodeLabelsSettings.getDefaultLabelExpression()); -queueInfo.setCurrentCapacity(getUsedCapacity()); -queueInfo.setQueueStatistics(getQueueStatistics()); -queueInfo.setPreemptionDisabled(preemptionSettings.isPreemptionDisabled()); -queueInfo.setIntraQueuePreemptionDisabled( -getIntraQueuePreemptionDisabled()); -queueInfo.setQueueConfigurations(getQueueConfigurations()); -queueInfo.setWeight(queueCapacities.getWeight()); - queueInfo.setMaxParallelApps(queueAppLifetimeSettings.getMaxParallelApps()); -return queueInfo; - } - - public QueueStatistics getQueueStatistics() { -// Deliberately doesn't use lock here, because this method will be invoked -// from schedulerApplicationAttempt, to avoid deadlock, sacrifice -// consistency here. -// TODO, improve this -QueueStatistics stats = recordFactory.newRecordInstance( -QueueStatistics.class); -stats.setNumAppsSubmitted(getMetrics().getAppsSubmitted()); -stats.setNumAppsRunning(getMetrics().getAppsRunning()); -stats.setNumAppsPending(getMetrics().getAppsPending()); -stats.setNumAppsCompleted(getMetrics().getAppsCompleted()); -stats.setNumAppsKilled(getMetrics().getAppsKilled()); -stats.setNumAppsFailed(getMetrics().getAppsFailed()); -stats.setNumActiveUsers(getMetrics().getActiveUsers()); -stats.setAvailableMemoryMB(getMetrics().getAvailableMB()); -stats.setAllocatedMemoryMB(getMetrics().getAllocatedMB()); -stats.setPendingMemoryMB(getMetrics().getPendingMB()); -stats.setReservedMemoryMB(getMetrics().getReservedMB()); -stats.setAvailableVCores(getMetrics().getAvailableVirtualCores()); -stats.setAllocatedVCores(getMetrics().getAllocatedVirtualCores()); -stats.setPendingVCores(getMetrics().getPendingVirtualCores()); -stats.setReservedVCores(getMetrics().getReservedVirtualCores()); -stats.setPendingContainers(getMetrics().getPendingContainers()); -stats.setAllocatedContainers(getMe
[hadoop] branch trunk updated: YARN-8262. get_executable in container-executor should provide meaningful error codes. Contributed by Susheel Gupta
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 3c37a01654e YARN-8262. get_executable in container-executor should provide meaningful error codes. Contributed by Susheel Gupta 3c37a01654e is described below commit 3c37a01654e0ddca3d88a9fe7ba7b0830ac44ea6 Author: Szilard Nemeth AuthorDate: Tue Nov 22 13:37:55 2022 +0100 YARN-8262. get_executable in container-executor should provide meaningful error codes. Contributed by Susheel Gupta --- .../server/nodemanager/LinuxContainerExecutor.java | 8 ++-- .../container-executor/impl/get_executable.c | 22 +++--- .../src/main/native/container-executor/impl/util.c | 10 ++ .../src/main/native/container-executor/impl/util.h | 7 ++- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index e899215291b..ea4595dffc4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -175,8 +175,12 @@ public class LinuxContainerExecutor extends ContainerExecutor { COULD_NOT_CREATE_WORK_DIRECTORIES(35), COULD_NOT_CREATE_APP_LOG_DIRECTORIES(36), COULD_NOT_CREATE_TMP_DIRECTORIES(37), -ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS(38); - +ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS(38), +CANNOT_GET_EXECUTABLE_NAME_FROM_READLINK(80), +TOO_LONG_EXECUTOR_PATH(81), +CANNOT_GET_EXECUTABLE_NAME_FROM_KERNEL(82), +CANNOT_GET_EXECUTABLE_NAME_FROM_PID(83), +WRONG_PATH_OF_EXECUTABLE(84); private final int code; ExitCode(int exitCode) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c index e1ec293cd47..b027e51bd31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c @@ -56,17 +56,17 @@ char *__get_exec_readproc(char *procfn) { filename = malloc(EXECUTOR_PATH_MAX); if (!filename) { fprintf(ERRORFILE,"cannot allocate memory for filename before readlink: %s\n",strerror(errno)); -exit(-1); +exit(OUT_OF_MEMORY); } len = readlink(procfn, filename, EXECUTOR_PATH_MAX); if (len == -1) { -fprintf(ERRORFILE,"Can't get executable name from %s - %s\n", procfn, +fprintf(ERRORFILE,"Cannot get executable name from %s - %s\n", procfn, strerror(errno)); -exit(-1); +exit(CANNOT_GET_EXECUTABLE_NAME_FROM_READLINK); } else if (len >= EXECUTOR_PATH_MAX) { fprintf(ERRORFILE,"Resolved path for %s [%s] is longer than %d characters.\n", procfn, filename, EXECUTOR_PATH_MAX); -exit(-1); +exit(TOO_LONG_EXECUTOR_PATH); } filename[len] = '\0'; return filename; @@ -88,14 +88,14 @@ char *__get_exec_sysctl(int *mib) len = sizeof(buffer); if (sysctl(mib, 4, buffer, &len, NULL, 0) == -1) { -fprintf(ERRORFILE,"Can't get executable name from kernel: %s\n", +fprintf(ERRORFILE,"Cannot get executable name from kernel: %s\n", strerror(errno)); -exit(-1); +exit(CANNOT_GET_EXECUTABLE_NAME_FROM_KERNEL); } filename=malloc(EXECUTOR_PATH_MAX); if (!filename) { fprintf(ERRORFILE,"cannot allocate memory for filename after sysctl: %s\n",strerror(errno)); -exit(-1); +exit(OUT_OF_MEMORY); } snprintf(filename,EXECUTOR_PATH_MAX,"%s",buffer); return filename; @@ -120,13 +120,13 @@ char* get_executable(char *argv0) { filename = malloc(PROC_PIDPATHINFO_MAXSIZE); if (!filename) { fprintf(ERRORFILE,"cannot allocate memory for filename before proc_pidpath: %s\n",strerror(errno)); -exit(-1); +exit(OUT_OF_MEMORY); } pid = getpid(); if (proc_pidpath(pid,filename,PROC_PIDPATHINFO_MAXSIZE) <= 0) { -fprintf(ERRORFILE,"Ca
[hadoop] branch trunk updated: YARN-11369. Commons.compress throws an IllegalArgumentException with large uids after 1.21. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 142df247eda YARN-11369. Commons.compress throws an IllegalArgumentException with large uids after 1.21. Contributed by Benjamin Teke 142df247eda is described below commit 142df247edab25f0f1e829ea8559fb42a925af7b Author: Szilard Nemeth AuthorDate: Wed Nov 16 13:07:05 2022 +0100 YARN-11369. Commons.compress throws an IllegalArgumentException with large uids after 1.21. Contributed by Benjamin Teke --- .../main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java index 862d68ebc0a..52b6dde3794 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java @@ -331,6 +331,8 @@ public class FrameworkUploader implements Runnable { LOG.info("Compressing tarball"); try (TarArchiveOutputStream out = new TarArchiveOutputStream( targetStream)) { + // Workaround for the compress issue present from 1.21: COMPRESS-587 + out.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR); for (String fullPath : filteredInputFiles) { LOG.info("Adding " + fullPath); File file = new File(fullPath); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10005. Code improvements in MutableCSConfigurationProvider. Contributed by Peter Szucs
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 22c9f28f4d8 YARN-10005. Code improvements in MutableCSConfigurationProvider. Contributed by Peter Szucs 22c9f28f4d8 is described below commit 22c9f28f4d85a92d23024b03c9094edb8e56f369 Author: Szilard Nemeth AuthorDate: Sat Nov 12 18:35:07 2022 +0100 YARN-10005. Code improvements in MutableCSConfigurationProvider. Contributed by Peter Szucs --- .../conf/ConfigurationUpdateAssembler.java | 181 .../conf/MutableCSConfigurationProvider.java | 188 ++--- .../conf/TestConfigurationUpdateAssembler.java | 173 +++ 3 files changed, 370 insertions(+), 172 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java new file mode 100644 index 000..88c93019680 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY; + +public final class ConfigurationUpdateAssembler { + + private ConfigurationUpdateAssembler() { + } + + public static Map constructKeyValueConfUpdate( + CapacitySchedulerConfiguration proposedConf, + SchedConfUpdateInfo mutationInfo) throws IOException { + +Map confUpdate = new HashMap<>(); +for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) { + removeQueue(queueToRemove, proposedConf, confUpdate); +} +for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) { + addQueue(addQueueInfo, proposedConf, confUpdate); +} +for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) { + updateQueue(updateQueueInfo, proposedConf, confUpdate); +} +for (Map.Entry global : mutationInfo.getGlobalParams() +.entrySet()) { + confUpdate.put(global.getKey(), global.getValue()); +} +return confUpdate; + } + + private static void removeQueue( + String queueToRemove, CapacitySchedulerConfiguration proposedConf, + Map confUpdate) throws IOException { +if (queueToRemove == null) { + return; +} +if (queueToRemove.lastIndexOf('.') == -1) { + throw new IOException("Can't remove queue " + queueToRemove); +} +String queueName = queueToRemove.substring( +queueToRemove.lastIndexOf('.') + 1); +List siblingQueues = getSiblingQueues(queueToRemove, +proposedConf); +if (!siblingQueues.contains(queueName)) { + throw new IOException("Queue " + queueToRemove + " not found"); +} +siblingQueues.remove(queueName); +String parentQueuePath = queueToRemove.substring(0, queueToRemove +.lastIndexOf('.')); +proposedConf.
[hadoop] branch trunk updated: YARN-10680. Revisit try blocks without catch blocks but having finally blocks. Contributed by Susheel Gupta
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new b0d5182c310 YARN-10680. Revisit try blocks without catch blocks but having finally blocks. Contributed by Susheel Gupta b0d5182c310 is described below commit b0d5182c31043d70d614d3378c42974d6f468b27 Author: Szilard Nemeth AuthorDate: Sat Oct 15 21:51:08 2022 +0200 YARN-10680. Revisit try blocks without catch blocks but having finally blocks. Contributed by Susheel Gupta --- .../apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java | 10 +- .../org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java | 10 +- .../org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java | 8 +++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java index 18e12cca05f..6cc4cf7597e 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java @@ -36,6 +36,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.sls.SLSRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Private @Unstable @@ -45,6 +47,7 @@ public class SLSCapacityScheduler extends CapacityScheduler implements private final SLSSchedulerCommons schedulerCommons; private Configuration conf; private SLSRunner runner; + private static final Logger LOG = LoggerFactory.getLogger(SLSCapacityScheduler.class); public SLSCapacityScheduler() { schedulerCommons = new SLSSchedulerCommons(this); @@ -105,7 +108,12 @@ public class SLSCapacityScheduler extends CapacityScheduler implements @Override public void handle(SchedulerEvent schedulerEvent) { -schedulerCommons.handle(schedulerEvent); +try { + schedulerCommons.handle(schedulerEvent); +} catch(Exception e) { + LOG.error("Caught exception while handling scheduler event", e); + throw e; +} } @Override diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java index 1b4d5ced69b..beb411025f8 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java @@ -31,6 +31,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.sls.SLSRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; @@ -40,6 +42,7 @@ public class SLSFairScheduler extends FairScheduler implements SchedulerWrapper, Configurable { private final SLSSchedulerCommons schedulerCommons; private SLSRunner runner; + private static final Logger LOG = LoggerFactory.getLogger(SLSFairScheduler.class); public SLSFairScheduler() { schedulerCommons = new SLSSchedulerCommons(this); @@ -63,7 +66,12 @@ public class SLSFairScheduler extends FairScheduler @Override public void handle(SchedulerEvent schedulerEvent) { -schedulerCommons.handle(schedulerEvent); +try { + schedulerCommons.handle(schedulerEvent); +} catch (Exception e){ + LOG.error("Caught exception while handling scheduler event", e); + throw e; +} } @Override diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java index a1e530a6f77..f66cf4384d9 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java @@ -178,7 +178,7 @@ public abstract class SchedulerMetrics { pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000, TimeUnit.MILLISECONDS); -// a thread to output metrics for real-tiem tracking +// a thread to output metrics fo
[hadoop] branch trunk updated: YARN-6766. Add helper method in FairSchedulerAppsBlock to print app info. Contributed by Riya Khandelwal
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 0c515b0ef09 YARN-6766. Add helper method in FairSchedulerAppsBlock to print app info. Contributed by Riya Khandelwal 0c515b0ef09 is described below commit 0c515b0ef09b21fcd3e800bf54c1a7920ebf23fc Author: Szilard Nemeth AuthorDate: Mon Oct 10 15:28:27 2022 +0200 YARN-6766. Add helper method in FairSchedulerAppsBlock to print app info. Contributed by Riya Khandelwal --- .../webapp/FairSchedulerAppsBlock.java | 26 +- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java index f6202cbcc51..c74e2ae3e1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java @@ -129,6 +129,12 @@ public class FairSchedulerAppsBlock extends HtmlBlock { return true; } + private static String printAppInfo(long value) { +if (value == -1) { + return "N/A"; +} +return String.valueOf(value); + } @Override public void render(Block html) { TBODY> tbody = html. @@ -193,16 +199,16 @@ public class FairSchedulerAppsBlock extends HtmlBlock { .append(appInfo.getFinishTime()).append("\",\"") .append(appInfo.getState()).append("\",\"") .append(appInfo.getFinalStatus()).append("\",\"") - .append(appInfo.getRunningContainers() == -1 ? "N/A" : String - .valueOf(appInfo.getRunningContainers())).append("\",\"") - .append(appInfo.getAllocatedVCores() == -1 ? "N/A" : String -.valueOf(appInfo.getAllocatedVCores())).append("\",\"") - .append(appInfo.getAllocatedMB() == -1 ? "N/A" : String -.valueOf(appInfo.getAllocatedMB())).append("\",\"") - .append(appInfo.getReservedVCores() == -1 ? "N/A" : String -.valueOf(appInfo.getReservedVCores())).append("\",\"") - .append(appInfo.getReservedMB() == -1 ? "N/A" : String -.valueOf(appInfo.getReservedMB())).append("\",\"") + .append(printAppInfo(appInfo.getRunningContainers())) + .append("\",\"") + .append(printAppInfo(appInfo.getAllocatedVCores())) + .append("\",\"") + .append(printAppInfo(appInfo.getAllocatedMB())) + .append("\",\"") + .append(printAppInfo(appInfo.getReservedVCores())) + .append("\",\"") + .append(printAppInfo(appInfo.getReservedMB())) + .append("\",\"") // Progress bar .append("
[hadoop] branch trunk updated: YARN-11211. QueueMetrics leaks Configuration objects when validation API is called multiple times. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new f4b635c4dcb YARN-11211. QueueMetrics leaks Configuration objects when validation API is called multiple times. Contributed by Andras Gyori f4b635c4dcb is described below commit f4b635c4dcb44fb180605d2c540b52eef7b8bde0 Author: Szilard Nemeth AuthorDate: Thu Jul 21 14:20:34 2022 +0200 YARN-11211. QueueMetrics leaks Configuration objects when validation API is called multiple times. Contributed by Andras Gyori --- .../scheduler/PartitionQueueMetrics.java| 2 +- .../server/resourcemanager/scheduler/QueueMetrics.java | 17 ++--- .../scheduler/capacity/CSQueueMetrics.java | 7 ++- .../capacity/CapacitySchedulerConfigValidator.java | 3 +++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java index 02eaa7bd9b7..150abdf51d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java @@ -78,7 +78,7 @@ public class PartitionQueueMetrics extends QueueMetrics { metrics = new PartitionQueueMetrics(this.metricsSystem, this.queueName, null, false, this.conf, this.partition); users.put(userName, metrics); - metricsSystem.register( + registerMetrics( pSourceName(partitionJMXStr).append(qSourceName(queueName)) .append(",user=").append(userName).toString(), "Metrics for user '" + userName + "' in queue '" + queueName + "'", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index d41cbed0b10..0bfee4d3350 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -289,7 +289,7 @@ public class QueueMetrics implements MetricsSource { metrics = new QueueMetrics(metricsSystem, queueName, null, false, conf); users.put(userName, metrics); - metricsSystem.register( + registerMetrics( sourceName(queueName).append(",user=").append(userName).toString(), "Metrics for user '"+ userName +"' in queue '"+ queueName +"'", metrics.tag(QUEUE_INFO, queueName).tag(USER_INFO, userName)); @@ -334,7 +334,7 @@ public class QueueMetrics implements MetricsSource { QueueMetrics queueMetrics = new PartitionQueueMetrics(metricsSystem, this.queueName, parentQueue, this.enableUserMetrics, this.conf, partition); - metricsSystem.register( + registerMetrics( pSourceName(partitionJMXStr).append(qSourceName(this.queueName)) .toString(), "Metrics for queue: " + this.queueName, @@ -378,7 +378,7 @@ public class QueueMetrics implements MetricsSource { // Register with the MetricsSystems if (metricsSystem != null) { -metricsSystem.register(pSourceName(partitionJMXStr).toString(), +registerMetrics(pSourceName(partitionJMXStr).toString(), "Metrics for partition: " + partitionJMXStr, (PartitionQueueMetrics) metrics.tag(PARTITION_INFO, partitionJMXStr)); @@ -1359,4 +1359,15 @@ public class QueueMetrics implements MetricsSource { } } } + + protected void registerMetrics(String sourceName, String desc, QueueMetrics metrics) { +MetricsSource source = metricsSystem.getSource(sourceName); +// Unregister metrics if a source is already present +if (source != null) { + LOG.info("Unregi
[hadoop] branch trunk updated: YARN-11185. Pending app metrics are increased doubly when a queue reaches its max-parallel-apps limit. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 3a66348fda2 YARN-11185. Pending app metrics are increased doubly when a queue reaches its max-parallel-apps limit. Contributed by Andras Gyori 3a66348fda2 is described below commit 3a66348fda29b18054853baed6664fec92dc5cc5 Author: Szilard Nemeth AuthorDate: Mon Jun 20 15:03:58 2022 +0200 YARN-11185. Pending app metrics are increased doubly when a queue reaches its max-parallel-apps limit. Contributed by Andras Gyori --- .../scheduler/capacity/AbstractLeafQueue.java | 4 +- .../capacity/TestCapacitySchedulerApps.java| 77 ++ 2 files changed, 51 insertions(+), 30 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java index ac5c8a15167..08fedb578ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java @@ -578,6 +578,8 @@ public class AbstractLeafQueue extends AbstractCSQueue { public void submitApplicationAttempt(FiCaSchedulerApp application, String userName, boolean isMoveApp) { // Careful! Locking order is important! +boolean isAppAlreadySubmitted = applicationAttemptMap.containsKey( +application.getApplicationAttemptId()); writeLock.lock(); try { // TODO, should use getUser, use this method just to avoid UT failure @@ -591,7 +593,7 @@ public class AbstractLeafQueue extends AbstractCSQueue { } // We don't want to update metrics for move app -if (!isMoveApp) { +if (!isMoveApp && !isAppAlreadySubmitted) { boolean unmanagedAM = application.getAppSchedulingInfo() != null && application.getAppSchedulingInfo().isUnmanagedAM(); usageTracker.getMetrics().submitAppAttempt(userName, unmanagedAM); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java index ea22c24b355..d192e7dcc69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -116,6 +117,8 @@ import static org.mockito.Mockito.when; public class TestCapacitySchedulerApps { + public static final int MAX_PARALLEL_APPS = 5; + public static final String USER_0 = "user_0"; private ResourceManager resourceManager = null; private RMContext mockContext; @@ -237,18 +240,7 @@ public class TestCapacitySchedulerApps { YarnScheduler scheduler = rm.getResourceScheduler(); // submit an app -MockRMAppSubmissionData data = -MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) -.withAppName("test-move-1") -.withUser("user_0") -.withAcls(null) -.withQueue("a1") -.withUnmanagedAM(false) -.build(); -RMApp app = MockRMAppSubmitter.submit(rm, data); -ApplicationAttemptId appAttemptId = -rm.getApplicationReport(app.getApplicationId()) -.getCurrentApplicationAttemptId(); +ApplicationAttemptId appAttemptId = submitApp(rm); // check preconditions List appsInA1 = scheduler.getAppsInQueue("a1"); @@ -1020,18 +1012,7 @@ public class TestCapacitySchedulerApps { (AbstractYarnScheduler) rm.getResourceScheduler(); // submit an app -MockRMAppSubmissionData data = -MockRMAppSubmissionDat
[hadoop] branch branch-3.3 updated: YARN-11141. Capacity Scheduler does not support ambiguous queue names when moving application across queues. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 90ec4418c76 YARN-11141. Capacity Scheduler does not support ambiguous queue names when moving application across queues. Contributed by Andras Gyori 90ec4418c76 is described below commit 90ec4418c769c1aefe236d9cb79ce3c3da1c2601 Author: Szilard Nemeth AuthorDate: Wed May 18 14:34:08 2022 +0200 YARN-11141. Capacity Scheduler does not support ambiguous queue names when moving application across queues. Contributed by Andras Gyori --- .../resourcemanager/scheduler/capacity/CapacityScheduler.java | 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index d0d95c388a6..ba1ab940f5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2531,7 +2531,12 @@ public class CapacityScheduler extends if (application == null) { throw new YarnException("App to be moved " + appId + " not found."); } - String sourceQueueName = application.getQueue().getQueueName(); + if (!(application.getQueue() instanceof CSQueue)) { +throw new YarnException("Source queue is not a Capacity Scheduler queue"); + } + + CSQueue csQueue = (CSQueue) application.getQueue(); + String sourceQueueName = csQueue.getQueuePath(); LeafQueue source = this.queueManager.getAndCheckLeafQueue(sourceQueueName); String destQueueName = handleMoveToPlanQueue(targetQueueName); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 4f112e3138c YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok 4f112e3138c is described below commit 4f112e3138cba4f74a00f48b268b4180a3eed4f1 Author: Szilard Nemeth AuthorDate: Wed May 18 14:25:35 2022 +0200 YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok --- .../capacity/conf/ZKConfigurationStore.java| 5 ++-- .../capacity/conf/TestZKConfigurationStore.java| 35 ++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java index 703193b1b29..d55435a4c4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; +import org.apache.commons.io.serialization.ValidatingObjectInputStream; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.zookeeper.KeeperException.NodeExistsException; import org.slf4j.Logger; @@ -35,7 +36,6 @@ import org.apache.zookeeper.data.ACL; import java.io.IOException; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.util.HashMap; import java.util.LinkedList; @@ -314,7 +314,8 @@ public class ZKConfigurationStore extends YarnConfigurationStore { private static Object deserializeObject(byte[] bytes) throws Exception { try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes); -ObjectInputStream ois = new ObjectInputStream(bais);) { + ValidatingObjectInputStream ois = new ValidatingObjectInputStream(bais);) { + ois.accept(LinkedList.class, LogMutation.class, HashMap.class, String.class); return ois.readObject(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java index 880ba77fa51..155996d11fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java @@ -42,15 +42,18 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.Yar import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.io.ByteArrayOutputStream; import java.io.ObjectOutputStream; import java.util.Arrays; +import java.util.Base64; import java.util.HashMap; import java.util.Map; @@ -67,6 +70,9 @@ public class TestZKConfigurationStore extends LoggerFactory.getLogger(TestZKConfigurationStore.class); private static final int ZK_TIMEOUT_MS = 1; + private static final String DESERIALIZATION_VULNERABILITY_FILEPATH = + "/tmp/ZK_DESERIALIZATION_VULNERABILITY"; + private TestingServer curatorTestingServer; private CuratorFramework curatorFramework; private ResourceManager rm; @@ -401,6 +407,35 @@ public class TestZKConfigurationStore extends rm2.close(); } + @Test(timeout = 3000) + @SuppressWarnings("checkstyle:linelength") + public void testDeserializationIsNotVulnerabl
[hadoop] branch branch-3.2 updated: YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 45801fba8b0 YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok 45801fba8b0 is described below commit 45801fba8b00257ab32c02a7d1a05948ba687a49 Author: Szilard Nemeth AuthorDate: Wed May 18 14:23:56 2022 +0200 YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok --- .../capacity/conf/ZKConfigurationStore.java| 5 ++-- .../capacity/conf/TestZKConfigurationStore.java| 35 ++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java index 6f3612c28da..cacf32b3911 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.io.serialization.ValidatingObjectInputStream; import org.apache.zookeeper.KeeperException.NodeExistsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +36,6 @@ import org.apache.zookeeper.data.ACL; import java.io.IOException; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.util.HashMap; import java.util.LinkedList; @@ -317,7 +317,8 @@ public class ZKConfigurationStore extends YarnConfigurationStore { private static Object deserializeObject(byte[] bytes) throws Exception { try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes); -ObjectInputStream ois = new ObjectInputStream(bais);) { + ValidatingObjectInputStream ois = new ValidatingObjectInputStream(bais);) { + ois.accept(LinkedList.class, LogMutation.class, HashMap.class, String.class); return ois.readObject(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java index 40454a4621b..cb1da9c8963 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java @@ -41,15 +41,18 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.io.ByteArrayOutputStream; import java.io.ObjectOutputStream; import java.util.Arrays; +import java.util.Base64; import java.util.HashMap; import java.util.Map; @@ -66,6 +69,9 @@ public class TestZKConfigurationStore extends LoggerFactory.getLogger(TestZKConfigurationStore.class); private static final int ZK_TIMEOUT_MS = 1; + private static final String DESERIALIZATION_VULNERABILITY_FILEPATH = + "/tmp/ZK_DESERIALIZATION_VULNERABILITY"; + private TestingServer curatorTestingServer; private CuratorFramework curatorFramework; private ResourceManager rm; @@ -398,6 +404,35 @@ public class TestZKConfigurationStore extends rm2.close(); } + @Test(timeout = 3000) + @SuppressWarnings("checkstyle:linelength") + public void testDeserializationIsNotVulnerabl
[hadoop] branch branch-3.3 updated: YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new b4550b33564 YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke b4550b33564 is described below commit b4550b3356439f012ca6f97383889e236067e689 Author: Szilard Nemeth AuthorDate: Wed May 18 14:08:41 2022 +0200 YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke --- .../hadoop/yarn/client/api/impl/AHSv2ClientImpl.java | 5 ++--- .../client/api/impl/TimelineReaderClientImpl.java| 16 ++-- .../api/impl/TestTimelineReaderClientImpl.java | 20 +++- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java index b6a0c591c90..5fd3ee21fc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java @@ -137,9 +137,8 @@ public class AHSv2ClientImpl extends AHSClient { ApplicationId appId = applicationAttemptId.getApplicationId(); ApplicationReport appReport = getApplicationReport(appId); Map filters = new HashMap<>(); -filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" + -applicationAttemptId.toString() + -"\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}"); +filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq " ++ "{\"type\":\"YARN_APPLICATION_ATTEMPT\",\"id\":\"" + applicationAttemptId + "\"}"); List entities = readerClient.getContainerEntities( appId, "ALL", filters, 0, null); List containers = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java index d5af8a0df56..de30fb84be0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java @@ -38,7 +38,10 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MultivaluedMap; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -208,12 +211,21 @@ public class TimelineReaderClientImpl extends TimelineReaderClient { return Arrays.asList(entity); } + @VisibleForTesting + protected String encodeValue(String value) throws UnsupportedEncodingException { +// Since URLEncoder doesn't use and doesn't have an option for percent-encoding +// (as specified in RFC 3986) the spaces are encoded to + signs, which need to be replaced +// manually +return URLEncoder.encode(value, StandardCharsets.UTF_8.toString()) +.replaceAll("\\+", "%20"); + } + private void mergeFilters(MultivaluedMap defaults, - Map filters) { +Map filters) throws UnsupportedEncodingException { if (filters != null && !filters.isEmpty()) { for (Map.Entry entry : filters.entrySet()) { if (!defaults.containsKey(entry.getKey())) { - defaults.add(entry.getKey(), filters.get(entry.getValue())); + defaults.add(entry.getKey(), encodeValue(entry.getValue())); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java index f668472256a..757aeb8c31d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/
[hadoop] branch branch-3.2 updated: YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 1ae834fe0b1 YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke 1ae834fe0b1 is described below commit 1ae834fe0b1efb28a4fec56e9532b88864392cbe Author: Szilard Nemeth AuthorDate: Wed May 18 14:04:51 2022 +0200 YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke --- .../hadoop/yarn/client/api/impl/AHSv2ClientImpl.java | 5 ++--- .../client/api/impl/TimelineReaderClientImpl.java| 16 ++-- .../api/impl/TestTimelineReaderClientImpl.java | 20 +++- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java index e797c281625..3422493aa1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java @@ -129,9 +129,8 @@ public class AHSv2ClientImpl extends AHSClient { applicationAttemptId) throws YarnException, IOException { ApplicationId appId = applicationAttemptId.getApplicationId(); Map filters = new HashMap<>(); -filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" + -applicationAttemptId.toString() + -"\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}"); +filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq " ++ "{\"type\":\"YARN_APPLICATION_ATTEMPT\",\"id\":\"" + applicationAttemptId + "\"}"); List entities = readerClient.getContainerEntities( appId, "ALL", filters, 0, null); List containers = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java index db53f93136f..aafe4c3db11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java @@ -38,7 +38,10 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MultivaluedMap; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -208,12 +211,21 @@ public class TimelineReaderClientImpl extends TimelineReaderClient { return Arrays.asList(entity); } + @VisibleForTesting + protected String encodeValue(String value) throws UnsupportedEncodingException { +// Since URLEncoder doesn't use and doesn't have an option for percent-encoding +// (as specified in RFC 3986) the spaces are encoded to + signs, which need to be replaced +// manually +return URLEncoder.encode(value, StandardCharsets.UTF_8.toString()) +.replaceAll("\\+", "%20"); + } + private void mergeFilters(MultivaluedMap defaults, - Map filters) { +Map filters) throws UnsupportedEncodingException { if (filters != null && !filters.isEmpty()) { for (Map.Entry entry : filters.entrySet()) { if (!defaults.containsKey(entry.getKey())) { - defaults.add(entry.getKey(), filters.get(entry.getValue())); + defaults.add(entry.getKey(), encodeValue(entry.getValue())); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java index f668472256a..d6ca25f9501 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-comm
[hadoop] branch trunk updated: YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 5e2f4339fad YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok 5e2f4339fad is described below commit 5e2f4339fadc88f20543915fc9b0aaeaf4f9e7bf Author: Szilard Nemeth AuthorDate: Thu May 12 13:42:06 2022 +0200 YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok --- .../capacity/conf/ZKConfigurationStore.java| 5 ++-- .../capacity/conf/TestZKConfigurationStore.java| 35 ++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java index 71226c300a8..ad8fb97a7a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; +import org.apache.commons.io.serialization.ValidatingObjectInputStream; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.zookeeper.KeeperException.NodeExistsException; import org.slf4j.Logger; @@ -35,7 +36,6 @@ import org.apache.zookeeper.data.ACL; import java.io.IOException; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.util.HashMap; import java.util.LinkedList; @@ -314,7 +314,8 @@ public class ZKConfigurationStore extends YarnConfigurationStore { private static Object deserializeObject(byte[] bytes) throws Exception { try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes); -ObjectInputStream ois = new ObjectInputStream(bais);) { + ValidatingObjectInputStream ois = new ValidatingObjectInputStream(bais);) { + ois.accept(LinkedList.class, LogMutation.class, HashMap.class, String.class); return ois.readObject(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java index 880ba77fa51..155996d11fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java @@ -42,15 +42,18 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.Yar import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.io.ByteArrayOutputStream; import java.io.ObjectOutputStream; import java.util.Arrays; +import java.util.Base64; import java.util.HashMap; import java.util.Map; @@ -67,6 +70,9 @@ public class TestZKConfigurationStore extends LoggerFactory.getLogger(TestZKConfigurationStore.class); private static final int ZK_TIMEOUT_MS = 1; + private static final String DESERIALIZATION_VULNERABILITY_FILEPATH = + "/tmp/ZK_DESERIALIZATION_VULNERABILITY"; + private TestingServer curatorTestingServer; private CuratorFramework curatorFramework; private ResourceManager rm; @@ -401,6 +407,35 @@ public class TestZKConfigurationStore extends rm2.close(); } + @Test(timeout = 3000) + @SuppressWarnings("checkstyle:linelength") + public void testDeserializationIsNotVulnerable() throws Exception { +confStore.initialize(conf,
[hadoop] branch trunk updated: MAPREDUCE-7379. RMContainerRequestor#makeRemoteRequest has confusing log message. Contributed by Ashutosh Gupta
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new f143e994285 MAPREDUCE-7379. RMContainerRequestor#makeRemoteRequest has confusing log message. Contributed by Ashutosh Gupta f143e994285 is described below commit f143e994285543e2f1a779274f826f38a78a27be Author: Szilard Nemeth AuthorDate: Wed May 11 16:55:19 2022 +0200 MAPREDUCE-7379. RMContainerRequestor#makeRemoteRequest has confusing log message. Contributed by Ashutosh Gupta --- .../hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java | 10 -- 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java index 61cc2eb898e..16ca585e9b4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java @@ -213,12 +213,10 @@ public abstract class RMContainerRequestor extends RMCommunicator { allocateResponse.getCompletedContainersStatuses().size(); if (ask.size() > 0 || release.size() > 0) { - LOG.info("getResources() for " + applicationId + ":" + " ask=" - + ask.size() + " release= " + release.size() + " newContainers=" - + allocateResponse.getAllocatedContainers().size() - + " finishedContainers=" + numCompletedContainers - + " resourcelimit=" + availableResources + " knownNMs=" - + clusterNmCount); + LOG.info("applicationId={}: ask={} release={} newContainers={} finishedContainers={}" + + " resourceLimit={} knownNMs={}", applicationId, ask.size(), release.size(), + allocateResponse.getAllocatedContainers().size(), numCompletedContainers, + availableResources, clusterNmCount); } ask.clear(); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new eeedc0c7e41 YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke eeedc0c7e41 is described below commit eeedc0c7e41c7932169af2b05556ca155c765b22 Author: Szilard Nemeth AuthorDate: Wed May 11 14:39:42 2022 +0200 YARN-10850. TimelineService v2 lists containers for all attempts when filtering for one. Contributed by Benjamin Teke --- .../hadoop/yarn/client/api/impl/AHSv2ClientImpl.java | 5 ++--- .../client/api/impl/TimelineReaderClientImpl.java| 16 ++-- .../api/impl/TestTimelineReaderClientImpl.java | 20 +++- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java index e3653baa2b7..26f45baaee0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java @@ -137,9 +137,8 @@ public class AHSv2ClientImpl extends AHSClient { ApplicationId appId = applicationAttemptId.getApplicationId(); ApplicationReport appReport = getApplicationReport(appId); Map filters = new HashMap<>(); -filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" + -applicationAttemptId.toString() + -"\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}"); +filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq " ++ "{\"type\":\"YARN_APPLICATION_ATTEMPT\",\"id\":\"" + applicationAttemptId + "\"}"); List entities = readerClient.getContainerEntities( appId, "ALL", filters, 0, null); List containers = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java index 29609b955fb..71bf13220b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java @@ -38,7 +38,10 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MultivaluedMap; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -208,12 +211,21 @@ public class TimelineReaderClientImpl extends TimelineReaderClient { return Arrays.asList(entity); } + @VisibleForTesting + protected String encodeValue(String value) throws UnsupportedEncodingException { +// Since URLEncoder doesn't use and doesn't have an option for percent-encoding +// (as specified in RFC 3986) the spaces are encoded to + signs, which need to be replaced +// manually +return URLEncoder.encode(value, StandardCharsets.UTF_8.toString()) +.replaceAll("\\+", "%20"); + } + private void mergeFilters(MultivaluedMap defaults, - Map filters) { +Map filters) throws UnsupportedEncodingException { if (filters != null && !filters.isEmpty()) { for (Map.Entry entry : filters.entrySet()) { if (!defaults.containsKey(entry.getKey())) { - defaults.add(entry.getKey(), filters.get(entry.getValue())); + defaults.add(entry.getKey(), encodeValue(entry.getValue())); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java index f668472256a..757aeb8c31d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/
[hadoop] branch trunk updated: YARN-11141. Capacity Scheduler does not support ambiguous queue names when moving application across queues. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new b62d6ce6fd6 YARN-11141. Capacity Scheduler does not support ambiguous queue names when moving application across queues. Contributed by Andras Gyori b62d6ce6fd6 is described below commit b62d6ce6fd6cd568e9b9a8729a03c7f773875fca Author: Szilard Nemeth AuthorDate: Wed May 11 14:27:34 2022 +0200 YARN-11141. Capacity Scheduler does not support ambiguous queue names when moving application across queues. Contributed by Andras Gyori --- .../scheduler/capacity/CapacityScheduler.java | 7 +++- .../capacity/CapacitySchedulerQueueHelpers.java| 30 .../capacity/CapacitySchedulerTestUtilities.java | 11 ++ .../capacity/TestCapacitySchedulerApps.java| 41 ++ 4 files changed, 88 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 44e80a6c234..cf5034ba228 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2678,7 +2678,12 @@ public class CapacityScheduler extends if (application == null) { throw new YarnException("App to be moved " + appId + " not found."); } - String sourceQueueName = application.getQueue().getQueueName(); + if (!(application.getQueue() instanceof CSQueue)) { +throw new YarnException("Source queue is not a Capacity Scheduler queue"); + } + + CSQueue csQueue = (CSQueue) application.getQueue(); + String sourceQueueName = csQueue.getQueuePath(); AbstractLeafQueue source = this.queueManager.getAndCheckLeafQueue(sourceQueueName); String destQueueName = handleMoveToPlanQueue(targetQueueName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java index bc7e2b317ac..7e362731f86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java @@ -29,6 +29,7 @@ public final class CapacitySchedulerQueueHelpers { public static final String A = CapacitySchedulerConfiguration.ROOT + ".a"; public static final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + public static final String A_CHILD = A + ".a"; public static final String A1 = A + ".a1"; public static final String A2 = A + ".a2"; public static final String B1 = B + ".b1"; @@ -89,6 +90,35 @@ public final class CapacitySchedulerQueueHelpers { return conf; } + /** + * @param conf, to be modified + * @return CS configuration which has deleted all children of queue(b) + * root + * / \ + *ab + * / \ + * a1 a2 + */ + public static CapacitySchedulerConfiguration setupQueueConfAmbiguousQueue( + CapacitySchedulerConfiguration conf) { + +// Define top-level queues +conf.setQueues(CapacitySchedulerConfiguration.ROOT, +new String[]{"a", "b"}); + +conf.setCapacity(A, A_CAPACITY); +conf.setCapacity(B, B_CAPACITY); + +// Define 2nd-level queues +conf.setQueues(A, new String[]{"a", "a1"}); +conf.setCapacity(A_CHILD, A1_CAPACITY); +conf.setUserLimitFactor(A1, 100.0f); +conf.setCapacity(A1, A2_CAPACITY); +conf.setUserLimitFactor(A2, 100.0f); + +return conf; + } + /** * @param conf, to be modified * @return CS configuration which has deleted all childred of que
[hadoop] branch trunk updated: YARN-11086. Add space in debug log of ParentQueue. Contributed by Junfan Zhang.
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new e2701e2 YARN-11086. Add space in debug log of ParentQueue. Contributed by Junfan Zhang. e2701e2 is described below commit e2701e227f82b89c9dcde53ef8fa49f757a61fa8 Author: Szilard Nemeth AuthorDate: Mon Mar 21 14:37:10 2022 +0100 YARN-11086. Add space in debug log of ParentQueue. Contributed by Junfan Zhang. --- .../yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 87ebc0b..c0ad7f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -465,7 +465,7 @@ public class ParentQueue extends AbstractCSQueue { "numChildQueue= " + childQueues.size() + ", " + getCapacityOrWeightString() + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " + -"usedResources=" + usageTracker.getQueueUsage().getUsed() + +"usedResources=" + usageTracker.getQueueUsage().getUsed() + ", " + "usedCapacity=" + getUsedCapacity() + ", " + "numApps=" + getNumApplications() + ", " + "numContainers=" + getNumContainers(); @@ -1148,7 +1148,7 @@ public class ParentQueue extends AbstractCSQueue { StringBuilder sb = new StringBuilder(); for (CSQueue q : childQueues) { sb.append(q.getQueuePath() + - "usedCapacity=(" + q.getUsedCapacity() + "), " + + " usedCapacity=(" + q.getUsedCapacity() + "), " + " label=(" + StringUtils.join(q.getAccessibleNodeLabels().iterator(), ",") + ")"); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11089. Fix typo in RM audit log. Contributed by Junfan Zhang.
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 4537b34 YARN-11089. Fix typo in RM audit log. Contributed by Junfan Zhang. 4537b34 is described below commit 4537b34e1c088f2b6d61c9bad8b82438cd94b944 Author: Szilard Nemeth AuthorDate: Mon Mar 21 14:18:27 2022 +0100 YARN-11089. Fix typo in RM audit log. Contributed by Junfan Zhang. --- .../org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 7ebcd65..928eeb1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -294,7 +294,7 @@ public class RMAppManager implements EventHandler, protected void writeAuditLog(ApplicationId appId) { RMApp app = rmContext.getRMApps().get(appId); -String operation = "UNKONWN"; +String operation = "UNKNOWN"; boolean success = false; switch (app.getState()) { case FAILED: - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11067. Resource overcommitment due to incorrect resource normalisation logical order. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new ed65aa2 YARN-11067. Resource overcommitment due to incorrect resource normalisation logical order. Contributed by Andras Gyori ed65aa2 is described below commit ed65aa23240b3dd6b56e86e5f0e9d38069fb3b01 Author: Szilard Nemeth AuthorDate: Thu Mar 10 22:22:58 2022 +0100 YARN-11067. Resource overcommitment due to incorrect resource normalisation logical order. Contributed by Andras Gyori --- .../scheduler/capacity/ParentQueue.java| 37 - .../TestAbsoluteResourceConfiguration.java | 47 ++ 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index c624aab..87ebc0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -1294,17 +1294,24 @@ public class ParentQueue extends AbstractCSQueue { private void calculateEffectiveResourcesAndCapacity(String label, Resource clusterResource) { +// Update effective resources for my self; +if (rootQueue) { + Resource resourceByLabel = labelManager.getResourceByLabel(label, clusterResource); + usageTracker.getQueueResourceQuotas().setEffectiveMinResource(label, resourceByLabel); + usageTracker.getQueueResourceQuotas().setEffectiveMaxResource(label, resourceByLabel); +} else { + super.updateEffectiveResources(clusterResource); +} + +recalculateEffectiveMinRatio(label, clusterResource); + } + + private void recalculateEffectiveMinRatio(String label, Resource clusterResource) { // For root queue, ensure that max/min resource is updated to latest // cluster resource. -Resource resourceByLabel = labelManager.getResourceByLabel(label, -clusterResource); - -/* - * == Below logic are added to calculate effectiveMinRatioPerResource == - */ +Resource resourceByLabel = labelManager.getResourceByLabel(label, clusterResource); -// Total configured min resources of direct children of this given parent -// queue +// Total configured min resources of direct children of this given parent queue Resource configuredMinResources = Resource.newInstance(0L, 0); for (CSQueue childQueue : getChildQueues()) { Resources.addTo(configuredMinResources, @@ -1312,8 +1319,7 @@ public class ParentQueue extends AbstractCSQueue { } // Factor to scale down effective resource: When cluster has sufficient -// resources, effective_min_resources will be same as configured -// min_resources. +// resources, effective_min_resources will be same as configured min_resources. Resource numeratorForMinRatio = null; if (getQueuePath().equals("root")) { if (!resourceByLabel.equals(Resources.none()) && Resources.lessThan(resourceCalculator, @@ -1324,21 +1330,12 @@ public class ParentQueue extends AbstractCSQueue { if (Resources.lessThan(resourceCalculator, clusterResource, usageTracker.getQueueResourceQuotas().getEffectiveMinResource(label), configuredMinResources)) { -numeratorForMinRatio = usageTracker.getQueueResourceQuotas() -.getEffectiveMinResource(label); +numeratorForMinRatio = usageTracker.getQueueResourceQuotas().getEffectiveMinResource(label); } } effectiveMinResourceRatio.put(label, getEffectiveMinRatio( configuredMinResources, numeratorForMinRatio)); - -// Update effective resources for my self; -if (rootQueue) { - usageTracker.getQueueResourceQuotas().setEffectiveMinResource(label, resourceByLabel); - usageTracker.getQueueResourceQuotas().setEffectiveMaxResource(label, resourceByLabel); -} else{ - super.updateEffectiveResources(clusterResource); -} } private Map getEffectiveMinRatio( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager
[hadoop] branch trunk updated: YARN-10049. FIFOOrderingPolicy Improvements. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 481da19 YARN-10049. FIFOOrderingPolicy Improvements. Contributed by Benjamin Teke 481da19 is described below commit 481da19494fe13ca42651305b519e0670cafabf0 Author: Szilard Nemeth AuthorDate: Thu Mar 10 22:15:35 2022 +0100 YARN-10049. FIFOOrderingPolicy Improvements. Contributed by Benjamin Teke --- .../scheduler/policy/FifoComparator.java | 5 ++ .../scheduler/policy/TestFifoOrderingPolicy.java | 91 -- 2 files changed, 71 insertions(+), 25 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java index 112c50f..c62b738 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java @@ -29,6 +29,11 @@ public class FifoComparator @Override public int compare(SchedulableEntity r1, SchedulableEntity r2) { int res = r1.compareInputOrderTo(r2); + +if (res == 0) { + res = (int) Math.signum(r1.getStartTime() - r2.getStartTime()); +} + return res; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java index 7ec2c01..62bc712 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java @@ -18,16 +18,19 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy; -import java.util.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; -import org.junit.Assert; -import org.junit.Test; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; import org.apache.hadoop.yarn.api.records.Priority; +import org.junit.Assert; +import org.junit.Test; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestFifoOrderingPolicy { +public +class TestFifoOrderingPolicy { @Test public void testFifoOrderingPolicy() { @@ -36,13 +39,17 @@ public class TestFifoOrderingPolicy { MockSchedulableEntity r1 = new MockSchedulableEntity(); MockSchedulableEntity r2 = new MockSchedulableEntity(); -assertThat(policy.getComparator().compare(r1, r2)).isEqualTo(0); +assertEquals("The comparator should return 0 because the entities are created with " + +"the same values.", 0, +policy.getComparator().compare(r1, r2)); r1.setSerial(1); -assertThat(policy.getComparator().compare(r1, r2)).isEqualTo(1); +assertEquals("The lhs entity has a larger serial, the comparator return " + +"value should be 1.", 1, policy.getComparator().compare(r1, r2)); r2.setSerial(2); -assertThat(policy.getComparator().compare(r1, r2)).isEqualTo(-1); +Assert.assertEquals("The rhs entity has a larger serial, the comparator return " + +"value should be -1.", -1, policy.getComparator().compare(r1, r2)); } @Test @@ -63,46 +70,80 @@ public class TestFifoOrderingPolicy { schedOrder.addSchedulableEntity(msp3); //Assignment, oldest to youngest - checkSerials(schedOrder.getAssignmentIterator(IteratorSelector.EMPTY_ITERATOR_SELECTOR), new long[]{1, 2, 3}); +checkSerials(Arrays.asList(1L, 2L, 3L), schedOrder.getAssignmentIterator( +IteratorSelector.EMPTY_ITERATOR_SELECTOR)); //Preemption, youngest to oldest -checkSerials(schedOrder.getPreemptionIterator(), new long[]{3, 2, 1}); +checkSerials(Arrays.asList(3L, 2L, 1L), schedOrder.getPreemptionIterator()); } - public void checkSerials(
[hadoop] branch trunk updated: YARN-10918. Simplify method: CapacitySchedulerQueueManager#parseQueue. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new db8ae4b YARN-10918. Simplify method: CapacitySchedulerQueueManager#parseQueue. Contributed by Andras Gyori db8ae4b is described below commit db8ae4b65448c506c9234641b2c1f9b8e894dc18 Author: Szilard Nemeth AuthorDate: Wed Mar 9 19:36:09 2022 +0100 YARN-10918. Simplify method: CapacitySchedulerQueueManager#parseQueue. Contributed by Andras Gyori --- .../scheduler/capacity/CSQueue.java| 7 ++ .../capacity/CapacitySchedulerQueueManager.java| 116 - .../scheduler/capacity/PlanQueue.java | 19 .../scheduler/capacity/ReservationQueue.java | 6 ++ 4 files changed, 74 insertions(+), 74 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index 90cb4f3..e2aeaab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -95,6 +95,13 @@ public interface CSQueue extends SchedulerQueue { */ QueuePath getQueuePathObject(); + /** + * Checks whether the queue is a dynamic queue (created dynamically in the fashion of auto queue + * creation v2). + * @return true, if it is a dynamic queue, false otherwise + */ + boolean isDynamicQueue(); + public PrivilegedEntity getPrivilegedEntity(); Resource getMaximumAllocation(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index 50e8d52..29e5de0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -43,13 +43,11 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.Permission; import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueueManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager; import org.apache.hadoop.classification.VisibleForTesting; @@ -231,99 +229,62 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager< * @throws IOException */ static CSQueue parseQueue( - CapacitySchedulerQueueContext queueContext, - CapacitySchedulerConfiguration conf, - CSQueue parent, String queueName, - CSQueueStore newQueues, - CSQueueStore oldQueues, + CapacitySchedulerQueueContext queueContext, CapacitySchedulerConfiguration conf, + CSQueue parent, String queueName, CSQueueStore newQueues, CSQueueStore oldQueues, QueueHook hook) throws IOException { CSQueue queue; -String fullQueueName = (parent == null) ? -queueName : -(parent.getQueuePath() + "." + queueName); +String fullQueueName = (parent == null) ? queueName : +(QueuePath.createFromQueues(parent.getQueuePath(), queueName).getFullPath()); String[] staticChildQueueNames = conf.getQueues(fullQueueName); List ch
[hadoop] branch trunk updated: YARN-10945. Add javadoc to all methods of AbstractCSQueue. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 2ece950 YARN-10945. Add javadoc to all methods of AbstractCSQueue. Contributed by Andras Gyori 2ece950 is described below commit 2ece95064b9bf11ec6a165baf6de7ced4986747f Author: Szilard Nemeth AuthorDate: Wed Mar 9 19:29:53 2022 +0100 YARN-10945. Add javadoc to all methods of AbstractCSQueue. Contributed by Andras Gyori --- .../scheduler/capacity/AbstractCSQueue.java| 85 +- 1 file changed, 81 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index b0ab336..87fa9da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -77,6 +77,10 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NO_LABEL; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; +/** + * Provides implementation of {@code CSQueue} methods common for every queue class in Capacity + * Scheduler. + */ public abstract class AbstractCSQueue implements CSQueue { private static final Logger LOG = LoggerFactory.getLogger(AbstractCSQueue.class); @@ -106,11 +110,9 @@ public abstract class AbstractCSQueue implements CSQueue { CSQueueUsageTracker usageTracker; public enum CapacityConfigType { -// FIXME, from what I can see, Percentage mode can almost apply to weighted -// and percentage mode at the same time, there's only small area need to be -// changed, we need to rename "PERCENTAGE" to "PERCENTAGE" and "WEIGHT" NONE, PERCENTAGE, ABSOLUTE_RESOURCE }; + protected CapacityConfigType capacityConfigType = CapacityConfigType.NONE; @@ -175,6 +177,9 @@ public abstract class AbstractCSQueue implements CSQueue { return new QueuePath(parent.getQueuePath(), queueName); } + /** + * Sets up capacity and weight values from configuration. + */ protected void setupConfigurableCapacities() { CSQueueUtils.loadCapacitiesByLabelsFromConf(queuePath, queueCapacities, queueContext.getConfiguration(), this.queueNodeLabelsSettings.getConfiguredNodeLabels()); @@ -272,6 +277,12 @@ public abstract class AbstractCSQueue implements CSQueue { return queueNodeLabelsSettings.getAccessibleNodeLabels(); } + /** + * Checks whether the user has the required permission to execute the action of {@code QueueACL}. + * @param acl the access type the user is checked for + * @param user UGI of the user + * @return true, if the user has permission, false otherwise + */ @Override public boolean hasAccess(QueueACL acl, UserGroupInformation user) { return authorizer.checkPermission( @@ -319,6 +330,11 @@ public abstract class AbstractCSQueue implements CSQueue { return this.queueNodeLabelsSettings.getDefaultLabelExpression(); } + /** + * Initialize queue properties that are based on configuration. + * @param clusterResource overall resource of the cluster + * @throws IOException if configuration is set in a way that is inconsistent + */ protected void setupQueueConfigs(Resource clusterResource) throws IOException { @@ -460,6 +476,11 @@ public abstract class AbstractCSQueue implements CSQueue { } } + /** + * Initializes configured minimum and maximum capacity from configuration, if capacity is defined + * in ABSOLUTE node. + * @param clusterResource overall resource of the cluster + */ protected void updateConfigurableResourceLimits(Resource clusterResource) { for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { final Resource minResource = getMinimumAbsoluteResource(getQueuePath(), label); @@ -647,6 +668,13 @@ public abstract class AbstractCSQueue implements CSQueue { return queueAllocationSettings.getMinimumAllocation(); } + /** + * Increments resource usage of the queue and all related statistics and metrics that depends on + * it. + * @param clusterResource overall cluster resource + * @param resource resour
[hadoop] branch trunk updated: YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 379baa5 YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori 379baa5 is described below commit 379baa5eb65a65df50461f337b88ff13f2134aeb Author: Szilard Nemeth AuthorDate: Thu Mar 3 16:44:12 2022 +0100 YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori --- .../scheduler/capacity/AbstractCSQueue.java| 56 + .../scheduler/capacity/CSQueueUtils.java | 14 +--- .../scheduler/capacity/QueueStateHelper.java | 98 ++ 3 files changed, 101 insertions(+), 67 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index e924932..b0ab336 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -349,7 +349,7 @@ public abstract class AbstractCSQueue implements CSQueue { // Initialize the queue state based on previous state, configured state // and its parent state - initializeQueueState(); + QueueStateHelper.setQueueState(this); authorizer = YarnAuthorizationProvider.getInstance(configuration); @@ -553,60 +553,6 @@ public abstract class AbstractCSQueue implements CSQueue { return configuredCapacityVectors.get(label); } - private void initializeQueueState() { -QueueState previousState = getState(); -QueueState configuredState = queueContext.getConfiguration() -.getConfiguredState(getQueuePath()); -QueueState parentState = (parent == null) ? null : parent.getState(); - -// verify that we can not any value for State other than RUNNING/STOPPED -if (configuredState != null && configuredState != QueueState.RUNNING -&& configuredState != QueueState.STOPPED) { - throw new IllegalArgumentException("Invalid queue state configuration." - + " We can only use RUNNING or STOPPED."); -} -// If we did not set state in configuration, use Running as default state -QueueState defaultState = QueueState.RUNNING; - -if (previousState == null) { - // If current state of the queue is null, we would inherit the state - // from its parent. If this queue does not has parent, such as root queue, - // we would use the configured state. - if (parentState == null) { -updateQueueState((configuredState == null) ? defaultState -: configuredState); - } else { -if (configuredState == null) { - updateQueueState((parentState == QueueState.DRAINING) ? - QueueState.STOPPED : parentState); -} else if (configuredState == QueueState.RUNNING -&& parentState != QueueState.RUNNING) { - throw new IllegalArgumentException( - "The parent queue:" + parent.getQueuePath() - + " cannot be STOPPED as the child queue:" + getQueuePath() - + " is in RUNNING state."); -} else { - updateQueueState(configuredState); -} - } -} else { - // when we get a refreshQueue request from AdminService, - if (previousState == QueueState.RUNNING) { -if (configuredState == QueueState.STOPPED) { - stopQueue(); -} - } else { -if (configuredState == QueueState.RUNNING) { - try { -activateQueue(); - } catch (YarnException ex) { -throw new IllegalArgumentException(ex.getMessage()); - } -} - } -} - } - protected QueueInfo getQueueInfo() { // Deliberately doesn't use lock here, because this method will be invoked // from schedulerApplicationAttempt, to avoid deadlock, sacrifice diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue
[hadoop] branch trunk updated: Revert "YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori"
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 1c27c69 Revert "YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori" 1c27c69 is described below commit 1c27c69f44e6ad34ad53209e92d4231f00c3ca13 Author: Szilard Nemeth AuthorDate: Thu Mar 3 16:38:56 2022 +0100 Revert "YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori" This reverts commit de526e166c349e49d657c5980901670c086d36b2. With this commit, the patch of https://issues.apache.org/jira/browse/YARN-10850 has been committed. --- .../yarn/client/api/impl/AHSv2ClientImpl.java | 4 +- .../yarn/client/api/impl/TestAHSv2ClientImpl.java | 70 -- 2 files changed, 3 insertions(+), 71 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java index 5f90930..e3653ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java @@ -137,7 +137,9 @@ public class AHSv2ClientImpl extends AHSClient { ApplicationId appId = applicationAttemptId.getApplicationId(); ApplicationReport appReport = getApplicationReport(appId); Map filters = new HashMap<>(); -filters.put("appattemptId", applicationAttemptId.toString()); +filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" + +applicationAttemptId.toString() + +"\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}"); List entities = readerClient.getContainerEntities( appId, "ALL", filters, 0, null); List containers = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java index 9664e40..f4d7a56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java @@ -22,7 +22,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; @@ -46,11 +45,8 @@ import org.junit.Before; import org.junit.Test; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; @@ -112,72 +108,6 @@ public class TestAHSv2ClientImpl { } @Test - public void testGetContainerByAppAttempt() throws IOException, YarnException { -int applicationId = 1; -ApplicationId appId = ApplicationId.newInstance(0, applicationId); - -ApplicationAttemptId appAttemptId = -ApplicationAttemptId.newInstance(appId, 1); - -ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); - -when(spyTimelineReaderClient.getContainerEntities( -appId, "ALL", -ImmutableMap.of("appattemptId", appAttemptId.toString()), -0, null)) -.thenReturn(Arrays.asList(createContainerEntity(containerId))); - -when(spyTimelineReaderClient.getApplicationEntity(appId, "ALL", null)) -.thenReturn(createApplicationTimelineEntity(appId, true, -false)); - -List containerList = client.getContainers(appAttemptId); - -assertThat(containerList.size()).isEqualTo(1); - -assertThat(containerList.get(0).getContainerId().getApplicationAttemptId() -.getApplicationId().getId()).isEqualTo(applicationId); - } - - @Test - public void testGetMultipleContainersByAppAttempt() throws IOException, YarnException { -int numContainers = 4; -int applicationId = 3; -int applicationAttemptId = 3; -ApplicationId appMultiple = ApplicationId.newInstance(0, applicationId); -ApplicationAttem
[hadoop] branch trunk updated: YARN-11042. Fix testQueueSubmitWithACLsEnabledWithQueueMapping in TestAppManager. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 356d337 YARN-11042. Fix testQueueSubmitWithACLsEnabledWithQueueMapping in TestAppManager. Contributed by Tamas Domok 356d337 is described below commit 356d337d1e5a97dc72ae209df919cfb99e0dd841 Author: Szilard Nemeth AuthorDate: Wed Mar 2 16:19:17 2022 +0100 YARN-11042. Fix testQueueSubmitWithACLsEnabledWithQueueMapping in TestAppManager. Contributed by Tamas Domok --- .../org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 87147ce..8c73a0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -331,7 +331,7 @@ public class TestAppManager extends AppManagerTestBase{ csConf.set(PREFIX + "root.test.acl_submit_applications", "test"); csConf.set(PREFIX + "root.test.acl_administer_queue", "test"); -asContext.setQueue("test"); +asContext.setQueue("oldQueue"); MockRM newMockRM = new MockRM(csConf); RMContext newMockRMContext = newMockRM.getRMContext(); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 856e483 YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok 856e483 is described below commit 856e483592040ae8df25f8978aead93004262843 Author: Szilard Nemeth AuthorDate: Wed Mar 2 16:13:35 2022 +0100 YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok --- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 9cdb579..277e29a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -142,7 +142,7 @@ Configuration |: |: | | `yarn.scheduler.capacity.maximum-applications` / `yarn.scheduler.capacity..maximum-applications` | Maximum number of applications in the system which can be concurrently active both running and pending. Limits on each queue are directly proportional to their queue capacities and user limits. This is a hard limit and any applications submitted when this limit is reached will be rejected. Default is 1. This can be set for all queues with `yarn.scheduler.capacity.maximum-a [...] | `yarn.scheduler.capacity.maximum-am-resource-percent` / `yarn.scheduler.capacity..maximum-am-resource-percent` | Maximum percent of resources in the cluster which can be used to run application masters - controls number of concurrent active applications. Limits on each queue are directly proportional to their queue capacities and user limits. Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for all queues with `yarn.scheduler.capacity.maximum-am-resource [...] -| `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity..max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.sch [...] +| `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity..max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.sch [...] You can also limit the number of parallel applications on a per user basis. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-10894. Follow up YARN-10237: fix the new test case in TestRMWebServicesCapacitySched. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 192f532 YARN-10894. Follow up YARN-10237: fix the new test case in TestRMWebServicesCapacitySched. Contributed by Tamas Domok 192f532 is described below commit 192f53283b8cbd44c9d7780cc65e68143e0cae43 Author: Szilard Nemeth AuthorDate: Wed Mar 2 16:04:38 2022 +0100 YARN-10894. Follow up YARN-10237: fix the new test case in TestRMWebServicesCapacitySched. Contributed by Tamas Domok --- .../resourcemanager/webapp/TestRMWebServicesCapacitySched.java| 8 +--- .../webapp/TestRMWebServicesSchedulerActivities.java | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index b24b7c0..2038433 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -130,7 +130,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { // Define top-level queues config.setQueues(CapacitySchedulerConfiguration.ROOT, -new String[] {"a", "b"}); +new String[] {"a", "b", "c"}); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; config.setCapacity(A, 10.5f); @@ -391,7 +391,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { assertEquals("utilization", info.getString("orderingPolicyInfo")); JSONArray arr = info.getJSONObject("queues").getJSONArray("queue"); -assertEquals("incorrect number of elements in: " + arr, 2, arr.length()); +assertEquals("incorrect number of elements in: " + arr, 3, arr.length()); // test subqueues for (int i = 0; i < arr.length(); i++) { @@ -433,6 +433,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { qi.numApplications = info.getInt("numApplications"); qi.queueName = info.getString("queueName"); qi.state = info.getString("state"); +qi.isAbsoluteResource = info.getBoolean("isAbsoluteResource"); verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity); @@ -506,7 +507,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { + " expected: " + q, qshortName.matches(info.queueName)); assertTrue("state doesn't match", (csConf.getState(q).toString()).matches(info.state)); -if (q.equals("c")) { + +if (q.equals("root.c")) { assertTrue("c queue is not configured in Absolute resource", info.isAbsoluteResource); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java index 1dd8020..3aea113 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java @@ -154,7 +154,7 @@ public class TestRMWebServicesSchedulerActivities verifyStateOfAllocations(allocation, FN_ACT_FINAL_ALLOCATION_STATE, "ALLOCATED"); verifyQueueOrder(allocation, - "root-root.a-root.b-root.b.b2-root.b.b3-root.b.b1"); + "root-root.a-root.c-root.b-root.b.b2-root.b.b3-root.b.b1"); } finally { rm.stop(); } @@ -378,7 +378,7 @@ public class TestRMWebServicesSchedulerActivities JSONObject allocations = getFirstSubNodeFromJson(json, FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS);
[hadoop] branch branch-3.3 updated: YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 3ef3c5a YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok 3ef3c5a is described below commit 3ef3c5a05be2df7ec3dcb6e546387a8b9caf057a Author: Szilard Nemeth AuthorDate: Wed Mar 2 14:45:31 2022 +0100 YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok --- .../resourcemanager/scheduler/capacity/AbstractCSQueue.java | 2 +- .../resourcemanager/webapp/dao/CapacitySchedulerInfo.java| 5 + .../webapp/dao/CapacitySchedulerQueueInfo.java | 12 +++- .../webapp/TestRMWebServicesCapacitySched.java | 4 +++- .../webapp/TestRMWebServicesForCSWithPartitions.java | 2 +- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index e7d9fa5..3870d42 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -128,7 +128,7 @@ public abstract class AbstractCSQueue implements CSQueue { // either at this level or anywhere in the queue's hierarchy. private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false; - protected enum CapacityConfigType { + public enum CapacityConfigType { NONE, PERCENTAGE, ABSOLUTE_RESOURCE }; protected CapacityConfigType capacityConfigType = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 8de7529..9b8773e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlType; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -46,6 +47,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected float usedCapacity; protected float maxCapacity; protected String queueName; + private boolean isAbsoluteResource; protected CapacitySchedulerQueueInfoList queues; protected QueueCapacitiesInfo capacities; protected CapacitySchedulerHealthInfo health; @@ -75,6 +77,9 @@ public class CapacitySchedulerInfo extends SchedulerInfo { health = new CapacitySchedulerHealthInfo(cs); maximumAllocation = new ResourceInfo(parent.getMaximumAllocation()); +isAbsoluteResource = parent.getCapacityConfigType() == +AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE; + CapacitySchedulerConfiguration conf = cs.getConfiguration(); queueAcls = new QueueAclsInfo(); for (Map.Entry e : conf diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index e38ab0d..56ac29b25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main
[hadoop] branch branch-3.3 updated: YARN-11014. YARN incorrectly validates maximum capacity resources on the validation API. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new f06f44b YARN-11014. YARN incorrectly validates maximum capacity resources on the validation API. Contributed by Benjamin Teke f06f44b is described below commit f06f44b1c2f6e7d752fa43953dd0fd600680d4ea Author: Szilard Nemeth AuthorDate: Wed Mar 2 14:23:00 2022 +0100 YARN-11014. YARN incorrectly validates maximum capacity resources on the validation API. Contributed by Benjamin Teke --- .../scheduler/capacity/CapacityScheduler.java | 16 ++ .../capacity/CapacitySchedulerConfigValidator.java | 2 + .../TestCapacitySchedulerConfigValidator.java | 270 - 3 files changed, 284 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 69e775f..d0d95c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2056,6 +2056,22 @@ public class CapacityScheduler extends } } + /** + * Add node to nodeTracker. Used when validating CS configuration by instantiating a new + * CS instance. + * @param nodesToAdd node to be added + */ + public void addNodes(List nodesToAdd) { +writeLock.lock(); +try { + for (FiCaSchedulerNode node : nodesToAdd) { +nodeTracker.addNode(node); + } +} finally { + writeLock.unlock(); +} + } + private void addNode(RMNode nodeManager) { writeLock.lock(); try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java index c3b4df4..d180ffb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java @@ -42,6 +42,7 @@ public final class CapacitySchedulerConfigValidator { public static boolean validateCSConfiguration( final Configuration oldConf, final Configuration newConf, final RMContext rmContext) throws IOException { +CapacityScheduler liveScheduler = (CapacityScheduler) rmContext.getScheduler(); CapacityScheduler newCs = new CapacityScheduler(); try { //TODO: extract all the validation steps and replace reinitialize with @@ -49,6 +50,7 @@ public final class CapacitySchedulerConfigValidator { newCs.setConf(oldConf); newCs.setRMContext(rmContext); newCs.init(oldConf); + newCs.addNodes(liveScheduler.getAllNodes()); newCs.reinitialize(newConf, rmContext, true); return true; } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java index 04f4349..ad114d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java @@ -19,13 +19,23 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import org.apache.hadoop.conf.Configuration; +import
[hadoop] branch branch-3.2 updated: YARN-11014. YARN incorrectly validates maximum capacity resources on the validation API. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 9e2e1c79 YARN-11014. YARN incorrectly validates maximum capacity resources on the validation API. Contributed by Benjamin Teke 9e2e1c79 is described below commit 9e2e1c795f192789fc8393c0935f45145f455a86 Author: Szilard Nemeth AuthorDate: Wed Mar 2 14:21:36 2022 +0100 YARN-11014. YARN incorrectly validates maximum capacity resources on the validation API. Contributed by Benjamin Teke --- .../scheduler/capacity/CapacityScheduler.java | 16 ++ .../capacity/CapacitySchedulerConfigValidator.java | 4 +- .../TestCapacitySchedulerConfigValidator.java | 270 - 3 files changed, 285 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 1455728..70cce75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -1994,6 +1994,22 @@ public class CapacityScheduler extends } } + /** + * Add node to nodeTracker. Used when validating CS configuration by instantiating a new + * CS instance. + * @param nodesToAdd node to be added + */ + public void addNodes(List nodesToAdd) { +writeLock.lock(); +try { + for (FiCaSchedulerNode node : nodesToAdd) { +nodeTracker.addNode(node); + } +} finally { + writeLock.unlock(); +} + } + private void addNode(RMNode nodeManager) { try { writeLock.lock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java index d8e1285..a162e76 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java @@ -42,7 +42,8 @@ public final class CapacitySchedulerConfigValidator { public static boolean validateCSConfiguration( final Configuration oldConf, final Configuration newConf, - final RMContext rmContext) throws IOException { + final RMContext rmContext) throws IOException { +CapacityScheduler liveScheduler = (CapacityScheduler) rmContext.getScheduler(); CapacityScheduler newCs = new CapacityScheduler(); try { //TODO: extract all the validation steps and replace reinitialize with @@ -50,6 +51,7 @@ public final class CapacitySchedulerConfigValidator { newCs.setConf(oldConf); newCs.setRMContext(rmContext); newCs.init(oldConf); + newCs.addNodes(liveScheduler.getAllNodes()); newCs.reinitialize(newConf, rmContext, true); return true; } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java index 2b35a7c..e64a241 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java @@ -18,14 +18,24 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
[hadoop] branch branch-3.1 updated: YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.1 by this push: new a3d8749 YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke a3d8749 is described below commit a3d8749b8ca5646bd7f6b272673452d3427ad9d7 Author: Szilard Nemeth AuthorDate: Tue Mar 1 18:08:38 2022 +0100 YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke --- .../resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java index 1cbe5df..abb7c3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java @@ -53,6 +53,7 @@ public abstract class YarnConfigurationStore { * audit logging and recovery. */ static class LogMutation implements Serializable { +private static final long serialVersionUID = 7754046036718906356L; private Map updates; private String user; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 4bd67d7 YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke 4bd67d7 is described below commit 4bd67d751663cc76d68d08af69bd22ac64900a92 Author: Szilard Nemeth AuthorDate: Tue Mar 1 18:06:39 2022 +0100 YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke --- .../resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java index 1d5a6ac..2e29f28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java @@ -53,6 +53,7 @@ public abstract class YarnConfigurationStore { * audit logging and recovery. */ static class LogMutation implements Serializable { +private static final long serialVersionUID = 7754046036718906356L; private Map updates; private String user; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 935619a YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke 935619a is described below commit 935619a28c0a81d99de7e46f2b8ccf5d552b0453 Author: Szilard Nemeth AuthorDate: Tue Mar 1 18:05:04 2022 +0100 YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke --- .../resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java index 4480bc3..425d63f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java @@ -53,6 +53,7 @@ public abstract class YarnConfigurationStore { * audit logging and recovery. */ public static class LogMutation implements Serializable { +private static final long serialVersionUID = 7754046036718906356L; private Map updates; private String user; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10590. Consider legacy auto queue creation absolute resource template to avoid rounding errors. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 3653754 YARN-10590. Consider legacy auto queue creation absolute resource template to avoid rounding errors. Contributed by Andras Gyori 3653754 is described below commit 365375412fe5eea82549630ee8c5598502b95caf Author: Szilard Nemeth AuthorDate: Tue Feb 22 12:26:35 2022 +0100 YARN-10590. Consider legacy auto queue creation absolute resource template to avoid rounding errors. Contributed by Andras Gyori --- .../scheduler/capacity/AutoCreatedLeafQueue.java | 23 +--- .../capacity/AutoCreatedLeafQueueConfig.java | 15 .../scheduler/capacity/ManagedParentQueue.java | 41 ++ .../GuaranteedOrZeroCapacityOverTimePolicy.java| 10 +++--- .../TestAbsoluteResourceWithAutoQueue.java | 8 ++--- .../TestCapacitySchedulerAutoCreatedQueueBase.java | 28 +-- 6 files changed, 83 insertions(+), 42 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java index 384a652..910d8de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; @@ -30,6 +31,8 @@ import java.io.IOException; import java.util.HashSet; import java.util.Set; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE; + /** * Leaf queues which are auto created by an underlying implementation of * AbstractManagedParentQueue. Eg: PlanQueue for reservations or @@ -81,14 +84,14 @@ public class AutoCreatedLeafQueue extends AbstractAutoCreatedLeafQueue { QueueCapacities capacities = leafQueueTemplate.getQueueCapacities(); //reset capacities for the leaf queue - mergeCapacities(capacities); + mergeCapacities(capacities, leafQueueTemplate.getResourceQuotas()); } finally { writeLock.unlock(); } } - public void mergeCapacities(QueueCapacities capacities) { + public void mergeCapacities(QueueCapacities capacities, QueueResourceQuotas resourceQuotas) { for ( String nodeLabel : capacities.getExistingNodeLabels()) { queueCapacities.setCapacity(nodeLabel, capacities.getCapacity(nodeLabel)); @@ -101,9 +104,19 @@ public class AutoCreatedLeafQueue extends AbstractAutoCreatedLeafQueue { Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel, queueContext.getClusterResource()); - getQueueResourceQuotas().setEffectiveMinResource(nodeLabel, - Resources.multiply(resourceByLabel, - queueCapacities.getAbsoluteCapacity(nodeLabel))); + // Update effective resource from template due to rounding errors. + // However, we need to consider deactivation as well, in which case we fall back to + // Percentage calculation (as absolute capacity will be 0, resource will be zero as well). + if (getCapacityConfigType().equals(ABSOLUTE_RESOURCE) + && queueCapacities.getAbsoluteCapacity(nodeLabel) > 0) { +getQueueResourceQuotas().setEffectiveMinResource(nodeLabel, +resourceQuotas.getConfiguredMinResource(nodeLabel)); + } else { +getQueueResourceQuotas().setEffectiveMinResource(nodeLabel, +Resources.multiply(resourceByLabel, +queueCapacities.getAbsoluteCapacity(nodeLabel))); + } + getQueueResourceQuotas().setEffectiveMaxResource(nodeLabel, Resources.multiply(resourceByLabel, queueCapacities .getAbsoluteMaximumCapacity(nodeLabel))); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/schedul
[hadoop] branch trunk updated: YARN-10944. AbstractCSQueue: Eliminate code duplication in overloaded versions of setMaxCapacity. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 0463498 YARN-10944. AbstractCSQueue: Eliminate code duplication in overloaded versions of setMaxCapacity. Contributed by Andras Gyori 0463498 is described below commit 0463498adcd3b62b83e6383cae556093d4270d66 Author: Szilard Nemeth AuthorDate: Tue Feb 22 11:55:34 2022 +0100 YARN-10944. AbstractCSQueue: Eliminate code duplication in overloaded versions of setMaxCapacity. Contributed by Andras Gyori --- .../scheduler/capacity/AbstractCSQueue.java| 32 -- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 4eb7b68..020c601 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -74,6 +74,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; +import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NO_LABEL; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; public abstract class AbstractCSQueue implements CSQueue { @@ -279,33 +280,23 @@ public abstract class AbstractCSQueue implements CSQueue { } /** - * Set maximum capacity - used only for testing. + * Set maximum capacity for empty node label. * @param maximumCapacity new max capacity */ @VisibleForTesting void setMaxCapacity(float maximumCapacity) { -writeLock.lock(); -try { - // Sanity check - CSQueueUtils.checkMaxCapacity(this.queuePath, - queueCapacities.getCapacity(), maximumCapacity); - float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity( - maximumCapacity, parent); - CSQueueUtils.checkAbsoluteCapacity(this.queuePath, - queueCapacities.getAbsoluteCapacity(), absMaxCapacity); - - queueCapacities.setMaximumCapacity(maximumCapacity); - queueCapacities.setAbsoluteMaximumCapacity(absMaxCapacity); -} finally { - writeLock.unlock(); -} +internalSetMaximumCapacity(maximumCapacity, NO_LABEL); } /** - * Set maximum capacity + * Set maximum capacity. * @param maximumCapacity new max capacity */ void setMaxCapacity(String nodeLabel, float maximumCapacity) { +internalSetMaximumCapacity(maximumCapacity, nodeLabel); + } + + private void internalSetMaximumCapacity(float maximumCapacity, String nodeLabel) { writeLock.lock(); try { // Sanity check @@ -323,7 +314,6 @@ public abstract class AbstractCSQueue implements CSQueue { } } - @Override public String getDefaultNodeLabelExpression() { return this.queueNodeLabelsSettings.getDefaultLabelExpression(); @@ -917,7 +907,7 @@ public abstract class AbstractCSQueue implements CSQueue { } private static String ensurePartition(String partition) { -return Optional.ofNullable(partition).orElse(RMNodeLabelsManager.NO_LABEL); +return Optional.ofNullable(partition).orElse(NO_LABEL); } @FunctionalInterface @@ -1016,8 +1006,8 @@ public abstract class AbstractCSQueue implements CSQueue { // Add NO_LABEL also to this list as NO_LABEL also can be granted with // resource in many general cases. -if (!nodeLabels.contains(RMNodeLabelsManager.NO_LABEL)) { - nodeLabels.add(RMNodeLabelsManager.NO_LABEL); +if (!nodeLabels.contains(NO_LABEL)) { + nodeLabels.add(NO_LABEL); } return nodeLabels; } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new de526e1 YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori de526e1 is described below commit de526e166c349e49d657c5980901670c086d36b2 Author: Szilard Nemeth AuthorDate: Thu Feb 17 21:05:01 2022 +0100 YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori --- .../yarn/client/api/impl/AHSv2ClientImpl.java | 4 +- .../yarn/client/api/impl/TestAHSv2ClientImpl.java | 70 ++ 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java index e3653ba..5f90930 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java @@ -137,9 +137,7 @@ public class AHSv2ClientImpl extends AHSClient { ApplicationId appId = applicationAttemptId.getApplicationId(); ApplicationReport appReport = getApplicationReport(appId); Map filters = new HashMap<>(); -filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" + -applicationAttemptId.toString() + -"\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}"); +filters.put("appattemptId", applicationAttemptId.toString()); List entities = readerClient.getContainerEntities( appId, "ALL", filters, 0, null); List containers = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java index f4d7a56..9664e40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java @@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; @@ -45,8 +46,11 @@ import org.junit.Before; import org.junit.Test; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -108,6 +112,72 @@ public class TestAHSv2ClientImpl { } @Test + public void testGetContainerByAppAttempt() throws IOException, YarnException { +int applicationId = 1; +ApplicationId appId = ApplicationId.newInstance(0, applicationId); + +ApplicationAttemptId appAttemptId = +ApplicationAttemptId.newInstance(appId, 1); + +ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); + +when(spyTimelineReaderClient.getContainerEntities( +appId, "ALL", +ImmutableMap.of("appattemptId", appAttemptId.toString()), +0, null)) +.thenReturn(Arrays.asList(createContainerEntity(containerId))); + +when(spyTimelineReaderClient.getApplicationEntity(appId, "ALL", null)) +.thenReturn(createApplicationTimelineEntity(appId, true, +false)); + +List containerList = client.getContainers(appAttemptId); + +assertThat(containerList.size()).isEqualTo(1); + +assertThat(containerList.get(0).getContainerId().getApplicationAttemptId() +.getApplicationId().getId()).isEqualTo(applicationId); + } + + @Test + public void testGetMultipleContainersByAppAttempt() throws IOException, YarnException { +int numContainers = 4; +int applicationId = 3; +int applicationAttemptId = 3; +ApplicationId appMultiple = ApplicationId.newInstance(0, applicationId); +ApplicationAttemptId appAttemptIdMultiple = +ApplicationAttemptId.newInstance(appMultiple, applicationAttemptId); + +List containerEntities = new ArrayList<>(); +for (int containe
[hadoop] branch trunk updated: YARN-11070. Minimum resource ratio is overridden by subsequent labels. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 7dd288c YARN-11070. Minimum resource ratio is overridden by subsequent labels. Contributed by Andras Gyori 7dd288c is described below commit 7dd288ce8ccc93acd4f182e3db4b7936402f1592 Author: Szilard Nemeth AuthorDate: Thu Feb 17 19:58:14 2022 +0100 YARN-11070. Minimum resource ratio is overridden by subsequent labels. Contributed by Andras Gyori --- .../scheduler/capacity/AbstractCSQueue.java| 2 +- .../scheduler/capacity/ParentQueue.java| 22 .../TestAbsoluteResourceConfiguration.java | 61 ++ 3 files changed, 73 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 809a860..4eb7b68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -1298,7 +1298,7 @@ public abstract class AbstractCSQueue implements CSQueue { CapacityConfigType.ABSOLUTE_RESOURCE)) { newEffectiveMinResource = createNormalizedMinResource( usageTracker.getQueueResourceQuotas().getConfiguredMinResource(label), -((ParentQueue) parent).getEffectiveMinRatioPerResource()); +((ParentQueue) parent).getEffectiveMinRatio(label)); // Max resource of a queue should be the minimum of {parent's maxResources, // this queue's maxResources}. Both parent's maxResources and this queue's diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index b77a90a..c624aab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; @@ -101,11 +102,11 @@ public class ParentQueue extends AbstractCSQueue { private AutoCreatedQueueTemplate autoCreatedQueueTemplate; - // effective min ratio per resource, it is used during updateClusterResource, - // leaf queue can use this to calculate effective resources. - // This field will not be edited, reference will point to a new immutable map - // after every time recalculation - private volatile Map effectiveMinRatioPerResource; + // A ratio of the queue's effective minimum resource and the summary of the configured + // minimum resource of its children grouped by labels and calculated for each resource names + // distinctively. + private final Map> effectiveMinResourceRatio = + new ConcurrentHashMap<>(); public ParentQueue(CapacitySchedulerQueueContext queueContext, String queueName, CSQueue parent, CSQueue old) throws IOException { @@ -1328,8 +1329,8 @@ public class ParentQueue extends AbstractCSQueue { } } -effectiveMinRatioPerResource = getEffectiveMinRatioPerResource( -configuredMinResources, numeratorForMinRatio); +effectiveMinResourceRatio.put(label, getEffectiveMinRatio( +configuredMinResources, numeratorForMinRatio)); // Update effective resources for my self; if (rootQueue) { @@ -1340,7 +1341,7 @@ public class ParentQueue extends AbstractCSQueue { } } - private Map getEffectiveMinRatioPerResource( + private Map getEffectiveMinRatio( Resource configuredMinResources, Resource numeratorForMinRatio) { Map effectiveMinRatioPerR
[hadoop] branch trunk updated: YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new ad0a1dc YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke ad0a1dc is described below commit ad0a1dc89723ed1630399203da614d6a338b1920 Author: Szilard Nemeth AuthorDate: Thu Feb 17 19:48:42 2022 +0100 YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke --- .../resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java index 4480bc3..425d63f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java @@ -53,6 +53,7 @@ public abstract class YarnConfigurationStore { * audit logging and recovery. */ public static class LogMutation implements Serializable { +private static final long serialVersionUID = 7754046036718906356L; private Map updates; private String user; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10995. Move PendingApplicationComparator from GuaranteedOrZeroCapacityOverTimePolicy. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 4c05d25 YARN-10995. Move PendingApplicationComparator from GuaranteedOrZeroCapacityOverTimePolicy. Contributed by Benjamin Teke 4c05d25 is described below commit 4c05d257ba3f3311b5bbc993f6e5e35637487d88 Author: Szilard Nemeth AuthorDate: Thu Feb 17 19:43:37 2022 +0100 YARN-10995. Move PendingApplicationComparator from GuaranteedOrZeroCapacityOverTimePolicy. Contributed by Benjamin Teke --- .../scheduler/capacity/CapacityScheduler.java | 31 + .../capacity/CapacitySchedulerContext.java | 3 +- .../capacity/CapacitySchedulerQueueContext.java| 7 ++--- .../GuaranteedOrZeroCapacityOverTimePolicy.java| 32 +- 4 files changed, 35 insertions(+), 38 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 16c18bc..44e80a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -201,6 +202,9 @@ public class CapacityScheduler extends private int threadNum = 0; + private final PendingApplicationComparator applicationComparator = + new PendingApplicationComparator(); + @Override public void setConf(Configuration conf) { yarnConf = conf; @@ -3555,4 +3559,31 @@ public class CapacityScheduler extends return asyncSchedulerThreads; } } + + @Override + public PendingApplicationComparator getPendingApplicationComparator(){ +return applicationComparator; + } + + /** + * Comparator that orders applications by their submit time. + */ + class PendingApplicationComparator + implements Comparator { + +@Override +public int compare(FiCaSchedulerApp app1, FiCaSchedulerApp app2) { + RMApp rmApp1 = rmContext.getRMApps().get(app1.getApplicationId()); + RMApp rmApp2 = rmContext.getRMApps().get(app2.getApplicationId()); + if (rmApp1 != null && rmApp2 != null) { +return Long.compare(rmApp1.getSubmitTime(), rmApp2.getSubmitTime()); + } else if (rmApp1 != null) { +return -1; + } else if (rmApp2 != null) { +return 1; + } else{ +return 0; + } +} + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java index 1d0600f..5795a7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java @@ -18,8 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import java.util.Comparator; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.NodeId; @@ -104,4 +102,5 @@ public interface CapacitySchedulerContext { */ Clock getClock(); + CapacityScheduler.PendingApplicationComparator getPendingApplicationComparator(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/s
[hadoop] branch trunk updated: YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 51001fc YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok 51001fc is described below commit 51001fc7bf3bf2b83aaa5f4d2b4af0397ad5dc4c Author: Szilard Nemeth AuthorDate: Thu Feb 17 19:38:18 2022 +0100 YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok --- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index a945e14..9173cb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -151,7 +151,7 @@ Configuration |: |: | | `yarn.scheduler.capacity.maximum-applications` / `yarn.scheduler.capacity..maximum-applications` | Maximum number of applications in the system which can be concurrently active both running and pending. Limits on each queue are directly proportional to their queue capacities and user limits. This is a hard limit and any applications submitted when this limit is reached will be rejected. Default is 1. This can be set for all queues with `yarn.scheduler.capacity.maximum-a [...] | `yarn.scheduler.capacity.maximum-am-resource-percent` / `yarn.scheduler.capacity..maximum-am-resource-percent` | Maximum percent of resources in the cluster which can be used to run application masters - controls number of concurrent active applications. Limits on each queue are directly proportional to their queue capacities and user limits. Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for all queues with `yarn.scheduler.capacity.maximum-am-resource [...] -| `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity..max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.sch [...] +| `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity..max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.sch [...] You can also limit the number of parallel applications on a per user basis. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10580. Fix some issues in TestRMWebServicesCapacitySchedDynamicConfig. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 48e02f7 YARN-10580. Fix some issues in TestRMWebServicesCapacitySchedDynamicConfig. Contributed by Tamas Domok 48e02f7 is described below commit 48e02f79983b32b64d1ae10e8ddf162c06f700d6 Author: Szilard Nemeth AuthorDate: Thu Feb 17 19:17:31 2022 +0100 YARN-10580. Fix some issues in TestRMWebServicesCapacitySchedDynamicConfig. Contributed by Tamas Domok --- .../TestRMWebServicesCapacitySchedDynamicConfig.java | 16 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index df4f18e..0e2ecd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -102,8 +102,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends throws Exception { Configuration config = CSConfigGenerator .createPercentageConfig(); -config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, -YarnConfiguration.MEMORY_CONFIGURATION_STORE); initResourceManager(config); @@ -121,8 +119,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends throws Exception { Configuration config = CSConfigGenerator .createPercentageConfigLegacyAutoCreation(); -config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, -YarnConfiguration.MEMORY_CONFIGURATION_STORE); initResourceManager(config); @@ -141,8 +137,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends throws Exception { Configuration config = CSConfigGenerator .createAbsoluteConfigLegacyAutoCreation(); -config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, -YarnConfiguration.MEMORY_CONFIGURATION_STORE); initResourceManager(config); initAutoQueueHandler(8192 * GB); @@ -157,8 +151,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends throws Exception { Configuration config = CSConfigGenerator .createAbsoluteConfig(); -config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, -YarnConfiguration.MEMORY_CONFIGURATION_STORE); initResourceManager(config); @@ -176,8 +168,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends throws Exception { Configuration config = CSConfigGenerator .createWeightConfig(); -config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, -YarnConfiguration.MEMORY_CONFIGURATION_STORE); initResourceManager(config); @@ -197,8 +187,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends throws Exception { Configuration config = CSConfigGenerator .createWeightConfigWithAutoQueueCreationEnabled(); -config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, -YarnConfiguration.MEMORY_CONFIGURATION_STORE); config.setInt(CapacitySchedulerConfiguration .getQueuePrefix("root.autoParent1") + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + @@ -332,6 +320,10 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends for (Map.Entry entry : configs.entrySet()) { config.set(entry.getKey(), entry.getValue()); } + + config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.MEMORY_CONFIGURATION_STORE); + return config; } } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10894. Follow up YARN-10237: fix the new test case in TestRMWebServicesCapacitySched. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new bb1135c YARN-10894. Follow up YARN-10237: fix the new test case in TestRMWebServicesCapacitySched. Contributed by Tamas Domok bb1135c is described below commit bb1135c77c34ca5dcd8d43fca31c635b65bf3638 Author: Szilard Nemeth AuthorDate: Thu Feb 17 19:12:51 2022 +0100 YARN-10894. Follow up YARN-10237: fix the new test case in TestRMWebServicesCapacitySched. Contributed by Tamas Domok --- .../webapp/TestRMWebServicesCapacitySched.java | 2 +- .../TestRMWebServicesSchedulerActivities.java | 4 +- .../scheduler-response-NodeLabelDefaultAPI.xml | 491 - .../scheduler-response-PerUserResources.json | 487 +++- .../webapp/scheduler-response-PerUserResources.xml | 490 +++- .../test/resources/webapp/scheduler-response.json | 487 +++- .../test/resources/webapp/scheduler-response.xml | 490 +++- 7 files changed, 2443 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 1f73870..2be9b54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -120,7 +120,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { // Define top-level queues config.setQueues(CapacitySchedulerConfiguration.ROOT, -new String[] {"a", "b"}); +new String[] {"a", "b", "c"}); final String a = CapacitySchedulerConfiguration.ROOT + ".a"; config.setCapacity(a, 10.5f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java index f864794..3fc3c26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java @@ -156,7 +156,7 @@ public class TestRMWebServicesSchedulerActivities verifyStateOfAllocations(allocation, FN_ACT_FINAL_ALLOCATION_STATE, "ALLOCATED"); verifyQueueOrder(allocation, - "root-root.a-root.b-root.b.b2-root.b.b3-root.b.b1"); + "root-root.a-root.c-root.b-root.b.b2-root.b.b3-root.b.b1"); } finally { rm.stop(); } @@ -380,7 +380,7 @@ public class TestRMWebServicesSchedulerActivities JSONObject allocations = getFirstSubNodeFromJson(json, FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS); verifyQueueOrder(allocations, - "root-root.a-root.b-root.b.b3-root.b.b1"); + "root-root.c-root.a-root.b-root.b.b3-root.b.b1"); verifyStateOfAllocations(allocations, FN_ACT_FINAL_ALLOCATION_STATE, "RESERVED"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml index 48b6893..d49491f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml @@ -10,6 +10,495 @@ 2147483647 false + +root.c +0.0 +0.0 +100.0 +
[hadoop] branch trunk updated: YARN-11071. AutoCreatedQueueTemplate incorrect wildcard level. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new d58c10c YARN-11071. AutoCreatedQueueTemplate incorrect wildcard level. Contributed by Tamas Domok d58c10c is described below commit d58c10c3a5f0d6c923d8a49457ce5a73a0ec7011 Author: Szilard Nemeth AuthorDate: Thu Feb 17 19:03:44 2022 +0100 YARN-11071. AutoCreatedQueueTemplate incorrect wildcard level. Contributed by Tamas Domok --- .../scheduler/capacity/AutoCreatedQueueTemplate.java | 3 +-- .../scheduler/capacity/TestAutoCreatedQueueTemplate.java | 11 +++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java index 1603b19..8fa1e13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java @@ -170,8 +170,7 @@ public class AutoCreatedQueueTemplate { int wildcardLevel = 0; // root can not be wildcarded // MAX_WILDCARD_LEVEL will be configurable in the future -int supportedWildcardLevel = Math.min(queuePathMaxIndex - 1, -MAX_WILDCARD_LEVEL); +int supportedWildcardLevel = Math.min(queuePathMaxIndex, MAX_WILDCARD_LEVEL); // Allow root to have template properties if (queuePath.isRoot()) { supportedWildcardLevel = 0; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java index 5b58feb..e73001b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java @@ -89,6 +89,17 @@ public class TestAutoCreatedQueueTemplate { } @Test + public void testWildcardAfterRoot() { +conf.set(getTemplateKey("root.*", "acl_submit_applications"), "user"); +AutoCreatedQueueTemplate template = +new AutoCreatedQueueTemplate(conf, new QueuePath("root.a")); +template.setTemplateEntriesForChild(conf, "root.a"); + +Assert.assertEquals("acl_submit_applications is set", "user", +template.getTemplateProperties().get("acl_submit_applications")); + } + + @Test public void testTemplatePrecedence() { conf.set(getTemplateKey("root.a.b", "capacity"), "6w"); conf.set(getTemplateKey("root.a.*", "capacity"), "4w"); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: YARN-10503. Support queue capacity in terms of absolute resources with custom resourceType. Contributed by Qi Zhu.
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 020acb8 YARN-10503. Support queue capacity in terms of absolute resources with custom resourceType. Contributed by Qi Zhu. 020acb8 is described below commit 020acb8d39fbc4800e4c6714195c6ec7733ec90c Author: Szilard Nemeth AuthorDate: Mon Dec 20 17:59:06 2021 +0100 YARN-10503. Support queue capacity in terms of absolute resources with custom resourceType. Contributed by Qi Zhu. --- .../hadoop/yarn/util/resource/ResourceUtils.java | 15 .../capacity/CapacitySchedulerConfiguration.java | 25 +-- .../capacity/TestCSAllocateCustomResource.java | 82 ++ .../src/test/resources/resource-types-test.xml | 2 +- 4 files changed, 118 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 5061604..790e754 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -766,4 +766,19 @@ public class ResourceUtils { return info; } + + public static StringBuilder + getCustomResourcesStrings(Resource resource) { +StringBuilder res = new StringBuilder(); +if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) { + ResourceInformation[] resources = + resource.getResources(); + for (int i = 2; i < resources.length; i++) { +ResourceInformation resInfo = resources[i]; +res.append("," ++ resInfo.getName() + "=" + resInfo.getValue()); + } +} +return res; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 06b56b1..59d6767 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -2096,11 +2096,14 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur } StringBuilder resourceString = new StringBuilder(); + resourceString .append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "=" + resource.getMemorySize() + "," + AbsoluteResourceType.VCORES.toString().toLowerCase() + "=" -+ resource.getVirtualCores() + "]"); ++ resource.getVirtualCores() ++ ResourceUtils. +getCustomResourcesStrings(resource) + "]"); String prefix = getQueuePrefix(queue) + type; if (!label.isEmpty()) { @@ -2160,8 +2163,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur private void updateResourceValuesFromConfig(Set resourceTypes, Resource resource, String[] splits) { +String resourceName = splits[0].trim(); + // If key is not a valid type, skip it. -if (!resourceTypes.contains(splits[0])) { +if (!resourceTypes.contains(resourceName) +&& !ResourceUtils.getResourceTypes().containsKey(resourceName)) { + LOG.error(resourceName + " not supported."); return; } @@ -2174,9 +2181,17 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur resourceValue = UnitsConversionUtil.convert(units, "Mi", resourceValue); } +// Custom resource type defined by user. +// Such as GPU FPGA etc. +if (!resourceTypes.contains(resourceName)) { + resource.setResourceInformation(resourceName, ResourceInformation + .newInstance(resourceName, units, resourceValue)); + return; +} + // map it based on key. AbsoluteResourceType resType = AbsoluteResourceType -.valueOf(StringUtils.toUpperCase(splits[0].trim())); +.valueOf(StringUtils.toUpperCase(resourceName)); swit
[hadoop] branch branch-3.2 updated: YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 01de8fd YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke 01de8fd is described below commit 01de8fd9d650a41aba850043f78e565de759a83e Author: Szilard Nemeth AuthorDate: Fri Dec 17 14:53:06 2021 +0100 YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke --- .../monitor/ContainersMonitorImpl.java | 8 .../MockCPUResourceCalculatorProcessTree.java | 10 + ...> MockMemoryResourceCalculatorProcessTree.java} | 47 +++--- .../monitor/MockResourceCalculatorProcessTree.java | 6 +++ .../TestContainersMonitorResourceChange.java | 24 --- 5 files changed, 75 insertions(+), 20 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index a7bf73f..d7da7d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -540,6 +540,14 @@ public class ContainersMonitorImpl extends AbstractService implements pTree.updateProcessTree();// update process-tree long currentVmemUsage = pTree.getVirtualMemorySize(); long currentPmemUsage = pTree.getRssMemorySize(); +if (currentVmemUsage < 0 || currentPmemUsage < 0) { + // YARN-6862/YARN-5021 If the container just exited or for + // another reason the physical/virtual memory is UNAVAILABLE (-1) + // the values shouldn't be aggregated. + LOG.info("Skipping monitoring container {} because " + + "memory usage is not available.", containerId); + continue; +} // if machine has 6 cores and 3 are used, // cpuUsagePercentPerCore should be 300% diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java index eb35c91..49161f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java @@ -57,6 +57,16 @@ public class MockCPUResourceCalculatorProcessTree } @Override + public long getVirtualMemorySize(int olderThanAge) { +return 0; + } + + @Override + public long getRssMemorySize(int olderThanAge) { +return 0; + } + + @Override public float getCpuUsagePercent() { long cpu = this.cpuPercentage; // First getter call will be returned with -1, and other calls will diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java similarity index 50% copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java index eb35c91..ea45ac4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yar
[hadoop] branch branch-3.3 updated: YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 357423b YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke 357423b is described below commit 357423b57a93249aebd635eb461ba1cff63c7353 Author: Szilard Nemeth AuthorDate: Fri Dec 17 14:51:41 2021 +0100 YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke --- .../monitor/ContainersMonitorImpl.java | 8 .../MockCPUResourceCalculatorProcessTree.java | 10 + ...> MockMemoryResourceCalculatorProcessTree.java} | 47 +++--- .../monitor/MockResourceCalculatorProcessTree.java | 6 +++ .../TestContainersMonitorResourceChange.java | 24 --- 5 files changed, 75 insertions(+), 20 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index a83ae3a..deccded 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -537,6 +537,14 @@ public class ContainersMonitorImpl extends AbstractService implements pTree.updateProcessTree();// update process-tree long currentVmemUsage = pTree.getVirtualMemorySize(); long currentPmemUsage = pTree.getRssMemorySize(); +if (currentVmemUsage < 0 || currentPmemUsage < 0) { + // YARN-6862/YARN-5021 If the container just exited or for + // another reason the physical/virtual memory is UNAVAILABLE (-1) + // the values shouldn't be aggregated. + LOG.info("Skipping monitoring container {} because " + + "memory usage is not available.", containerId); + continue; +} // if machine has 6 cores and 3 are used, // cpuUsagePercentPerCore should be 300% diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java index eb35c91..49161f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java @@ -57,6 +57,16 @@ public class MockCPUResourceCalculatorProcessTree } @Override + public long getVirtualMemorySize(int olderThanAge) { +return 0; + } + + @Override + public long getRssMemorySize(int olderThanAge) { +return 0; + } + + @Override public float getCpuUsagePercent() { long cpu = this.cpuPercentage; // First getter call will be returned with -1, and other calls will diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java similarity index 50% copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java index eb35c91..ea45ac4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yar
[hadoop] branch trunk updated: YARN-11044. Fix TestApplicationLimits.testLimitsComputation() ineffective asserts. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 6ea1685 YARN-11044. Fix TestApplicationLimits.testLimitsComputation() ineffective asserts. Contributed by Benjamin Teke 6ea1685 is described below commit 6ea1685abd879301ee5aa13942028e55c5ec90f8 Author: Szilard Nemeth AuthorDate: Thu Dec 16 00:01:09 2021 +0100 YARN-11044. Fix TestApplicationLimits.testLimitsComputation() ineffective asserts. Contributed by Benjamin Teke --- .../scheduler/capacity/TestApplicationLimits.java | 15 --- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 7cb0ccd..f06631d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -267,6 +267,8 @@ public class TestApplicationLimits { @Test public void testLimitsComputation() throws Exception { +final float epsilon = 1e-5f; + CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(csConf); @@ -340,10 +342,9 @@ public class TestApplicationLimits { assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser()); // should default to global setting if per queue setting not set -assertEquals((long) - CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, -(long)csConf.getMaximumApplicationMasterResourcePerQueuePercent( -queue.getQueuePath())); + assertEquals(CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, +csConf.getMaximumApplicationMasterResourcePerQueuePercent( +queue.getQueuePath()), epsilon); // Change the per-queue max AM resources percentage. csConf.setFloat(PREFIX + queue.getQueuePath() @@ -361,9 +362,9 @@ public class TestApplicationLimits { queue = (LeafQueue)queues.get(A); -assertEquals((long) 0.5, -(long) csConf.getMaximumApplicationMasterResourcePerQueuePercent( - queue.getQueuePath())); +assertEquals(0.5f, +csConf.getMaximumApplicationMasterResourcePerQueuePercent( + queue.getQueuePath()), epsilon); assertThat(queue.calculateAndGetAMResourceLimit()).isEqualTo( Resource.newInstance(800 * GB, 1)); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: Clean up checkstyle warnings from YARN-11024/10907/10929. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 843f66f Clean up checkstyle warnings from YARN-11024/10907/10929. Contributed by Benjamin Teke 843f66f is described below commit 843f66f4dc012dff402bfebc183d46673cd47419 Author: Szilard Nemeth AuthorDate: Tue Dec 14 22:00:43 2021 +0100 Clean up checkstyle warnings from YARN-11024/10907/10929. Contributed by Benjamin Teke --- .../scheduler/capacity/AbstractCSQueue.java| 3 +- .../scheduler/capacity/AbstractLeafQueue.java | 38 +- .../GuaranteedOrZeroCapacityOverTimePolicy.java| 1 - .../scheduler/capacity/TestApplicationLimits.java | 35 +--- .../capacity/TestApplicationLimitsByPartition.java | 1 - .../scheduler/capacity/TestReservations.java | 1 - 6 files changed, 33 insertions(+), 46 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 3a0e2ae..809a860 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -145,7 +145,8 @@ public abstract class AbstractCSQueue implements CSQueue { CSQueueMetrics metrics = old != null ? (CSQueueMetrics) old.getMetrics() : CSQueueMetrics.forQueue(getQueuePath(), parent, -queueContext.getConfiguration().getEnableUserMetrics(), queueContext.getConfiguration()); +queueContext.getConfiguration().getEnableUserMetrics(), +queueContext.getConfiguration()); this.usageTracker = new CSQueueUsageTracker(metrics); this.queueCapacities = new QueueCapacities(parent == null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java index 8b31241..e194800 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java @@ -453,8 +453,7 @@ public class AbstractLeafQueue extends AbstractCSQueue { } @Override - public List - getQueueUserAclInfo(UserGroupInformation user) { + public List getQueueUserAclInfo(UserGroupInformation user) { readLock.lock(); try { QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance( @@ -527,8 +526,8 @@ public class AbstractLeafQueue extends AbstractCSQueue { } @Override - public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) throws - IOException { + public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) + throws IOException { writeLock.lock(); try { @@ -634,7 +633,8 @@ public class AbstractLeafQueue extends AbstractCSQueue { // Check submission limits for queues //TODO recalculate max applications because they can depend on capacity - if (getNumApplications() >= getMaxApplications() && !(this instanceof AutoCreatedLeafQueue)) { + if (getNumApplications() >= getMaxApplications() && + !(this instanceof AutoCreatedLeafQueue)) { String msg = "Queue " + getQueuePath() + " already has " + getNumApplications() + " applications," @@ -646,7 +646,8 @@ public class AbstractLeafQueue extends AbstractCSQueue { // Check submission limits for the user on this queue User user = usersManager.getUserAndAddIfAbsent(userName); //TODO recalculate max applications because they can depend on capacity - if (user.getTotalApplications() >= getMaxApplicationsPerUser() && !(this instanceof AutoCreatedLeafQueue)) { + if (user.getTotalApplications() >= getMaxApplicat
[hadoop] branch trunk updated: YARN-10929. Do not use a separate config in legacy CS AQC. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 364d38f YARN-10929. Do not use a separate config in legacy CS AQC. Contributed by Benjamin Teke 364d38f is described below commit 364d38f00d75e06099ef030030fca7ec90a8a817 Author: Szilard Nemeth AuthorDate: Tue Dec 14 13:41:22 2021 +0100 YARN-10929. Do not use a separate config in legacy CS AQC. Contributed by Benjamin Teke --- .../scheduler/capacity/AbstractCSQueue.java| 44 ++ .../scheduler/capacity/AbstractLeafQueue.java | 69 +- .../capacity/AbstractManagedParentQueue.java | 17 +- .../scheduler/capacity/AutoCreatedLeafQueue.java | 10 ++-- .../capacity/CSQueuePreemptionSettings.java| 19 +++--- .../capacity/CapacitySchedulerQueueContext.java| 4 ++ .../scheduler/capacity/LeafQueue.java | 2 +- .../scheduler/capacity/ManagedParentQueue.java | 28 +++-- .../scheduler/capacity/ParentQueue.java| 10 ++-- .../scheduler/capacity/PlanQueue.java | 2 +- .../capacity/QueueAllocationSettings.java | 11 +--- .../scheduler/capacity/ReservationQueue.java | 6 +- .../capacity/TestCapacitySchedulerPerf.java| 2 +- 13 files changed, 84 insertions(+), 140 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 5040b02..3a0e2ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -173,10 +173,9 @@ public abstract class AbstractCSQueue implements CSQueue { return new QueuePath(parent.getQueuePath(), queueName); } - protected void setupConfigurableCapacities( - CapacitySchedulerConfiguration configuration) { + protected void setupConfigurableCapacities() { CSQueueUtils.loadCapacitiesByLabelsFromConf(queuePath, queueCapacities, -configuration, this.queueNodeLabelsSettings.getConfiguredNodeLabels()); +queueContext.getConfiguration(), this.queueNodeLabelsSettings.getConfiguredNodeLabels()); } @Override @@ -329,14 +328,14 @@ public abstract class AbstractCSQueue implements CSQueue { return this.queueNodeLabelsSettings.getDefaultLabelExpression(); } - protected void setupQueueConfigs(Resource clusterResource, - CapacitySchedulerConfiguration configuration) throws + protected void setupQueueConfigs(Resource clusterResource) throws IOException { writeLock.lock(); try { + CapacitySchedulerConfiguration configuration = queueContext.getConfiguration(); if (isDynamicQueue() || this instanceof AbstractAutoCreatedLeafQueue) { -setDynamicQueueProperties(configuration); +setDynamicQueueProperties(); } // Collect and set the Node label configuration @@ -344,7 +343,7 @@ public abstract class AbstractCSQueue implements CSQueue { getQueuePath(), queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues()); // Initialize the queue capacities - setupConfigurableCapacities(configuration); + setupConfigurableCapacities(); updateAbsoluteCapacities(); updateCapacityConfigType(); @@ -354,26 +353,23 @@ public abstract class AbstractCSQueue implements CSQueue { // Setup queue's maximumAllocation respecting the global // and the queue settings - // TODO remove the getConfiguration() param after the AQC configuration duplication - // removal is resolved - this.queueAllocationSettings.setupMaximumAllocation(configuration, - queueContext.getConfiguration(), getQueuePath(), + this.queueAllocationSettings.setupMaximumAllocation(configuration, getQueuePath(), parent); // Initialize the queue state based on previous state, configured state // and its parent state - initializeQueueState(configuration); + initializeQueueState(); authorizer = YarnAuthorizationProvider.getInstance(configuration); this.acls = configuration.getAcls(getQueuePath()); - this.userWeights = getUserWeightsFromHierarchy(configuration); + this.userWeights = getUserWeightsFromHier
[hadoop] branch trunk updated: YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new f544ef4 YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok f544ef4 is described below commit f544ef4a51007be24592b9b3dc91bfd73c2f7a88 Author: Szilard Nemeth AuthorDate: Fri Dec 10 15:09:53 2021 +0100 YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok --- .../webapp/dao/CapacitySchedulerInfo.java | 5 + .../webapp/dao/CapacitySchedulerQueueInfo.java | 12 +- ...estRMWebServicesCapacitySchedDynamicConfig.java | 35 +- .../TestRMWebServicesForCSWithPartitions.java | 2 +- .../webapp/scheduler-response-AbsoluteMode.json| 3 +- ...r-response-AbsoluteModeLegacyAutoCreation.json} | 825 +++-- .../scheduler-response-NodeLabelDefaultAPI.xml | 3 +- .../scheduler-response-PerUserResources.json | 3 +- .../webapp/scheduler-response-PerUserResources.xml | 3 +- .../webapp/scheduler-response-PercentageMode.json | 3 +- ...-response-PercentageModeLegacyAutoCreation.json | 3 +- .../webapp/scheduler-response-WeightMode.json | 3 +- ...onse-WeightModeWithAutoCreatedQueues-After.json | 3 +- ...nse-WeightModeWithAutoCreatedQueues-Before.json | 3 +- .../test/resources/webapp/scheduler-response.json | 3 +- .../test/resources/webapp/scheduler-response.xml | 3 +- 16 files changed, 477 insertions(+), 435 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index c09ab54..74c7c20 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -49,6 +50,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected String queueName; private String queuePath; protected int maxParallelApps; + private boolean isAbsoluteResource; protected CapacitySchedulerQueueInfoList queues; protected QueueCapacitiesInfo capacities; protected CapacitySchedulerHealthInfo health; @@ -90,6 +92,9 @@ public class CapacitySchedulerInfo extends SchedulerInfo { health = new CapacitySchedulerHealthInfo(cs); maximumAllocation = new ResourceInfo(parent.getMaximumAllocation()); +isAbsoluteResource = parent.getCapacityConfigType() == +AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE; + CapacitySchedulerConfiguration conf = cs.getConfiguration(); queueAcls = new QueueAclsInfo(); queueAcls.addAll(getSortedQueueAclInfoList(queueName, conf)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index e27054d..78b5392 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; +import
[hadoop] branch trunk updated: YARN-10982. Replace all occurences of queuePath with the new QueuePath class. Contributed by Tibor Kovacs
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new a0d8cde YARN-10982. Replace all occurences of queuePath with the new QueuePath class. Contributed by Tibor Kovacs a0d8cde is described below commit a0d8cde133a7d8719217567efaa4e66c583f790d Author: Szilard Nemeth AuthorDate: Thu Dec 9 17:51:44 2021 +0100 YARN-10982. Replace all occurences of queuePath with the new QueuePath class. Contributed by Tibor Kovacs --- .../scheduler/capacity/AbstractCSQueue.java| 25 +++--- .../capacity/AbstractManagedParentQueue.java | 2 +- .../capacity/AutoCreatedQueueTemplate.java | 14 ++-- .../scheduler/capacity/CSQueue.java| 6 ++ .../scheduler/capacity/CSQueueUtils.java | 8 +- .../capacity/CapacitySchedulerConfiguration.java | 57 +++--- .../scheduler/capacity/ManagedParentQueue.java | 9 +-- .../scheduler/capacity/ParentQueue.java| 22 +++--- .../scheduler/capacity/QueuePath.java | 23 ++ .../TestAbsoluteResourceConfiguration.java | 90 +++--- .../TestAbsoluteResourceWithAutoQueue.java | 18 ++--- .../capacity/TestAutoCreatedQueueTemplate.java | 57 +++--- .../capacity/TestCSAllocateCustomResource.java | 4 +- .../scheduler/capacity/TestCapacityScheduler.java | 9 ++- .../TestCapacitySchedulerConfigValidator.java | 35 + .../scheduler/capacity/TestParentQueue.java| 4 +- .../scheduler/capacity/TestQueuePath.java | 10 +++ .../TestRMWebServicesConfigurationMutation.java| 15 ++-- 18 files changed, 233 insertions(+), 175 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index efdfa8e..097a9df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -82,7 +82,6 @@ public abstract class AbstractCSQueue implements CSQueue { protected final QueueAllocationSettings queueAllocationSettings; volatile CSQueue parent; protected final QueuePath queuePath; - final String queueName; protected QueueNodeLabelsSettings queueNodeLabelsSettings; private volatile QueueAppLifetimeAndLimitSettings queueAppLifetimeSettings; private CSQueuePreemptionSettings preemptionSettings; @@ -143,7 +142,6 @@ public abstract class AbstractCSQueue implements CSQueue { this.labelManager = cs.getRMContext().getNodeLabelManager(); this.parent = parent; this.queuePath = createQueuePath(parent, queueName); -this.queueName = queuePath.getLeafName(); this.resourceCalculator = cs.getResourceCalculator(); this.activitiesManager = cs.getActivitiesManager(); @@ -176,7 +174,7 @@ public abstract class AbstractCSQueue implements CSQueue { protected void setupConfigurableCapacities( CapacitySchedulerConfiguration configuration) { -CSQueueUtils.loadCapacitiesByLabelsFromConf(getQueuePath(), queueCapacities, +CSQueueUtils.loadCapacitiesByLabelsFromConf(queuePath, queueCapacities, configuration, this.queueNodeLabelsSettings.getConfiguredNodeLabels()); } @@ -186,6 +184,11 @@ public abstract class AbstractCSQueue implements CSQueue { } @Override + public QueuePath getQueuePathObject() { +return this.queuePath; + } + + @Override public float getCapacity() { return queueCapacities.getCapacity(); } @@ -241,7 +244,7 @@ public abstract class AbstractCSQueue implements CSQueue { @Override public String getQueueName() { -return queueName; +return this.queuePath.getLeafName(); } @Override @@ -279,11 +282,11 @@ public abstract class AbstractCSQueue implements CSQueue { writeLock.lock(); try { // Sanity check - CSQueueUtils.checkMaxCapacity(getQueuePath(), + CSQueueUtils.checkMaxCapacity(this.queuePath, queueCapacities.getCapacity(), maximumCapacity); float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity( maximumCapacity, parent); - CSQueueUtils.checkAbsoluteCapacity(getQueuePath(), + CSQueueUtils.checkAbsoluteCapacity(this.queuePath, queueCapacities.getAbsoluteCapacity(), absMaxCapacity
[hadoop] branch branch-3.3 updated: YARN-11020. [UI2] No container is found for an application attempt with a single AM container. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 8053b0f YARN-11020. [UI2] No container is found for an application attempt with a single AM container. Contributed by Andras Gyori 8053b0f is described below commit 8053b0f2059c3ddfb80ece12c08d545a3a24eba6 Author: Szilard Nemeth AuthorDate: Thu Dec 9 13:02:55 2021 +0100 YARN-11020. [UI2] No container is found for an application attempt with a single AM container. Contributed by Andras Gyori --- .../src/main/webapp/app/serializers/yarn-jhs-container.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js index 599cf7f..8aebb71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js @@ -45,8 +45,8 @@ export default DS.JSONAPISerializer.extend({ }, normalizeArrayResponse(store, primaryModelClass, payload/*, id, requestType*/) { - -payload = payload["containerLogsInfo"] +// Handling single container and multiple containers case at the same time +payload = [].concat(payload["containerLogsInfo"]); var normalizedArrayResponse = { data: [] - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11038. Fix testQueueSubmitWithACL* tests in TestAppManager. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 66a08e9 YARN-11038. Fix testQueueSubmitWithACL* tests in TestAppManager. Contributed by Tamas Domok 66a08e9 is described below commit 66a08e933ae8f1737f8771e1316bb44c97aa6500 Author: Szilard Nemeth AuthorDate: Wed Dec 8 16:41:35 2021 +0100 YARN-11038. Fix testQueueSubmitWithACL* tests in TestAppManager. Contributed by Tamas Domok --- .../server/resourcemanager/TestAppManager.java | 129 ++--- 1 file changed, 62 insertions(+), 67 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index f916264..4a393cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -274,10 +274,41 @@ public class TestAppManager extends AppManagerTestBase{ setupDispatcher(rmContext, conf); } + private static PlacementManager createMockPlacementManager( + String userRegex, String placementQueue, String placementParentQueue + ) throws YarnException { +PlacementManager placementMgr = mock(PlacementManager.class); +doAnswer(new Answer() { + + @Override + public ApplicationPlacementContext answer(InvocationOnMock invocation) + throws Throwable { +return new ApplicationPlacementContext(placementQueue, placementParentQueue); + } + +}).when(placementMgr).placeApplication( +any(ApplicationSubmissionContext.class), +matches(userRegex), +any(Boolean.class)); + +return placementMgr; + } + + private TestRMAppManager createAppManager(RMContext context, Configuration configuration) { +ApplicationMasterService masterService = new ApplicationMasterService(context, +context.getScheduler()); + +return new TestRMAppManager(context, +new ClientToAMTokenSecretManagerInRM(), +context.getScheduler(), masterService, +new ApplicationACLsManager(configuration), configuration); + } + @Test public void testQueueSubmitWithACLsEnabledWithQueueMapping() - throws IOException, YarnException, InterruptedException { -YarnConfiguration conf = new YarnConfiguration(); + throws YarnException { +YarnConfiguration conf = new YarnConfiguration(new Configuration(false)); +conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true"); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); @@ -285,11 +316,11 @@ public class TestAppManager extends AppManagerTestBase{ CapacitySchedulerConfiguration(conf, false); csConf.set(PREFIX + "root.queues", "default,test"); -csConf.setFloat(PREFIX + "root.default.capacity", 50.0f); -csConf.setFloat(PREFIX + "root.default.maximum-capacity", 100.0f); +csConf.setCapacity("root.default", 50.0f); +csConf.setMaximumCapacity("root.default", 100.0f); -csConf.setFloat(PREFIX + "root.test.capacity", 50.0f); -csConf.setFloat(PREFIX + "root.test.maximum-capacity", 100.0f); +csConf.setCapacity("root.test", 50.0f); +csConf.setMaximumCapacity("root.test", 100.0f); csConf.set(PREFIX + "root.acl_submit_applications", " "); csConf.set(PREFIX + "root.acl_administer_queue", " "); @@ -300,55 +331,30 @@ public class TestAppManager extends AppManagerTestBase{ csConf.set(PREFIX + "root.test.acl_submit_applications", "test"); csConf.set(PREFIX + "root.test.acl_administer_queue", "test"); -csConf.set(PREFIX + "root.test.acl_submit_applications", "test"); -csConf.set(PREFIX + "root.test.acl_administer_queue", "test"); - -csConf.set(YarnConfiguration.YARN_ACL_ENABLE, "true"); - -// Setup a PlacementManager returns a new queue -PlacementManager placementMgr = mock(PlacementManager.class); -doAnswer(new Answer() { - - @Override - public ApplicationPlacementContext answer(InvocationOnMock invocation) - throws Throwable { -return new ApplicationPlacementContext("test"); - } - -}).when(placementMgr).placeAppli
[hadoop] branch trunk updated: YARN-11016. Queue weight is incorrectly reset to zero. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 1260796 YARN-11016. Queue weight is incorrectly reset to zero. Contributed by Andras Gyori 1260796 is described below commit 126079612c8b567041ec3ea466e787a932bdf635 Author: Szilard Nemeth AuthorDate: Tue Dec 7 15:55:16 2021 +0100 YARN-11016. Queue weight is incorrectly reset to zero. Contributed by Andras Gyori --- .../scheduler/capacity/QueueCapacities.java| 2 +- .../capacity/TestCapacitySchedulerWeightMode.java | 46 ++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java index 2d82966..0edf83f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java @@ -312,7 +312,7 @@ public class QueueCapacities { _set(label, CapacityType.MAX_CAP, 0); _set(label, CapacityType.ABS_CAP, 0); _set(label, CapacityType.ABS_MAX_CAP, 0); -_set(label, CapacityType.WEIGHT, 0); +_set(label, CapacityType.WEIGHT, -1); } } finally { writeLock.unlock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java index a5d80dc..115813a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java @@ -26,6 +26,7 @@ import org.apache.hadoop.util.Sets; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; @@ -38,6 +39,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; @@ -74,6 +76,28 @@ public class TestCapacitySchedulerWeightMode { return set; } + public static CapacitySchedulerConfiguration getConfigWithInheritedAccessibleNodeLabel( + Configuration config) { +CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration( +config); + +// Define top-level queues +conf.setQueues(CapacitySchedulerConfiguration.ROOT, +new String[] { "a"}); + +conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 100f); +conf.setCapacityByLabel(A, "newLabel", 100f); +conf.setAccessibleNodeLabels(A, toSet("newLabel")); +conf.setAllowZeroCapacitySum(A, true); + +// Define 2nd-level queues +conf.setQueues(A, new String[] { "a1" }); +conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100f); + +return conf; + } + + /* * Queue structure: * root (*) @@ -368,6 +392,28 @@ public class TestCapacitySchedulerWeightMode { rm.close(); } + /** + * Tests whether weight is correctly reset to
[hadoop] branch trunk updated: YARN-11006. Allow overriding user limit factor and maxAMResourcePercent with AQCv2 templates. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 7284d23 YARN-11006. Allow overriding user limit factor and maxAMResourcePercent with AQCv2 templates. Contributed by Benjamin Teke 7284d23 is described below commit 7284d234765547072ba00477128d811aa302fc3f Author: Szilard Nemeth AuthorDate: Tue Dec 7 15:43:55 2021 +0100 YARN-11006. Allow overriding user limit factor and maxAMResourcePercent with AQCv2 templates. Contributed by Benjamin Teke --- .../resourcemanager/scheduler/capacity/LeafQueue.java| 2 +- .../TestCapacitySchedulerNewQueueAutoCreation.java | 16 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index c7f42d1..4592f2a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -1707,12 +1707,12 @@ public class LeafQueue extends AbstractCSQueue { @Override protected void setDynamicQueueProperties( CapacitySchedulerConfiguration configuration) { -super.setDynamicQueueProperties(configuration); // set to -1, to disable it configuration.setUserLimitFactor(getQueuePath(), -1); // Set Max AM percentage to a higher value configuration.setMaximumApplicationMasterResourcePerQueuePercent( getQueuePath(), 1f); +super.setDynamicQueueProperties(configuration); } private void updateSchedulerHealthForCompletedContainer( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index 9b23fdd..3d40ccf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -688,11 +688,27 @@ public class TestCapacitySchedulerNewQueueAutoCreation LeafQueue a2 = createQueue("root.a.a-auto.a2"); Assert.assertEquals("weight is not set by template", 6f, a2.getQueueCapacities().getWeight(), 1e-6); +Assert.assertEquals("user limit factor should be disabled with dynamic queues", +-1f, a2.getUserLimitFactor(), 1e-6); +Assert.assertEquals("maximum AM resource percent should be 1 with dynamic queues", +1f, a2.getMaxAMResourcePerQueuePercent(), 1e-6); + +// Set the user-limit-factor and maximum-am-resource-percent via templates to ensure their +// modified defaults are indeed overridden +csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix( +"root.a.*") + "user-limit-factor", "10"); +csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix( +"root.a.*") + "maximum-am-resource-percent", "0.8"); cs.reinitialize(csConf, mockRM.getRMContext()); a2 = (LeafQueue) cs.getQueue("root.a.a-auto.a2"); Assert.assertEquals("weight is overridden", 6f, a2.getQueueCapacities().getWeight(), 1e-6); +Assert.assertEquals("user limit factor should be modified by templates", +10f, a2.getUserLimitFactor(), 1e-6); +Assert.assertEquals("maximum AM resource percent should be modified by templates", +0.8f, a2.getMaxAMResourcePerQueuePercent(), 1e-6); + csConf.setNonLabeledQueueWeight("root.a.a-auto.a2", 4f); cs.reinitialize(csConf, mockRM.getRMContext()); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11023. Extend the root QueueInfo with max-parallel-apps in CapacityScheduler. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 60440a8 YARN-11023. Extend the root QueueInfo with max-parallel-apps in CapacityScheduler. Contributed by Tamas Domok 60440a8 is described below commit 60440a836ab602b9c76efbc78c05c4621a8f7b45 Author: Szilard Nemeth AuthorDate: Tue Dec 7 15:26:07 2021 +0100 YARN-11023. Extend the root QueueInfo with max-parallel-apps in CapacityScheduler. Contributed by Tamas Domok --- .../resourcemanager/webapp/dao/CapacitySchedulerInfo.java | 2 ++ .../webapp/TestRMWebServicesCapacitySched.java| 11 +++ .../webapp/TestRMWebServicesForCSWithPartitions.java | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 79d11e2..abf53d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -50,6 +50,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected float normalizedWeight; protected String queueName; private String queuePath; + protected int maxParallelApps; protected CapacitySchedulerQueueInfoList queues; protected QueueCapacitiesInfo capacities; protected CapacitySchedulerHealthInfo health; @@ -83,6 +84,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { this.maxCapacity = max * 100; this.weight = parent.getQueueCapacities().getWeight(); this.normalizedWeight = parent.getQueueCapacities().getNormalizedWeight(); +this.maxParallelApps = parent.getMaxParallelApps(); capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), parent.getQueueResourceQuotas(), false); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index d4e7ca6..e9df9d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -270,7 +270,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { WebServicesTestUtils.getXmlFloat(element, "capacity"), WebServicesTestUtils.getXmlFloat(element, "maxCapacity"), WebServicesTestUtils.getXmlString(element, "queueName"), - WebServicesTestUtils.getXmlString(element, "queuePath")); + WebServicesTestUtils.getXmlString(element, "queuePath"), + WebServicesTestUtils.getXmlInt(element, "maxParallelApps")); NodeList children = element.getChildNodes(); for (int j = 0; j < children.getLength(); j++) { @@ -374,13 +375,14 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements in: " + info, 1, info.length()); info = info.getJSONObject("schedulerInfo"); -assertEquals("incorrect number of elements in: " + info, 22, info.length()); +assertEquals("incorrect number of elements in: " + info, 23, info.length()); verifyClusterSchedulerGeneric(info.getString("type"), (float) info.getDouble("usedCapacity"), (float) info.getDouble("capacity"), (float) info.getDouble("maxCapacity"), info.getString("queueName"), -info.getString("queuePath")); +info.getString("queuePath"), +info.getInt("maxParallelApps")); JSONObject health = info.getJSONObject("health");
[hadoop] branch trunk updated: YARN-10757. jsonschema2pojo-maven-plugin version is not defined. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 79cc7b4 YARN-10757. jsonschema2pojo-maven-plugin version is not defined. Contributed by Tamas Domok 79cc7b4 is described below commit 79cc7b46aff8155b5501acdfd1c605aaae6a6a4d Author: Szilard Nemeth AuthorDate: Tue Dec 7 14:56:55 2021 +0100 YARN-10757. jsonschema2pojo-maven-plugin version is not defined. Contributed by Tamas Domok --- pom.xml | 6 ++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index 5bdf9fd..09f1ccb 100644 --- a/pom.xml +++ b/pom.xml @@ -116,6 +116,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x 1.4.3 4.2.2 4.2.0 + 1.1.1 bash @@ -472,6 +473,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x + + org.jsonschema2pojo + jsonschema2pojo-maven-plugin + ${jsonschema2pojo-maven-plugin.version} + - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11020. [UI2] No container is found for an application attempt with a single AM container. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 7a8c2e0 YARN-11020. [UI2] No container is found for an application attempt with a single AM container. Contributed by Andras Gyori 7a8c2e0 is described below commit 7a8c2e009761b8405406e185ab6302113be1f487 Author: Szilard Nemeth AuthorDate: Tue Dec 7 08:51:03 2021 +0100 YARN-11020. [UI2] No container is found for an application attempt with a single AM container. Contributed by Andras Gyori --- .../src/main/webapp/app/serializers/yarn-jhs-container.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js index 599cf7f..8aebb71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js @@ -45,8 +45,8 @@ export default DS.JSONAPISerializer.extend({ }, normalizeArrayResponse(store, primaryModelClass, payload/*, id, requestType*/) { - -payload = payload["containerLogsInfo"] +// Handling single container and multiple containers case at the same time +payload = [].concat(payload["containerLogsInfo"]); var normalizedArrayResponse = { data: [] - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11014. YARN incorrectly validates maximum capacity resources on the validation API. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new d75f2db YARN-11014. YARN incorrectly validates maximum capacity resources on the validation API. Contributed by Benjamin Teke d75f2db is described below commit d75f2db2f952c5cb785345d912f169d0d45a491b Author: Szilard Nemeth AuthorDate: Tue Dec 7 08:42:57 2021 +0100 YARN-11014. YARN incorrectly validates maximum capacity resources on the validation API. Contributed by Benjamin Teke --- .../scheduler/capacity/CapacityScheduler.java | 16 ++ .../capacity/CapacitySchedulerConfigValidator.java | 2 + .../TestCapacitySchedulerConfigValidator.java | 270 - 3 files changed, 284 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index d2c84e7..bd1089b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2239,6 +2239,22 @@ public class CapacityScheduler extends } } + /** + * Add node to nodeTracker. Used when validating CS configuration by instantiating a new + * CS instance. + * @param nodesToAdd node to be added + */ + public void addNodes(List nodesToAdd) { +writeLock.lock(); +try { + for (FiCaSchedulerNode node : nodesToAdd) { +nodeTracker.addNode(node); + } +} finally { + writeLock.unlock(); +} + } + private void addNode(RMNode nodeManager) { writeLock.lock(); try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java index ca0d586..fd601ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java @@ -42,6 +42,7 @@ public final class CapacitySchedulerConfigValidator { public static boolean validateCSConfiguration( final Configuration oldConf, final Configuration newConf, final RMContext rmContext) throws IOException { +CapacityScheduler liveScheduler = (CapacityScheduler) rmContext.getScheduler(); CapacityScheduler newCs = new CapacityScheduler(); try { //TODO: extract all the validation steps and replace reinitialize with @@ -49,6 +50,7 @@ public final class CapacitySchedulerConfigValidator { newCs.setConf(oldConf); newCs.setRMContext(rmContext); newCs.init(oldConf); + newCs.addNodes(liveScheduler.getAllNodes()); newCs.reinitialize(newConf, rmContext, true); return true; } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java index 04f4349..ad114d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java @@ -19,13 +19,23 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import org.apache.hadoop.conf.Configuration; +import
[hadoop] branch trunk updated: YARN-11001. Add docs on removing node label mapping from a node. Contributed by Manu Zhang
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new b933f5f YARN-11001. Add docs on removing node label mapping from a node. Contributed by Manu Zhang b933f5f is described below commit b933f5f54b7cc6e46f33da3e15065ab9ebe52939 Author: Szilard Nemeth AuthorDate: Mon Nov 15 16:02:18 2021 +0100 YARN-11001. Add docs on removing node label mapping from a node. Contributed by Manu Zhang --- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md| 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md index 39d8311..32545f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md @@ -85,7 +85,8 @@ Notes: ###Add/modify node-to-labels mapping to YARN * Configuring nodes to labels mapping in **Centralized** NodeLabel setup -* Executing ```yarn rmadmin -replaceLabelsOnNode “node1[:port]=label1 node2=label2” [-failOnUnknownNodes]```. Added label1 to node1, label2 to node2. If user don’t specify port, it adds the label to all ```NodeManagers``` running on the node. If option ```-failOnUnknownNodes``` is set, this command will fail if specified nodes are unknown. +* Executing ```yarn rmadmin -replaceLabelsOnNode "node1[:port]=label1 node2=label2" [-failOnUnknownNodes]``` adds label1 to node1, label2 to node2. If user doesn’t specify port, it adds the label to all ```NodeManagers``` running on the node. If option ```-failOnUnknownNodes``` is set, this command will fail if specified nodes are unknown. +* To remove mapping from a node, use `-replaceLabelsOnNode` but do not specify any labels. For example, ```yarn rmadmin -replaceLabelsOnNode "node1"``` removes label1 from node1. * Configuring nodes to labels mapping in **Distributed** NodeLabel setup - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10997. Revisit allocation and reservation logging. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 7cb887e YARN-10997. Revisit allocation and reservation logging. Contributed by Andras Gyori 7cb887e is described below commit 7cb887e6c2bfc058bb7b7d7bde2554d261432bdb Author: Szilard Nemeth AuthorDate: Fri Nov 12 15:42:14 2021 +0100 YARN-10997. Revisit allocation and reservation logging. Contributed by Andras Gyori --- .../scheduler/capacity/CapacityScheduler.java | 2 -- .../scheduler/common/fica/FiCaSchedulerApp.java | 13 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index b8091c7..09ae99e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -3294,13 +3294,11 @@ public class CapacityScheduler extends long commitSuccess = System.nanoTime() - commitStart; CapacitySchedulerMetrics.getMetrics() .addCommitSuccess(commitSuccess); - LOG.info("Allocation proposal accepted"); isSuccess = true; } else{ long commitFailed = System.nanoTime() - commitStart; CapacitySchedulerMetrics.getMetrics() .addCommitFailure(commitFailed); - LOG.info("Failed to accept allocation proposal"); } LOG.debug("Allocation proposal accepted={}, proposal={}", isSuccess, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index a1b8205..011a254 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -357,6 +357,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { // adding NP check as this proposal could not be allocated from reserved // container in async-scheduling mode if (allocation.getAllocateFromReservedContainer() == null) { +LOG.debug("Trying to allocate from reserved container in async scheduling mode"); return false; } RMContainer fromReservedContainer = @@ -572,6 +573,8 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { if (updatePending && getOutstandingAsksCount(schedulerContainer.getSchedulerRequestKey()) <= 0) { + LOG.debug("Rejecting appliance of allocation due to existing pending allocation " + + "request for " + schedulerContainer); return false; } @@ -670,10 +673,12 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { schedulerContainer.getRmContainer().getContainer(), reReservation); - LOG.info("Reserved container=" + rmContainer.getContainerId() - + ", on node=" + schedulerContainer.getSchedulerNode() - + " with resource=" + rmContainer - .getAllocatedOrReservedResource()); + if (LOG.isDebugEnabled()) { +LOG.debug("Reserved container=" + rmContainer.getContainerId() ++ ", on node=" + schedulerContainer.getSchedulerNode() ++ " with resource=" + rmContainer +.getAllocatedOrReservedResource()); + } } } } finally { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10997. Revisit allocation and reservation logging. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 2da7364 YARN-10997. Revisit allocation and reservation logging. Contributed by Andras Gyori 2da7364 is described below commit 2da7364c7e052bb1e1f2005b187efafa10b69c2f Author: Szilard Nemeth AuthorDate: Fri Nov 12 15:42:14 2021 +0100 YARN-10997. Revisit allocation and reservation logging. Contributed by Andras Gyori --- .../scheduler/capacity/CapacityScheduler.java | 2 -- .../scheduler/common/fica/FiCaSchedulerApp.java | 13 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index b8091c7..09ae99e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -3294,13 +3294,11 @@ public class CapacityScheduler extends long commitSuccess = System.nanoTime() - commitStart; CapacitySchedulerMetrics.getMetrics() .addCommitSuccess(commitSuccess); - LOG.info("Allocation proposal accepted"); isSuccess = true; } else{ long commitFailed = System.nanoTime() - commitStart; CapacitySchedulerMetrics.getMetrics() .addCommitFailure(commitFailed); - LOG.info("Failed to accept allocation proposal"); } LOG.debug("Allocation proposal accepted={}, proposal={}", isSuccess, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index a1b8205..011a254 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -357,6 +357,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { // adding NP check as this proposal could not be allocated from reserved // container in async-scheduling mode if (allocation.getAllocateFromReservedContainer() == null) { +LOG.debug("Trying to allocate from reserved container in async scheduling mode"); return false; } RMContainer fromReservedContainer = @@ -572,6 +573,8 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { if (updatePending && getOutstandingAsksCount(schedulerContainer.getSchedulerRequestKey()) <= 0) { + LOG.debug("Rejecting appliance of allocation due to existing pending allocation " + + "request for " + schedulerContainer); return false; } @@ -670,10 +673,12 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { schedulerContainer.getRmContainer().getContainer(), reReservation); - LOG.info("Reserved container=" + rmContainer.getContainerId() - + ", on node=" + schedulerContainer.getSchedulerNode() - + " with resource=" + rmContainer - .getAllocatedOrReservedResource()); + if (LOG.isDebugEnabled()) { +LOG.debug("Reserved container=" + rmContainer.getContainerId() ++ ", on node=" + schedulerContainer.getSchedulerNode() ++ " with resource=" + rmContainer +.getAllocatedOrReservedResource()); + } } } } finally { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10996. Fix race condition of User object acquisitions. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new e220e88 YARN-10996. Fix race condition of User object acquisitions. Contributed by Andras Gyori e220e88 is described below commit e220e88eca26311af707f7969da36d402f022a8d Author: Szilard Nemeth AuthorDate: Fri Nov 12 15:33:39 2021 +0100 YARN-10996. Fix race condition of User object acquisitions. Contributed by Andras Gyori --- .../capacity/FifoIntraQueuePreemptionPlugin.java| 10 ++ .../scheduler/capacity/LeafQueue.java | 21 ++--- .../scheduler/capacity/UsersManager.java| 3 ++- .../capacity/mockframework/MockApplications.java| 1 + 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java index a9d3c05..ea17fed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java @@ -30,6 +30,7 @@ import java.util.PriorityQueue; import java.util.Set; import java.util.TreeSet; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -419,8 +420,10 @@ public class FifoIntraQueuePreemptionPlugin String userName = app.getUser(); TempUserPerPartition tmpUser = usersPerPartition.get(userName); if (tmpUser == null) { -ResourceUsage userResourceUsage = tq.leafQueue.getUser(userName) -.getResourceUsage(); +// User might have already been removed, but preemption still accounts for this app, +// therefore reinserting the user will not cause a memory leak +User user = tq.leafQueue.getOrCreateUser(userName); +ResourceUsage userResourceUsage = user.getResourceUsage(); // perUserAMUsed was populated with running apps, now we are looping // through both running and pending apps. @@ -428,8 +431,7 @@ public class FifoIntraQueuePreemptionPlugin amUsed = (userSpecificAmUsed == null) ? Resources.none() : userSpecificAmUsed; -tmpUser = new TempUserPerPartition( -tq.leafQueue.getUser(userName), tq.queueName, +tmpUser = new TempUserPerPartition(user, tq.queueName, Resources.clone(userResourceUsage.getUsed(partition)), Resources.clone(amUsed), Resources.clone(userResourceUsage.getReserved(partition)), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index a4e2a82..c7f42d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -519,6 +519,11 @@ public class LeafQueue extends AbstractCSQueue { return usersManager.getUser(userName); } + @VisibleForTesting + public User getOrCreateUser(String userName) { +return usersManager.getUserAndAddIfAbsent(userName); + } + @Private public List getPriorityACLs() { readLock.lock(); @@ -2007,7 +2012,12 @@ public class LeafQueue extends AbstractCSQueue { public void incAMUsedResource(String nodeLabel, Resource resourceToInc, SchedulerApplicationAttempt application) { -getUser(application.getUser()).getResourceUsage().incAMUsed(nodeLabel, +User user = getUser(application.getUser()); +if (user == null) { + return; +} + +user.getResourceUsage().incAMUsed(nodeLabel, resourceToInc); // ResourceUsage has its own lock, no addition lock needs here
[hadoop] branch trunk updated: YARN-10909. AbstractCSQueue: Annotate all methods with VisibleForTesting that are only used by test code. Contributed by JackWangCS, Szilard Nemeth
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 9cfd8d0 YARN-10909. AbstractCSQueue: Annotate all methods with VisibleForTesting that are only used by test code. Contributed by JackWangCS, Szilard Nemeth 9cfd8d0 is described below commit 9cfd8d0a83fac8fd54d9d744b79c8898d7adaebe Author: Jack AuthorDate: Sat Oct 23 14:47:00 2021 +0200 YARN-10909. AbstractCSQueue: Annotate all methods with VisibleForTesting that are only used by test code. Contributed by JackWangCS, Szilard Nemeth --- .../resourcemanager/scheduler/capacity/AbstractCSQueue.java | 11 +++ 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index e3feb51..e89a547 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -287,6 +287,7 @@ public abstract class AbstractCSQueue implements CSQueue { * Set maximum capacity - used only for testing. * @param maximumCapacity new max capacity */ + @VisibleForTesting void setMaxCapacity(float maximumCapacity) { writeLock.lock(); try { @@ -1023,7 +1024,8 @@ public abstract class AbstractCSQueue implements CSQueue { return getEffectiveMaxCapacity(nodePartition); } - public boolean hasChildQueues() { + @VisibleForTesting + boolean hasChildQueues() { List childQueues = getChildQueues(); return childQueues != null && !childQueues.isEmpty(); } @@ -1608,7 +1610,7 @@ public abstract class AbstractCSQueue implements CSQueue { getAutoExpiredDeletionTime()); } - public void updateLastSubmittedTimeStamp() { + void updateLastSubmittedTimeStamp() { writeLock.lock(); try { usageTracker.setLastSubmittedTimestamp(Time.monotonicNow()); @@ -1617,7 +1619,8 @@ public abstract class AbstractCSQueue implements CSQueue { } } - public long getLastSubmittedTimestamp() { + @VisibleForTesting + long getLastSubmittedTimestamp() { readLock.lock(); try { @@ -1628,7 +1631,7 @@ public abstract class AbstractCSQueue implements CSQueue { } @VisibleForTesting - public void setLastSubmittedTimestamp(long lastSubmittedTimestamp) { + void setLastSubmittedTimestamp(long lastSubmittedTimestamp) { writeLock.lock(); try { usageTracker.setLastSubmittedTimestamp(lastSubmittedTimestamp); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10930. Introduce universal capacity resource vector. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 32ecaed YARN-10930. Introduce universal capacity resource vector. Contributed by Andras Gyori 32ecaed is described below commit 32ecaed9c3c06a48ef01d0437e62e8faccd3e9f3 Author: 9uapaw AuthorDate: Fri Oct 22 17:32:33 2021 +0200 YARN-10930. Introduce universal capacity resource vector. Contributed by Andras Gyori --- .../scheduler/capacity/AbstractCSQueue.java| 12 +- .../scheduler/capacity/CSQueue.java| 8 + .../capacity/CapacitySchedulerConfiguration.java | 19 +- .../scheduler/capacity/QueueCapacityVector.java| 258 + .../scheduler/capacity/ResourceVector.java | 129 +++ .../capacity/conf/QueueCapacityConfigParser.java | 215 + .../capacity/TestQueueCapacityVector.java | 111 + .../scheduler/capacity/TestResourceVector.java | 118 ++ .../conf/TestQueueCapacityConfigParser.java| 241 +++ 9 files changed, 1108 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 2d9bf85..e3feb51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -131,6 +131,8 @@ public abstract class AbstractCSQueue implements CSQueue { protected CapacityConfigType capacityConfigType = CapacityConfigType.NONE; + protected Map configuredCapacityVectors; + private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); protected CapacitySchedulerContext csContext; @@ -374,6 +376,8 @@ public abstract class AbstractCSQueue implements CSQueue { this.reservationsContinueLooking = configuration.getReservationContinueLook(); + this.configuredCapacityVectors = csContext.getConfiguration() + .parseConfiguredResourceVector(queuePath, configuredNodeLabels); // Update metrics CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, @@ -688,6 +692,12 @@ public abstract class AbstractCSQueue implements CSQueue { minimumAllocation); } + @Override + public QueueCapacityVector getConfiguredCapacityVector( + String label) { +return configuredCapacityVectors.get(label); + } + private void initializeQueueState(CapacitySchedulerConfiguration configuration) { QueueState previousState = getState(); QueueState configuredState = configuration @@ -978,7 +988,7 @@ public abstract class AbstractCSQueue implements CSQueue { "Default lifetime " + defaultAppLifetime + " can't exceed maximum lifetime " + myMaxAppLifetime); } - + if (defaultAppLifetime <= 0) { defaultAppLifetime = myMaxAppLifetime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index 03a5afb..2acc1d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -420,6 +420,14 @@ public interface CSQueue extends SchedulerQueue { Resource getEffectiveCapacity(String label); /** + * Get configured capacity resource vector parsed from the capacity config + * of the queue. + * @param label node label (partition) + * @return capacity resource vector + */ + QueueCapacityVector getConfiguredCapacityVector(String label); + + /** * Get effective capacity of queue. If min/max resource is configured, * preference will be given to absolute configuration over normal capacity. * Also round down the result to normalizeDow
[hadoop] branch trunk updated: YARN-10948. Rename SchedulerQueue#activeQueue to activateQueue. Contributed by Adam Antal
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 23772d9 YARN-10948. Rename SchedulerQueue#activeQueue to activateQueue. Contributed by Adam Antal 23772d9 is described below commit 23772d946bdf26a72fb611a9719afe743cdebd52 Author: Adam Antal AuthorDate: Fri Oct 22 16:33:03 2021 +0200 YARN-10948. Rename SchedulerQueue#activeQueue to activateQueue. Contributed by Adam Antal --- .../yarn/server/resourcemanager/scheduler/QueueStateManager.java | 2 +- .../hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueue.java | 4 ++-- .../server/resourcemanager/scheduler/capacity/AbstractCSQueue.java| 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java index 0127e1e..ab0f445 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java @@ -77,7 +77,7 @@ public class QueueStateManager extends Queue { void stopQueue(); /** - * Active the queue. + * Activate the queue. * @throws YarnException if the queue can not be activated. */ - void activeQueue() throws YarnException; + void activateQueue() throws YarnException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index ca861fb..2d9bf85 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -733,7 +733,7 @@ public abstract class AbstractCSQueue implements CSQueue { } else { if (configuredState == QueueState.RUNNING) { try { -activeQueue(); +activateQueue(); } catch (YarnException ex) { throw new IllegalArgumentException(ex.getMessage()); } @@ -1322,7 +1322,7 @@ public abstract class AbstractCSQueue implements CSQueue { } @Override - public void activeQueue() throws YarnException { + public void activateQueue() throws YarnException { this.writeLock.lock(); try { if (getState() == QueueState.RUNNING) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10954. Remove commented code block from CSQueueUtils#loadCapacitiesByLabelsFromConf. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 616cea2 YARN-10954. Remove commented code block from CSQueueUtils#loadCapacitiesByLabelsFromConf. Contributed by Andras Gyori 616cea2 is described below commit 616cea2e8068e990d24057d2b0d6090f35e21371 Author: 9uapaw AuthorDate: Tue Oct 19 13:06:35 2021 +0200 YARN-10954. Remove commented code block from CSQueueUtils#loadCapacitiesByLabelsFromConf. Contributed by Andras Gyori --- .../server/resourcemanager/scheduler/capacity/CSQueueUtils.java | 9 - 1 file changed, 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index f43ea30..410117a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -92,15 +92,6 @@ public class CSQueueUtils { queueCapacities.setWeight(label, csConf.getLabeledQueueWeight(queuePath, label)); } - - /*float absCapacity = queueCapacities.getCapacity(label); - float absMaxCapacity = queueCapacities.getMaximumCapacity(label); - if (absCapacity > absMaxCapacity) { -throw new IllegalArgumentException("Illegal queue capacity setting " -+ "(abs-capacity=" + absCapacity + ") > (abs-maximum-capacity=" -+ absMaxCapacity + ") for queue=[" -+ queuePath + "],label=[" + label + "]"); - }*/ } } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-10869. CS considers only the default maximum-allocation-mb/vcore property as a maximum when it creates dynamic queues (#3504)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 7000458 YARN-10869. CS considers only the default maximum-allocation-mb/vcore property as a maximum when it creates dynamic queues (#3504) 7000458 is described below commit 700045896c6175913d9071c132b533871dcbc20e Author: Benjamin Teke AuthorDate: Tue Oct 12 18:05:50 2021 +0200 YARN-10869. CS considers only the default maximum-allocation-mb/vcore property as a maximum when it creates dynamic queues (#3504) Co-authored-by: Benjamin Teke --- .../scheduler/capacity/AbstractCSQueue.java| 50 -- .../capacity/CapacitySchedulerConfiguration.java | 14 ++ .../TestCapacitySchedulerAutoCreatedQueueBase.java | 32 -- .../TestCapacitySchedulerAutoQueueCreation.java| 33 +- 4 files changed, 102 insertions(+), 27 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index fda5e16..e7d9fa5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -100,7 +100,7 @@ public abstract class AbstractCSQueue implements CSQueue { String defaultLabelExpression; private String multiNodeSortingPolicyName = null; - Map acls = + Map acls = new HashMap(); volatile boolean reservationsContinueLooking; private volatile boolean preemptionDisabled; @@ -112,7 +112,7 @@ public abstract class AbstractCSQueue implements CSQueue { volatile ResourceUsage queueUsage; private final boolean fullPathQueueNamingPolicy = false; - + // Track capacities like used-capcity/abs-used-capacity/capacity/abs-capacity, // etc. QueueCapacities queueCapacities; @@ -134,7 +134,7 @@ public abstract class AbstractCSQueue implements CSQueue { protected CapacityConfigType capacityConfigType = CapacityConfigType.NONE; - private final RecordFactory recordFactory = + private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); protected CapacitySchedulerContext csContext; protected YarnAuthorizationProvider authorizer = null; @@ -250,12 +250,12 @@ public abstract class AbstractCSQueue implements CSQueue { public QueueState getState() { return state; } - + @Override public CSQueueMetrics getMetrics() { return metrics; } - + @Override public String getQueueShortName() { return queueName; @@ -283,7 +283,7 @@ public abstract class AbstractCSQueue implements CSQueue { public void setParent(CSQueue newParentQueue) { this.parent = newParentQueue; } - + public Set getAccessibleNodeLabels() { return accessibleLabels; } @@ -344,7 +344,7 @@ public abstract class AbstractCSQueue implements CSQueue { public String getDefaultNodeLabelExpression() { return defaultLabelExpression; } - + void setupQueueConfigs(Resource clusterResource) throws IOException { setupQueueConfigs(clusterResource, csContext.getConfiguration()); @@ -471,8 +471,14 @@ public abstract class AbstractCSQueue implements CSQueue { private void setupMaximumAllocation(CapacitySchedulerConfiguration csConf) { String myQueuePath = getQueuePath(); +/* YARN-10869: When using AutoCreatedLeafQueues, the passed configuration +* object is a cloned one containing only the template configs +* (see ManagedParentQueue#getLeafQueueConfigs). To ensure that the actual +* cluster maximum allocation is fetched the original config object should +* be used. +*/ Resource clusterMax = ResourceUtils -.fetchMaximumAllocationFromConfig(csConf); +.fetchMaximumAllocationFromConfig(this.csContext.getConfiguration()); Resource queueMax = csConf.getQueueMaximumAllocation(myQueuePath); maximumAllocation = Resources.clone( @@ -740,7 +746,7 @@ public abstract class AbstractCSQueue implements CSQueue { stats.setReservedContainers(getMetrics().getReservedContainers()); return stats; } - + public Map getQueueConfigurations() { Map queueConfigurations = new HashMap<>(); Set nodeLabels = getNodeLabelsForQueue(); @@ -776,12 +782,12 @@ public abstract
[hadoop] branch trunk updated: YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 8c271ca YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke 8c271ca is described below commit 8c271caa81db683fe4b92853bd8ca5da060e6593 Author: Szilard Nemeth AuthorDate: Tue Oct 12 17:32:40 2021 +0200 YARN-6862. Nodemanager resource usage metrics sometimes are negative. Contributed by Benjamin Teke --- .../monitor/ContainersMonitorImpl.java | 8 .../MockCPUResourceCalculatorProcessTree.java | 10 + ...> MockMemoryResourceCalculatorProcessTree.java} | 47 +++--- .../monitor/MockResourceCalculatorProcessTree.java | 6 +++ .../TestContainersMonitorResourceChange.java | 24 --- 5 files changed, 75 insertions(+), 20 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index a83ae3a..deccded 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -537,6 +537,14 @@ public class ContainersMonitorImpl extends AbstractService implements pTree.updateProcessTree();// update process-tree long currentVmemUsage = pTree.getVirtualMemorySize(); long currentPmemUsage = pTree.getRssMemorySize(); +if (currentVmemUsage < 0 || currentPmemUsage < 0) { + // YARN-6862/YARN-5021 If the container just exited or for + // another reason the physical/virtual memory is UNAVAILABLE (-1) + // the values shouldn't be aggregated. + LOG.info("Skipping monitoring container {} because " + + "memory usage is not available.", containerId); + continue; +} // if machine has 6 cores and 3 are used, // cpuUsagePercentPerCore should be 300% diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java index eb35c91..49161f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java @@ -57,6 +57,16 @@ public class MockCPUResourceCalculatorProcessTree } @Override + public long getVirtualMemorySize(int olderThanAge) { +return 0; + } + + @Override + public long getRssMemorySize(int olderThanAge) { +return 0; + } + + @Override public float getCpuUsagePercent() { long cpu = this.cpuPercentage; // First getter call will be returned with -1, and other calls will diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java similarity index 50% copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockMemoryResourceCalculatorProcessTree.java index eb35c91..ea45ac4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-no
[hadoop] branch trunk updated: YARN-10950. Code cleanup in QueueCapacities (#3454)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new a9b2469 YARN-10950. Code cleanup in QueueCapacities (#3454) a9b2469 is described below commit a9b2469a534c5bc554c09aaf2d460a5a00922aca Author: Adam Antal AuthorDate: Sun Sep 19 14:42:02 2021 +0200 YARN-10950. Code cleanup in QueueCapacities (#3454) --- .../scheduler/capacity/QueueCapacities.java| 51 -- 1 file changed, 18 insertions(+), 33 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java index 86d35d6..2d82966 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java @@ -33,17 +33,16 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana public class QueueCapacities { private static final String NL = CommonNodeLabelsManager.NO_LABEL; private static final float LABEL_DOESNT_EXIST_CAP = 0f; - private Map capacitiesMap; - private ReadLock readLock; - private WriteLock writeLock; + private final Map capacitiesMap; + private final ReadLock readLock; + private final WriteLock writeLock; private final boolean isRoot; public QueueCapacities(boolean isRoot) { ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); writeLock = lock.writeLock(); - -capacitiesMap = new HashMap(); +capacitiesMap = new HashMap<>(); this.isRoot = isRoot; } @@ -52,15 +51,15 @@ public class QueueCapacities { USED_CAP(0), ABS_USED_CAP(1), MAX_CAP(2), ABS_MAX_CAP(3), CAP(4), ABS_CAP(5), MAX_AM_PERC(6), RESERVED_CAP(7), ABS_RESERVED_CAP(8), WEIGHT(9), NORMALIZED_WEIGHT(10); -private int idx; +private final int idx; -private CapacityType(int idx) { +CapacityType(int idx) { this.idx = idx; } } private static class Capacities { -private float[] capacitiesArr; +private final float[] capacitiesArr; public Capacities() { capacitiesArr = new float[CapacityType.values().length]; @@ -71,19 +70,17 @@ public class QueueCapacities { @Override public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("{used=" + capacitiesArr[0] + "%, ") - .append("abs_used=" + capacitiesArr[1] + "%, ") - .append("max_cap=" + capacitiesArr[2] + "%, ") - .append("abs_max_cap=" + capacitiesArr[3] + "%, ") - .append("cap=" + capacitiesArr[4] + "%, ") - .append("abs_cap=" + capacitiesArr[5] + "%, ") - .append("max_am_perc=" + capacitiesArr[6] + "%, ") - .append("reserved_cap=" + capacitiesArr[7] + "%, ") - .append("abs_reserved_cap=" + capacitiesArr[8] + "%, ") - .append("weight=" + capacitiesArr[9] + "w, ") - .append("normalized_weight=" + capacitiesArr[10] + "w}"); - return sb.toString(); + return "{used=" + capacitiesArr[0] + "%, " + + "abs_used=" + capacitiesArr[1] + "%, " + + "max_cap=" + capacitiesArr[2] + "%, " + + "abs_max_cap=" + capacitiesArr[3] + "%, " + + "cap=" + capacitiesArr[4] + "%, " + + "abs_cap=" + capacitiesArr[5] + "%, " + + "max_am_perc=" + capacitiesArr[6] + "%, " + + "reserved_cap=" + capacitiesArr[7] + "%, " + + "abs_reserved_cap=" + capacitiesArr[8] + "%, " + + "weight=" + capacitiesArr[9] + "w, " + + "normalized_weight=" + capacitiesArr[10] + "w}"; } } @@ -118,7 +115,6 @@ public class QueueCapacities { } } - /* Used Capacity Getter and Setter */ public float getUsedCapacity() { return _get(NL, CapacityType.USED_CAP); } @@ -135,7 +131,6 @@ public class QueueCapacities { _set(la
[hadoop] branch trunk updated: YARN-10937. Fix log message arguments in LogAggregationFileController (#3450)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 5dcb23c YARN-10937. Fix log message arguments in LogAggregationFileController (#3450) 5dcb23c is described below commit 5dcb23c1f005dfa8203c156abfc3d538a67028dc Author: TiborKovacsCloudera <90398134+tiborkovacscloud...@users.noreply.github.com> AuthorDate: Sun Sep 19 14:35:34 2021 +0200 YARN-10937. Fix log message arguments in LogAggregationFileController (#3450) --- .../logaggregation/filecontroller/LogAggregationFileController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index bd159ee..3c3380c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -390,7 +390,7 @@ public abstract class LogAggregationFileController { remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS)); } catch ( UnsupportedOperationException use) { LOG.info("Unable to set permissions for configured filesystem since" - + " it does not support this", remoteFS.getScheme()); + + " it does not support this {}", remoteFS.getScheme()); fsSupportsChmod = false; } @@ -437,7 +437,7 @@ public abstract class LogAggregationFileController { remoteFS.setPermission(permissionCheckFile, new FsPermission(TLDIR_PERMISSIONS)); } catch (UnsupportedOperationException use) { LOG.info("Unable to set permissions for configured filesystem since" -+ " it does not support this", remoteFS.getScheme()); ++ " it does not support this {}", remoteFS.getScheme()); fsSupportsChmod = false; } catch (IOException e) { LOG.warn("Failed to check if FileSystem supports permissions on " - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10913. AbstractCSQueue: Group preemption methods and fields into a separate class (#3420)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new aa74a30 YARN-10913. AbstractCSQueue: Group preemption methods and fields into a separate class (#3420) aa74a30 is described below commit aa74a303ed30a057893c6d2d9fb1e07e7d1f4a7d Author: Szilard Nemeth <954799+szilard-nem...@users.noreply.github.com> AuthorDate: Sun Sep 19 13:11:56 2021 +0200 YARN-10913. AbstractCSQueue: Group preemption methods and fields into a separate class (#3420) --- .../scheduler/capacity/AbstractCSQueue.java| 100 ++--- .../scheduler/capacity/CSQueuePreemption.java | 119 + 2 files changed, 129 insertions(+), 90 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 75aebfe..f0d0639 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -36,7 +36,6 @@ import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.QueueStatistics; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -106,10 +105,6 @@ public abstract class AbstractCSQueue implements CSQueue { Map acls = new HashMap(); volatile boolean reservationsContinueLooking; - private volatile boolean preemptionDisabled; - // Indicates if the in-queue preemption setting is ever disabled within the - // hierarchy of this queue. - private boolean intraQueuePreemptionDisabledInHierarchy; // Track resource usage-by-label like used-resource/pending-resource, etc. volatile ResourceUsage queueUsage; @@ -129,6 +124,7 @@ public abstract class AbstractCSQueue implements CSQueue { // Indicates if this queue's default lifetime was set by a config property, // either at this level or anywhere in the queue's hierarchy. private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false; + private CSQueuePreemption preemptionSettings; public enum CapacityConfigType { // FIXME, from what I can see, Percentage mode can almost apply to weighted @@ -401,10 +397,7 @@ public abstract class AbstractCSQueue implements CSQueue { this, labelManager, null); // Store preemption settings - this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(this, - configuration); - this.intraQueuePreemptionDisabledInHierarchy = - isIntraQueueHierarchyPreemptionDisabled(this, configuration); + this.preemptionSettings = new CSQueuePreemption(this, csContext, configuration); this.priority = configuration.getQueuePriority( getQueuePath()); @@ -777,7 +770,7 @@ public abstract class AbstractCSQueue implements CSQueue { queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression); queueInfo.setCurrentCapacity(getUsedCapacity()); queueInfo.setQueueStatistics(getQueueStatistics()); -queueInfo.setPreemptionDisabled(preemptionDisabled); +queueInfo.setPreemptionDisabled(getPreemptionDisabled()); queueInfo.setIntraQueuePreemptionDisabled( getIntraQueuePreemptionDisabled()); queueInfo.setQueueConfigurations(getQueueConfigurations()); @@ -902,17 +895,18 @@ public abstract class AbstractCSQueue implements CSQueue { @Private public boolean getPreemptionDisabled() { -return preemptionDisabled; +return preemptionSettings.isPreemptionDisabled(); } @Private public boolean getIntraQueuePreemptionDisabled() { -return intraQueuePreemptionDisabledInHierarchy || preemptionDisabled; +return preemptionSettings.isIntraQueuePreemptionDisabledInHierarchy() || +preemptionSettings.isPreemptionDisabled(); } @Private public boolean getIntraQueuePreemptionDisabledInHierarchy() { -return intraQueuePreemptionDisabledInHierarchy; +return preemptionSettings.isIntraQueuePreemptionDisabledInHierarchy(); } @Private @@ -935,43 +929,6 @@ publi
[hadoop] branch branch-3.2 updated: YARN-10870. Missing user filtering check -> yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by Gergely Pollak
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 4066cb0 YARN-10870. Missing user filtering check -> yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by Gergely Pollak 4066cb0 is described below commit 4066cb0793b2aa49a585384dc798934ef50f8ba5 Author: Szilard Nemeth AuthorDate: Tue Sep 14 18:09:35 2021 +0200 YARN-10870. Missing user filtering check -> yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by Gergely Pollak --- .../webapp/FairSchedulerAppsBlock.java | 70 +++--- 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java index 14ad277..0ba8f1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java @@ -23,18 +23,21 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; -import java.util.Collection; -import java.util.HashSet; -import java.util.Map; +import java.security.Principal; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.commons.text.StringEscapeUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -49,6 +52,8 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import com.google.inject.Inject; +import javax.servlet.http.HttpServletRequest; + /** * Shows application information specific to the fair * scheduler as part of the fair scheduler page. @@ -58,10 +63,19 @@ public class FairSchedulerAppsBlock extends HtmlBlock { final FairSchedulerInfo fsinfo; final Configuration conf; final ResourceManager rm; + final boolean filterAppsByUser; + @Inject public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, Configuration conf) { super(ctx); +this.conf = conf; +this.rm = rm; + +this.filterAppsByUser = conf.getBoolean( +YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, +YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER); + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); fsinfo = new FairSchedulerInfo(scheduler); apps = new ConcurrentHashMap(); @@ -70,13 +84,53 @@ public class FairSchedulerAppsBlock extends HtmlBlock { if (!(RMAppState.NEW.equals(entry.getValue().getState()) || RMAppState.NEW_SAVING.equals(entry.getValue().getState()) || RMAppState.SUBMITTED.equals(entry.getValue().getState( { -apps.put(entry.getKey(), entry.getValue()); +if (!filterAppsByUser || hasAccess(entry.getValue(), +ctx.requestContext().getRequest())) { + apps.put(entry.getKey(), entry.getValue()); +} } } -this.conf = conf; -this.rm = rm; } - + + private UserGroupInformation getCallerUserGroupInformation( + HttpServletRequest hsr, boolean usePrincipal) { + +String remoteUser = hsr.getRemoteUser(); +if (usePrincipal) { + Principal princ = hsr.getUserPrincipal(); + remoteUser = princ == null ? null : princ.getName(); +} + +UserGroupInformation callerUGI = null; +if (remoteUser != null) { + callerUGI = UserGroupInformation.createRemoteUser(remoteUser); +} + +return callerUGI; + } + + protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) { +//
[hadoop] branch branch-3.3 updated: YARN-10870. Missing user filtering check -> yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by Gergely Pollak
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 6c68211 YARN-10870. Missing user filtering check -> yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by Gergely Pollak 6c68211 is described below commit 6c6821106221e95e08fbacefb4c4ef74dc56b8e7 Author: Szilard Nemeth AuthorDate: Tue Sep 14 18:08:34 2021 +0200 YARN-10870. Missing user filtering check -> yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by Gergely Pollak --- .../webapp/FairSchedulerAppsBlock.java | 70 +++--- 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java index 14ad277..0ba8f1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java @@ -23,18 +23,21 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; -import java.util.Collection; -import java.util.HashSet; -import java.util.Map; +import java.security.Principal; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.commons.text.StringEscapeUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -49,6 +52,8 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import com.google.inject.Inject; +import javax.servlet.http.HttpServletRequest; + /** * Shows application information specific to the fair * scheduler as part of the fair scheduler page. @@ -58,10 +63,19 @@ public class FairSchedulerAppsBlock extends HtmlBlock { final FairSchedulerInfo fsinfo; final Configuration conf; final ResourceManager rm; + final boolean filterAppsByUser; + @Inject public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, Configuration conf) { super(ctx); +this.conf = conf; +this.rm = rm; + +this.filterAppsByUser = conf.getBoolean( +YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, +YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER); + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); fsinfo = new FairSchedulerInfo(scheduler); apps = new ConcurrentHashMap(); @@ -70,13 +84,53 @@ public class FairSchedulerAppsBlock extends HtmlBlock { if (!(RMAppState.NEW.equals(entry.getValue().getState()) || RMAppState.NEW_SAVING.equals(entry.getValue().getState()) || RMAppState.SUBMITTED.equals(entry.getValue().getState( { -apps.put(entry.getKey(), entry.getValue()); +if (!filterAppsByUser || hasAccess(entry.getValue(), +ctx.requestContext().getRequest())) { + apps.put(entry.getKey(), entry.getValue()); +} } } -this.conf = conf; -this.rm = rm; } - + + private UserGroupInformation getCallerUserGroupInformation( + HttpServletRequest hsr, boolean usePrincipal) { + +String remoteUser = hsr.getRemoteUser(); +if (usePrincipal) { + Principal princ = hsr.getUserPrincipal(); + remoteUser = princ == null ? null : princ.getName(); +} + +UserGroupInformation callerUGI = null; +if (remoteUser != null) { + callerUGI = UserGroupInformation.createRemoteUser(remoteUser); +} + +return callerUGI; + } + + protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) { +//
[hadoop] branch trunk updated: YARN-10915. AbstractCSQueue: Simplify complex logic in methods: deriveCapacityFromAbsoluteConfigurations and updateEffectiveResources (#3418)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 5dc2f7b YARN-10915. AbstractCSQueue: Simplify complex logic in methods: deriveCapacityFromAbsoluteConfigurations and updateEffectiveResources (#3418) 5dc2f7b is described below commit 5dc2f7b1376cdc392a5d204fb0ff3b9dec0ba36c Author: Benjamin Teke AuthorDate: Tue Sep 14 18:05:40 2021 +0200 YARN-10915. AbstractCSQueue: Simplify complex logic in methods: deriveCapacityFromAbsoluteConfigurations and updateEffectiveResources (#3418) Co-authored-by: Benjamin Teke --- .../scheduler/capacity/AbstractCSQueue.java| 163 +++-- .../scheduler/capacity/ParentQueue.java| 6 +- 2 files changed, 91 insertions(+), 78 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index f97460c..75aebfe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -1516,8 +1516,8 @@ public abstract class AbstractCSQueue implements CSQueue { parentQueueCapacities, queueCapacities.getExistingNodeLabels()); } - private Resource getMinResourceNormalized(String name, - Map effectiveMinRatio, Resource minResource) { + private Resource createNormalizedMinResource(Resource minResource, + Map effectiveMinRatio) { Resource ret = Resource.newInstance(minResource); int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { @@ -1527,18 +1527,35 @@ public abstract class AbstractCSQueue implements CSQueue { Float ratio = effectiveMinRatio.get(nResourceInformation.getName()); if (ratio != null) { ret.setResourceValue(i, -(long) (nResourceInformation.getValue() * ratio.floatValue())); +(long) (nResourceInformation.getValue() * ratio)); if (LOG.isDebugEnabled()) { - LOG.debug("Updating min resource for Queue: " + name + " as " + ret + LOG.debug("Updating min resource for Queue: " + queuePath + " as " + ret .getResourceInformation(i) + ", Actual resource: " - + nResourceInformation.getValue() + ", ratio: " + ratio - .floatValue()); + + nResourceInformation.getValue() + ", ratio: " + ratio); } } } return ret; } + private Resource getOrInheritMaxResource(Resource resourceByLabel, String label) { +Resource parentMaxResource = +parent.getQueueResourceQuotas().getConfiguredMaxResource(label); +if (parentMaxResource.equals(Resources.none())) { + parentMaxResource = + parent.getQueueResourceQuotas().getEffectiveMaxResource(label); +} + +Resource configuredMaxResource = +getQueueResourceQuotas().getConfiguredMaxResource(label); +if (configuredMaxResource.equals(Resources.none())) { + return Resources.clone(parentMaxResource); +} + +return Resources.clone(Resources.min(resourceCalculator, resourceByLabel, +configuredMaxResource, parentMaxResource)); + } + void updateMaxAppRelatedField(CapacitySchedulerConfiguration conf, LeafQueue leafQueue) { int maxApplications = conf.getMaximumApplicationsPerQueue(queuePath); @@ -1586,99 +1603,97 @@ public abstract class AbstractCSQueue implements CSQueue { .getMaximumCapacity(maxLabel)); } - private void deriveCapacityFromAbsoluteConfigurations(String label, - Resource clusterResource, ResourceCalculator rc) { - -/* - * In case when queues are configured with absolute resources, it is better - * to update capacity/max-capacity etc w.r.t absolute resource as well. In - * case of computation, these values wont be used any more. However for - * metrics and UI, its better these values are pre-computed here itself. - */ - -// 1. Update capacity as a float based on parent's minResource -float f = rc.divide(clusterResource, + void deriveCapacityFromAbsoluteConfigurations(String label, + Resource clusterResource) { +// Update capacity with a float calculated from the parent's
[hadoop] branch trunk updated: YARN-10917. Investigate and simplify CapacitySchedulerConfigValidator#validateQueueHierarchy (#3403)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 783d94f YARN-10917. Investigate and simplify CapacitySchedulerConfigValidator#validateQueueHierarchy (#3403) 783d94f is described below commit 783d94f5cdf2f3b03aee5ae5a1bcd4cc14dcb292 Author: Tamas Domok AuthorDate: Tue Sep 14 17:54:25 2021 +0200 YARN-10917. Investigate and simplify CapacitySchedulerConfigValidator#validateQueueHierarchy (#3403) * YARN-10917. Investigate and simplify CapacitySchedulerConfigValidator#validateQueueHierarchy. Co-authored-by: Tamas Domok --- .../capacity/CapacitySchedulerConfigValidator.java | 152 - 1 file changed, 88 insertions(+), 64 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java index ef9f97a..ca0d586 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java @@ -106,12 +106,8 @@ public final class CapacitySchedulerConfigValidator { } } - private static boolean isDynamicQueue(CSQueue csQueue) { -return ((AbstractCSQueue)csQueue).isDynamicQueue(); - } - /** - * Ensure all existing queues are present. Queues cannot be deleted if its not + * Ensure all existing queues are present. Queues cannot be deleted if it's not * in Stopped state, Queue's cannot be moved from one hierarchy to other also. * Previous child queue could be converted into parent queue if it is in * STOPPED state. @@ -125,78 +121,106 @@ public final class CapacitySchedulerConfigValidator { CapacitySchedulerConfiguration newConf) throws IOException { // check that all static queues are included in the newQueues list for (CSQueue oldQueue : queues.getQueues()) { - if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom( - oldQueue.getClass( { -String queuePath = oldQueue.getQueuePath(); -CSQueue newQueue = newQueues.get(queuePath); -String configPrefix = newConf.getQueuePrefix( -oldQueue.getQueuePath()); -String state = newConf.get(configPrefix + "state"); -QueueState newQueueState = null; -if (state != null) { - try { -newQueueState = QueueState.valueOf(state); - } catch (Exception ex) { -LOG.warn("Not a valid queue state for queue " -+ oldQueue.getQueuePath()); + if (AbstractAutoCreatedLeafQueue.class.isAssignableFrom(oldQueue.getClass())) { +continue; + } + + final String queuePath = oldQueue.getQueuePath(); + final String configPrefix = CapacitySchedulerConfiguration.getQueuePrefix( + oldQueue.getQueuePath()); + final QueueState newQueueState = createQueueState(newConf.get(configPrefix + "state"), + queuePath); + final CSQueue newQueue = newQueues.get(queuePath); + + if (null == newQueue) { +// old queue doesn't exist in the new XML +if (isEitherQueueStopped(oldQueue.getState(), newQueueState)) { + LOG.info("Deleting Queue {}, as it is not present in the modified capacity " + + "configuration xml", queuePath); +} else { + if (!isDynamicQueue(oldQueue)) { +throw new IOException(oldQueue.getQueuePath() + " cannot be" ++ " deleted from the capacity scheduler configuration, as the" ++ " queue is not yet in stopped state. Current State : " ++ oldQueue.getState()); } } -if (null == newQueue) { - // old queue doesn't exist in the new XML - if (oldQueue.getState() == QueueState.STOPPED || - newQueueState == QueueState.STOPPED) { -LOG.info("Deleting Queue " + queuePath + ", as it is not" -+ " present in the modified capacity configuration xml"); - } else { -if (!isDynamicQueue(oldQueue)) { - throw new IOException(oldQueue.getQueuePath() + " cannot be"
[hadoop] branch branch-3.2 updated: YARN-10901. Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir (#3410)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 478d2f2 YARN-10901. Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir (#3410) 478d2f2 is described below commit 478d2f2897abba5bad2f5204b0d4ec02cbb7ba30 Author: Tamas Domok AuthorDate: Tue Sep 14 17:35:07 2021 +0200 YARN-10901. Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir (#3410) Co-authored-by: Tamas Domok --- .../LogAggregationFileController.java | 13 +++- .../TestLogAggregationFileController.java | 75 ++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index 35f2d45..697ecbc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.logaggregation.filecontroller; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang3.RandomStringUtils; import java.io.FileNotFoundException; import java.io.IOException; @@ -368,10 +369,13 @@ public abstract class LogAggregationFileController { throw new YarnRuntimeException("Failed to create remoteLogDir [" + remoteRootLogDir + "]", e); } -} else{ +} else { //Check if FS has capability to set/modify permissions + Path permissionCheckFile = new Path(qualified, String.format("%s.permission_check", + RandomStringUtils.randomAlphanumeric(8))); try { -remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS)); +remoteFS.createNewFile(permissionCheckFile); +remoteFS.setPermission(permissionCheckFile, new FsPermission(TLDIR_PERMISSIONS)); } catch (UnsupportedOperationException use) { LOG.info("Unable to set permissions for configured filesystem since" + " it does not support this", remoteFS.getScheme()); @@ -379,6 +383,11 @@ public abstract class LogAggregationFileController { } catch (IOException e) { LOG.warn("Failed to check if FileSystem suppports permissions on " + "remoteLogDir [" + remoteRootLogDir + "]", e); + } finally { +try { + remoteFS.delete(permissionCheckFile, false); +} catch (IOException ignored) { +} } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java new file mode 100644 index 000..3566884 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.logaggregation.filecontroller; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.A
[hadoop] branch branch-3.3 updated: YARN-10901. Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir (#3409)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 8e4ac01 YARN-10901. Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir (#3409) 8e4ac01 is described below commit 8e4ac01135e386fb5ed09846c0f7bb9deffdadde Author: Tamas Domok AuthorDate: Tue Sep 14 17:34:32 2021 +0200 YARN-10901. Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir (#3409) Co-authored-by: Tamas Domok --- .../LogAggregationFileController.java | 13 +- .../TestLogAggregationFileController.java | 53 ++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index cf305ba..4844ae2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.logaggregation.filecontroller; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import java.io.FileNotFoundException; @@ -373,10 +374,13 @@ public abstract class LogAggregationFileController { throw new YarnRuntimeException("Failed to create remoteLogDir [" + remoteRootLogDir + "]", e); } -} else{ +} else { //Check if FS has capability to set/modify permissions + Path permissionCheckFile = new Path(qualified, String.format("%s.permission_check", + RandomStringUtils.randomAlphanumeric(8))); try { -remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS)); +remoteFS.createNewFile(permissionCheckFile); +remoteFS.setPermission(permissionCheckFile, new FsPermission(TLDIR_PERMISSIONS)); } catch (UnsupportedOperationException use) { LOG.info("Unable to set permissions for configured filesystem since" + " it does not support this", remoteFS.getScheme()); @@ -384,6 +388,11 @@ public abstract class LogAggregationFileController { } catch (IOException e) { LOG.warn("Failed to check if FileSystem suppports permissions on " + "remoteLogDir [" + remoteRootLogDir + "]", e); + } finally { +try { + remoteFS.delete(permissionCheckFile, false); +} catch (IOException ignored) { +} } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java index 5ade0fa..818e011 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java @@ -19,17 +19,23 @@ package org.apache.hadoop.yarn.logaggregation.filecontroller; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentMatcher; import org.mockito.Mockito; import java.io.FileNotFoundException; import java.net.URI; +import static org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController.TLDIR_PERMISSIONS; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; @@ -88,4 +94,51 @@ public class TestLogAggregationFileController { verify(fs).setOwner(any(), eq("yarn_user"), eq(tes
[hadoop] branch trunk updated (4f563ff -> 63c8922)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 4f563ff HDFS-16225. Fix typo for FederationTestUtils (#3428). Contributed by tomscut. add 63c8922 YARN-10912. AbstractCSQueue#updateConfigurableResourceRequirement: Separate validation logic from initialization logic (#3390) No new revisions were added by this update. Summary of changes: .../scheduler/capacity/AbstractCSQueue.java| 70 -- 1 file changed, 37 insertions(+), 33 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10872. Replace getPropsWithPrefix calls in AutoCreatedQueueTemplate (#3396)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 971f1b8 YARN-10872. Replace getPropsWithPrefix calls in AutoCreatedQueueTemplate (#3396) 971f1b8 is described below commit 971f1b8b0a37b4d3eafdecf705afa4f684253e26 Author: Benjamin Teke AuthorDate: Fri Sep 10 17:32:42 2021 +0200 YARN-10872. Replace getPropsWithPrefix calls in AutoCreatedQueueTemplate (#3396) Co-authored-by: Benjamin Teke --- .../capacity/AbstractManagedParentQueue.java | 37 -- .../capacity/AutoCreatedQueueTemplate.java | 24 -- .../capacity/CapacitySchedulerConfiguration.java | 5 +-- .../capacity/ConfigurationProperties.java | 12 ++- .../capacity/TestConfigurationProperties.java | 11 +++ 5 files changed, 46 insertions(+), 43 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java index a9e82a6..9c16de0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java @@ -28,11 +28,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Comparator; import java.util.Iterator; import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; /** * A container class for automatically created child leaf queues. @@ -173,44 +170,22 @@ public abstract class AbstractManagedParentQueue extends ParentQueue { return queueManagementPolicy; } - protected SortedMap getConfigurationsWithPrefix - (SortedMap sortedConfigs, String prefix) { -return sortedConfigs.subMap( prefix, prefix + Character.MAX_VALUE ); - } - - protected SortedMap sortCSConfigurations() { -SortedMap sortedConfigs = new TreeMap( -new Comparator() { - public int compare(String s1, String s2) { -return s1.compareToIgnoreCase(s2); - } - -}); - -for (final Iterator> iterator = - csContext.getConfiguration().iterator(); iterator.hasNext(); ) { - final Map.Entry confKeyValuePair = iterator.next(); - sortedConfigs.put(confKeyValuePair.getKey(), confKeyValuePair.getValue()); -} -return sortedConfigs; - } - protected CapacitySchedulerConfiguration initializeLeafQueueConfigs(String configPrefix) { CapacitySchedulerConfiguration leafQueueConfigs = new CapacitySchedulerConfiguration(new Configuration(false), false); -String prefix = YarnConfiguration.RESOURCE_TYPES + "."; Map rtProps = csContext -.getConfiguration().getPropsWithPrefix(prefix); +.getConfiguration().getConfigurationProperties() +.getPropertiesWithPrefix(YarnConfiguration.RESOURCE_TYPES + ".", true); for (Map.Entry entry : rtProps.entrySet()) { - leafQueueConfigs.set(prefix + entry.getKey(), entry.getValue()); + leafQueueConfigs.set(entry.getKey(), entry.getValue()); } -SortedMap sortedConfigs = sortCSConfigurations(); -SortedMap templateConfigs = getConfigurationsWithPrefix -(sortedConfigs, configPrefix); +Map templateConfigs = csContext +.getConfiguration().getConfigurationProperties() +.getPropertiesWithPrefix(configPrefix, true); for (final Iterator> iterator = templateConfigs.entrySet().iterator(); iterator.hasNext(); ) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java index cb0f789..eff8c4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resource
[hadoop] branch trunk updated (b229e5a -> 811fd23)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from b229e5a YARN-10910. AbstractCSQueue#setupQueueConfigs: Separate validation logic from initialization logic (#3407) add 811fd23 YARN-10852. Optimise CSConfiguration getAllUserWeightsForQueue (#3392) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/conf/Configuration.java | 2 +- .../capacity/CapacitySchedulerConfiguration.java | 34 +++--- .../scheduler/capacity/TestLeafQueue.java | 1 + 3 files changed, 25 insertions(+), 12 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (29a6f14 -> b229e5a)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 29a6f14 YARN-10914. Simplify duplicated code for tracking ResourceUsage in AbstractCSQueue (#3402) add b229e5a YARN-10910. AbstractCSQueue#setupQueueConfigs: Separate validation logic from initialization logic (#3407) No new revisions were added by this update. Summary of changes: .../scheduler/capacity/AbstractCSQueue.java| 215 +++-- 1 file changed, 113 insertions(+), 102 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10914. Simplify duplicated code for tracking ResourceUsage in AbstractCSQueue (#3402)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 29a6f14 YARN-10914. Simplify duplicated code for tracking ResourceUsage in AbstractCSQueue (#3402) 29a6f14 is described below commit 29a6f141d4bcc5307d2ac72c742f87611d7092d5 Author: Tamas Domok AuthorDate: Fri Sep 10 15:57:46 2021 +0200 YARN-10914. Simplify duplicated code for tracking ResourceUsage in AbstractCSQueue (#3402) Co-authored-by: Tamas Domok --- .../scheduler/capacity/AbstractCSQueue.java| 97 ++ 1 file changed, 43 insertions(+), 54 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 1f5820d..2da52d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -73,6 +73,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -1189,84 +1190,72 @@ public abstract class AbstractCSQueue implements CSQueue { } + private static String ensurePartition(String partition) { +return Optional.ofNullable(partition).orElse(RMNodeLabelsManager.NO_LABEL); + } + + @FunctionalInterface + interface Counter { +void count(String partition, Resource resource); + } + + @FunctionalInterface + interface CounterWithApp { +void count(String partition, Resource reservedRes, SchedulerApplicationAttempt application); + } + + private void count(String partition, Resource resource, Counter counter, Counter parentCounter) { +final String checkedPartition = ensurePartition(partition); +counter.count(checkedPartition, resource); +Optional.ofNullable(parentCounter).ifPresent(c -> c.count(checkedPartition, resource)); + } + + private void countAndUpdate(String partition, Resource resource, + Counter counter, CounterWithApp parentCounter) { +final String checkedPartition = ensurePartition(partition); +counter.count(checkedPartition, resource); +CSQueueUtils.updateUsedCapacity(resourceCalculator, +labelManager.getResourceByLabel(checkedPartition, Resources.none()), +checkedPartition, this); +Optional.ofNullable(parentCounter).ifPresent(c -> c.count(checkedPartition, resource, null)); + } + @Override public void incReservedResource(String partition, Resource reservedRes) { -if (partition == null) { - partition = RMNodeLabelsManager.NO_LABEL; -} - -queueUsage.incReserved(partition, reservedRes); -if(null != parent){ - parent.incReservedResource(partition, reservedRes); -} +count(partition, reservedRes, queueUsage::incReserved, +parent == null ? null : parent::incReservedResource); } @Override public void decReservedResource(String partition, Resource reservedRes) { -if (partition == null) { - partition = RMNodeLabelsManager.NO_LABEL; -} - -queueUsage.decReserved(partition, reservedRes); -if(null != parent){ - parent.decReservedResource(partition, reservedRes); -} +count(partition, reservedRes, queueUsage::decReserved, +parent == null ? null : parent::decReservedResource); } @Override public void incPendingResource(String nodeLabel, Resource resourceToInc) { -if (nodeLabel == null) { - nodeLabel = RMNodeLabelsManager.NO_LABEL; -} -// ResourceUsage has its own lock, no addition lock needs here. -queueUsage.incPending(nodeLabel, resourceToInc); -if (null != parent) { - parent.incPendingResource(nodeLabel, resourceToInc); -} +count(nodeLabel, resourceToInc, queueUsage::incPending, +parent == null ? null : parent::incPendingResource); } @Override public void decPendingResource(String nodeLabel, Resource resourceToDec) { -if (nodeLabel == null) { - nodeLabel = RMNodeLabelsManager.NO_LABEL; -} -// ResourceUsage has its own lock, no addition lock needs here. -queueUsage.decPending(nodeLabel, resourceToDec); -if (null != parent) { - parent.decPendingResource(nodeLabel, resourceToDec); -} +
[hadoop] branch trunk updated: YARN-10870. Missing user filtering check -> yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by Gergely Pollak
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 2ff3fc5 YARN-10870. Missing user filtering check -> yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by Gergely Pollak 2ff3fc5 is described below commit 2ff3fc50e4a9bc60a1ca968bd495a18728084eaa Author: Szilard Nemeth AuthorDate: Wed Sep 8 18:01:39 2021 +0200 YARN-10870. Missing user filtering check -> yarn.webapp.filter-entity-list-by-user for RM Scheduler page. Contributed by Gergely Pollak --- .../webapp/FairSchedulerAppsBlock.java | 69 +++--- 1 file changed, 61 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java index 14ad277..f6202cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java @@ -23,18 +23,21 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; -import java.util.Collection; -import java.util.HashSet; -import java.util.Map; +import java.security.Principal; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.commons.text.StringEscapeUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -49,6 +52,8 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import com.google.inject.Inject; +import javax.servlet.http.HttpServletRequest; + /** * Shows application information specific to the fair * scheduler as part of the fair scheduler page. @@ -58,10 +63,19 @@ public class FairSchedulerAppsBlock extends HtmlBlock { final FairSchedulerInfo fsinfo; final Configuration conf; final ResourceManager rm; + final boolean filterAppsByUser; + @Inject public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, Configuration conf) { super(ctx); +this.conf = conf; +this.rm = rm; + +this.filterAppsByUser = conf.getBoolean( +YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, +YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER); + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); fsinfo = new FairSchedulerInfo(scheduler); apps = new ConcurrentHashMap(); @@ -70,13 +84,52 @@ public class FairSchedulerAppsBlock extends HtmlBlock { if (!(RMAppState.NEW.equals(entry.getValue().getState()) || RMAppState.NEW_SAVING.equals(entry.getValue().getState()) || RMAppState.SUBMITTED.equals(entry.getValue().getState( { -apps.put(entry.getKey(), entry.getValue()); +if (!filterAppsByUser || hasAccess(entry.getValue(), +ctx.requestContext().getRequest())) { + apps.put(entry.getKey(), entry.getValue()); +} } } -this.conf = conf; -this.rm = rm; } - + + private UserGroupInformation getCallerUserGroupInformation( + HttpServletRequest hsr, boolean usePrincipal) { +String remoteUser = hsr.getRemoteUser(); +if (usePrincipal) { + Principal princ = hsr.getUserPrincipal(); + remoteUser = princ == null ? null : princ.getName(); +} + +UserGroupInformation callerUGI = null; +if (remoteUser != null) { + callerUGI = UserGroupInformation.createRemoteUser(remoteUser); +} + +return callerUGI; + } + + protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) { +// Check f
[hadoop] branch trunk updated: HADOOP-17857. Check real user ACLs in addition to proxied user ACLs. Contributed by Eric Payne
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 5428d36 HADOOP-17857. Check real user ACLs in addition to proxied user ACLs. Contributed by Eric Payne 5428d36 is described below commit 5428d36b56fab319ab68258139d6133ded9bbafc Author: Szilard Nemeth AuthorDate: Wed Sep 8 17:27:22 2021 +0200 HADOOP-17857. Check real user ACLs in addition to proxied user ACLs. Contributed by Eric Payne --- .../hadoop/security/authorize/AccessControlList.java | 12 +--- .../security/authorize/TestAccessControlList.java | 18 ++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java index e86d918..aa5b01f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java @@ -56,6 +56,7 @@ public class AccessControlList implements Writable { // Indicates an ACL string that represents access to all users public static final String WILDCARD_ACL_VALUE = "*"; private static final int INITIAL_CAPACITY = 256; + public static final String USE_REAL_ACLS = "~"; // Set of users who are granted access. private Collection users; @@ -224,9 +225,12 @@ public class AccessControlList implements Writable { /** * Checks if a user represented by the provided {@link UserGroupInformation} - * is a member of the Access Control List + * is a member of the Access Control List. If user was proxied and + * USE_REAL_ACLS + the real user name is in the control list, then treat this + * case as if user were in the ACL list. * @param ugi UserGroupInformation to check if contained in the ACL - * @return true if ugi is member of the list + * @return true if ugi is member of the list or if USE_REAL_ACLS + real user + * is in the list */ public final boolean isUserInList(UserGroupInformation ugi) { if (allAllowed || users.contains(ugi.getShortUserName())) { @@ -239,7 +243,9 @@ public class AccessControlList implements Writable { } } } -return false; +UserGroupInformation realUgi = ugi.getRealUser(); +return realUgi != null && + users.contains(USE_REAL_ACLS + realUgi.getShortUserName()); } public boolean isUserAllowed(UserGroupInformation ugi) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestAccessControlList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestAccessControlList.java index 8e1b82b..53ab275 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestAccessControlList.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestAccessControlList.java @@ -471,4 +471,22 @@ public class TestAccessControlList { + " is incorrectly granted the access-control!!", acl.isUserAllowed(ugi)); } + + @Test + public void testUseRealUserAclsForProxiedUser() { +String realUser = "realUser"; +AccessControlList acl = new AccessControlList(realUser); +UserGroupInformation realUserUgi = +UserGroupInformation.createRemoteUser(realUser); +UserGroupInformation user1 = +UserGroupInformation.createProxyUserForTesting("regularJane", +realUserUgi, new String [] {"group1"}); +assertFalse("User " + user1 + " should not have been granted access.", +acl.isUserAllowed(user1)); + +acl = new AccessControlList(AccessControlList.USE_REAL_ACLS + realUser); + +assertTrue("User " + user1 + " should have access but was denied.", +acl.isUserAllowed(user1)); + } } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (4e209a3 -> 5e16689)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 4e209a3 YARN-10919. Remove LeafQueue#scheduler field (#3382) add 5e16689 YARN-10901. Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir (#3355) No new revisions were added by this update. Summary of changes: .../LogAggregationFileController.java | 13 +- .../TestLogAggregationFileController.java | 53 ++ 2 files changed, 64 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (40e639a -> 4e209a3)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 40e639a YARN-10646. TestCapacitySchedulerWeightMode test descriptor comments doesnt reflect the correct scenario (#3339) add 4e209a3 YARN-10919. Remove LeafQueue#scheduler field (#3382) No new revisions were added by this update. Summary of changes: .../resourcemanager/scheduler/capacity/LeafQueue.java | 13 + 1 file changed, 5 insertions(+), 8 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10646. TestCapacitySchedulerWeightMode test descriptor comments doesnt reflect the correct scenario (#3339)
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 40e639a YARN-10646. TestCapacitySchedulerWeightMode test descriptor comments doesnt reflect the correct scenario (#3339) 40e639a is described below commit 40e639ad078a5f6f0503f206c4a0c4c012793349 Author: Benjamin Teke AuthorDate: Wed Sep 8 16:11:04 2021 +0200 YARN-10646. TestCapacitySchedulerWeightMode test descriptor comments doesnt reflect the correct scenario (#3339) Co-authored-by: Benjamin Teke --- .../capacity/TestCapacitySchedulerWeightMode.java | 24 +++--- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java index 171123a..a5d80dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java @@ -198,9 +198,9 @@ public class TestCapacitySchedulerWeightMode { * a x(=100%), y(50%) b y(=50%), z(=100%) * __ * / / \ - * a1 ([x,y]: w=100)b1(no) b2([y,z]: w=100) + * a1 ([x,y]: w=1)b1(no) b2([y,z]: w=1) * - * Parent uses weight, child uses percentage + * Parent uses percentages, child uses weights */ public static Configuration getCSConfWithLabelsParentUsePctChildUseWeight( Configuration config) { @@ -210,9 +210,9 @@ public class TestCapacitySchedulerWeightMode { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a", "b" }); -conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "x", 100); -conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100); -conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100); +conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); +conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100); +conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100); conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 10); conf.setMaximumCapacity(A, 10); @@ -228,23 +228,23 @@ public class TestCapacitySchedulerWeightMode { // Define 2nd-level queues conf.setQueues(A, new String[] { "a1" }); -conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100); +conf.setLabeledQueueWeight(A1, RMNodeLabelsManager.NO_LABEL, 1); conf.setMaximumCapacity(A1, 100); conf.setAccessibleNodeLabels(A1, toSet("x", "y")); conf.setDefaultNodeLabelExpression(A1, "x"); -conf.setCapacityByLabel(A1, "x", 100); -conf.setCapacityByLabel(A1, "y", 100); +conf.setLabeledQueueWeight(A1, "x", 1); +conf.setLabeledQueueWeight(A1, "y", 1); conf.setQueues(B, new String[] { "b1", "b2" }); -conf.setCapacityByLabel(B1, RMNodeLabelsManager.NO_LABEL, 50); +conf.setLabeledQueueWeight(B1, RMNodeLabelsManager.NO_LABEL, 1); conf.setMaximumCapacity(B1, 50); conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); -conf.setCapacityByLabel(B2, RMNodeLabelsManager.NO_LABEL, 50); +conf.setLabeledQueueWeight(B2, RMNodeLabelsManager.NO_LABEL, 1); conf.setMaximumCapacity(B2, 50); conf.setAccessibleNodeLabels(B2, toSet("y", "z")); -conf.setCapacityByLabel(B2, "y", 100); -conf.setCapacityByLabel(B2, "z", 100); +conf.setLabeledQueueWeight(B2, "y", 1); +conf.setLabeledQueueWeight(B2, "z", 1); return conf; } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org