[hadoop] branch branch-3.3 updated: HADOOP-18433. Fix main thread name for . (#4838) (#5692)
This is an automated email from the ASF dual-hosted git repository. adoroszlai pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 60b37bbdf71 HADOOP-18433. Fix main thread name for . (#4838) (#5692) 60b37bbdf71 is described below commit 60b37bbdf712f5c17230318c4e7c7ec5ce57cedc Author: Doroszlai, Attila <6454655+adorosz...@users.noreply.github.com> AuthorDate: Mon Jun 5 07:51:53 2023 +0200 HADOOP-18433. Fix main thread name for . (#4838) (#5692) (cherry picked from commit f68f1a45783d7eb3f982fadef800b58ab8b763f3) Co-authored-by: zhengchenyu --- .../hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 07cff6eac08..939180b195c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -1284,8 +1284,7 @@ public abstract class Server { bind(acceptChannel.socket(), address, backlogLength, conf, portRangeConfig); //Could be an ephemeral port this.listenPort = acceptChannel.socket().getLocalPort(); - Thread.currentThread().setName("Listener at " + - bindAddress + "/" + this.listenPort); + LOG.info("Listener at {}:{}", bindAddress, this.listenPort); // create a selector; selector= Selector.open(); readers = new Reader[readThreads]; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11277. Trigger log-dir deletion by size for NonAggregatingLogHandler. (#4797)
This is an automated email from the ASF dual-hosted git repository. slfan1989 pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new ee94f6cdcbc YARN-11277. Trigger log-dir deletion by size for NonAggregatingLogHandler. (#4797) ee94f6cdcbc is described below commit ee94f6cdcbc4914cbdf1a2132f318083c4d38494 Author: Xianming Lei <31424839+le...@users.noreply.github.com> AuthorDate: Mon Jun 5 11:08:06 2023 +0800 YARN-11277. Trigger log-dir deletion by size for NonAggregatingLogHandler. (#4797) Reviewed-by: Akira Ajisaka Reviewed-by: Ashutosh Gupta Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 11 +++ .../src/main/resources/yarn-default.xml| 23 + .../loghandler/NonAggregatingLogHandler.java | 49 +-- .../loghandler/TestNonAggregatingLogHandler.java | 99 ++ 4 files changed, 174 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index b6601b835d6..ff531cdc2a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -4960,6 +4960,17 @@ public class YarnConfiguration extends Configuration { public static final String APPS_CACHE_EXPIRE = YARN_PREFIX + "apps.cache.expire"; public static final String DEFAULT_APPS_CACHE_EXPIRE = "30s"; + /** Enabled trigger log-dir deletion by size for NonAggregatingLogHandler. */ + public static final String NM_LOG_TRIGGER_DELETE_BY_SIZE_ENABLED = NM_PREFIX + + "log.trigger.delete.by-size.enabled"; + public static final boolean DEFAULT_NM_LOG_TRIGGER_DELETE_BY_SIZE_ENABLED = false; + + /** Trigger log-dir deletion when the total log size of an app is greater than + * yarn.nodemanager.log.delete.threshold. + * Depends on yarn.nodemanager.log.trigger.delete.by-size.enabled = true. */ + public static final String NM_LOG_DELETE_THRESHOLD = NM_PREFIX + "log.delete.threshold"; + public static final long DEFAULT_NM_LOG_DELETE_THRESHOLD = 100L * 1024 * 1024 * 1024; + public YarnConfiguration() { super(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 1f0982aede0..aea92260013 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5293,4 +5293,27 @@ + +yarn.nodemanager.log.trigger.delete.by-size.enabled +false + + Optional. + Enabled trigger log-dir deletion by size for NonAggregatingLogHandler + + + + +yarn.nodemanager.log.delete.threshold +100g + + Optional. + Trigger log-dir deletion when the total log size of an app is greater than + yarn.nodemanager.log.delete.threshold and + yarn.nodemanager.log.trigger.delete.by-size.enabled = true. + You can use the following suffix (case insensitive): k(kilo), m(mega), g(giga), t(tera), p(peta), + e(exa) to specify the size (such as 128k, 512m, 1g, etc.), + Or provide complete size in bytes (such as 134217728 for 128 MB). + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java index e8b2f472fb4..1721756f056 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java @@ -71,6 +71,8 @@ public class NonAggregatingLogHandler extends AbstractService implements private final LocalDirsHandlerService dirsHandler; private final NMStateStoreService stateStore; private long deleteDelaySeconds; + private boolean enableTriggerDeleteBySize; + private long deleteThreshold; private ScheduledThreadPoolExecutor sched; public NonAggregatingLogHandle
[hadoop] branch HDFS-2139 deleted (was be4c638e4cc)
This is an automated email from the ASF dual-hosted git repository. zanderxu pushed a change to branch HDFS-2139 in repository https://gitbox.apache.org/repos/asf/hadoop.git was be4c638e4cc HDFS-16748. RBF: DFSClient should uniquely identify writing files by namespace id and iNodeId via RBF (#4813). Contributed by ZanderXu. The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch HDFS-2139 created (now e0a339223a8)
This is an automated email from the ASF dual-hosted git repository. zanderxu pushed a change to branch HDFS-2139 in repository https://gitbox.apache.org/repos/asf/hadoop.git at e0a339223a8 HADOOP-18709. Add curator based ZooKeeper communication support over SSL/TLS into the common library. Contributed by Ferenc Erdelyi No new revisions were added by this update. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-18709. Add curator based ZooKeeper communication support over SSL/TLS into the common library. Contributed by Ferenc Erdelyi
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new e0a339223a8 HADOOP-18709. Add curator based ZooKeeper communication support over SSL/TLS into the common library. Contributed by Ferenc Erdelyi e0a339223a8 is described below commit e0a339223a89367d9391758ff62c99923369b5ee Author: Szilard Nemeth AuthorDate: Sun Jun 4 14:40:41 2023 -0400 HADOOP-18709. Add curator based ZooKeeper communication support over SSL/TLS into the common library. Contributed by Ferenc Erdelyi --- hadoop-common-project/hadoop-common/pom.xml| 8 + .../apache/hadoop/fs/CommonConfigurationKeys.java | 8 + .../hadoop/util/curator/ZKCuratorManager.java | 222 ++--- .../src/main/resources/core-default.xml| 29 +++ .../util/curator/TestSecureZKCuratorManager.java | 214 .../util/curator/resources/data/ssl/keystore.jks | Bin 0 -> 4160 bytes .../util/curator/resources/data/ssl/truststore.jks | Bin 0 -> 994 bytes 7 files changed, 458 insertions(+), 23 deletions(-) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 58006c011da..426f7a4af41 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -340,6 +340,14 @@ + + io.netty + netty-handler + + + io.netty + netty-transport-native-epoll + io.dropwizard.metrics metrics-core diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index 9d6224366d1..63f494a20ac 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -417,6 +417,14 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { /** How often to retry a ZooKeeper operation in milliseconds. */ public static final String ZK_RETRY_INTERVAL_MS = ZK_PREFIX + "retry-interval-ms"; + /** Keystore location for ZooKeeper client connection over SSL. */ + public static final String ZK_SSL_KEYSTORE_LOCATION = ZK_PREFIX + "ssl.keystore.location"; + /** Keystore password for ZooKeeper client connection over SSL. */ + public static final String ZK_SSL_KEYSTORE_PASSWORD = ZK_PREFIX + "ssl.keystore.password"; + /** Truststore location for ZooKeeper client connection over SSL. */ + public static final String ZK_SSL_TRUSTSTORE_LOCATION = ZK_PREFIX + "ssl.truststore.location"; + /** Truststore password for ZooKeeper client connection over SSL. */ + public static final String ZK_SSL_TRUSTSTORE_PASSWORD = ZK_PREFIX + "ssl.truststore.password"; public static final intZK_RETRY_INTERVAL_MS_DEFAULT = 1000; /** Default domain name resolver for hadoop to use. */ public static final String HADOOP_DOMAINNAME_RESOLVER_IMPL = diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java index c11b8683867..81ee4663e1c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.AuthInfo; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -39,13 +40,17 @@ import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.client.ZKClientConfig; +import org.apache.zookeeper.common.ClientX509Util; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.util.Preconditions; +import javax.naming.ConfigurationException; + /** * Helper class that provides utility methods specific to ZK operations. */ @@ -122,7 +127,7 @@ public final class ZKCuratorManager { * Start the connection to the ZooKeeper ensemble. * @throws IOException If the connection cannot be started. */ - public void start() throws IOException { + public void start() throws IOException{ this.start(new ArrayList<>()); } @@ -132,6 +137,20 @@ public final class ZKCuratorManager