[hadoop] branch branch-3.3 updated: HADOOP-18433. Fix main thread name for . (#4838) (#5692)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 60b37bbdf71 HADOOP-18433. Fix main thread name for . (#4838) (#5692)
60b37bbdf71 is described below
commit 60b37bbdf712f5c17230318c4e7c7ec5ce57cedc
Author: Doroszlai, Attila <6454655+adorosz...@users.noreply.github.com>
AuthorDate: Mon Jun 5 07:51:53 2023 +0200
HADOOP-18433. Fix main thread name for . (#4838) (#5692)
(cherry picked from commit f68f1a45783d7eb3f982fadef800b58ab8b763f3)
Co-authored-by: zhengchenyu
---
.../hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 07cff6eac08..939180b195c 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -1284,8 +1284,7 @@ public abstract class Server {
bind(acceptChannel.socket(), address, backlogLength, conf,
portRangeConfig);
//Could be an ephemeral port
this.listenPort = acceptChannel.socket().getLocalPort();
- Thread.currentThread().setName("Listener at " +
- bindAddress + "/" + this.listenPort);
+ LOG.info("Listener at {}:{}", bindAddress, this.listenPort);
// create a selector;
selector= Selector.open();
readers = new Reader[readThreads];
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11277. Trigger log-dir deletion by size for NonAggregatingLogHandler. (#4797)
This is an automated email from the ASF dual-hosted git repository.
slfan1989 pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new ee94f6cdcbc YARN-11277. Trigger log-dir deletion by size for
NonAggregatingLogHandler. (#4797)
ee94f6cdcbc is described below
commit ee94f6cdcbc4914cbdf1a2132f318083c4d38494
Author: Xianming Lei <31424839+le...@users.noreply.github.com>
AuthorDate: Mon Jun 5 11:08:06 2023 +0800
YARN-11277. Trigger log-dir deletion by size for NonAggregatingLogHandler.
(#4797)
Reviewed-by: Akira Ajisaka
Reviewed-by: Ashutosh Gupta
Reviewed-by: Shilun Fan
Signed-off-by: Shilun Fan
---
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 11 +++
.../src/main/resources/yarn-default.xml| 23 +
.../loghandler/NonAggregatingLogHandler.java | 49 +--
.../loghandler/TestNonAggregatingLogHandler.java | 99 ++
4 files changed, 174 insertions(+), 8 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b6601b835d6..ff531cdc2a7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -4960,6 +4960,17 @@ public class YarnConfiguration extends Configuration {
public static final String APPS_CACHE_EXPIRE = YARN_PREFIX +
"apps.cache.expire";
public static final String DEFAULT_APPS_CACHE_EXPIRE = "30s";
+ /** Enabled trigger log-dir deletion by size for NonAggregatingLogHandler. */
+ public static final String NM_LOG_TRIGGER_DELETE_BY_SIZE_ENABLED = NM_PREFIX
+
+ "log.trigger.delete.by-size.enabled";
+ public static final boolean DEFAULT_NM_LOG_TRIGGER_DELETE_BY_SIZE_ENABLED =
false;
+
+ /** Trigger log-dir deletion when the total log size of an app is greater
than
+ * yarn.nodemanager.log.delete.threshold.
+ * Depends on yarn.nodemanager.log.trigger.delete.by-size.enabled = true. */
+ public static final String NM_LOG_DELETE_THRESHOLD = NM_PREFIX +
"log.delete.threshold";
+ public static final long DEFAULT_NM_LOG_DELETE_THRESHOLD = 100L * 1024 *
1024 * 1024;
+
public YarnConfiguration() {
super();
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 1f0982aede0..aea92260013 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -5293,4 +5293,27 @@
+
+yarn.nodemanager.log.trigger.delete.by-size.enabled
+false
+
+ Optional.
+ Enabled trigger log-dir deletion by size for NonAggregatingLogHandler
+
+
+
+
+yarn.nodemanager.log.delete.threshold
+100g
+
+ Optional.
+ Trigger log-dir deletion when the total log size of an app is greater
than
+ yarn.nodemanager.log.delete.threshold and
+ yarn.nodemanager.log.trigger.delete.by-size.enabled = true.
+ You can use the following suffix (case insensitive): k(kilo), m(mega),
g(giga), t(tera), p(peta),
+ e(exa) to specify the size (such as 128k, 512m, 1g, etc.),
+ Or provide complete size in bytes (such as 134217728 for 128 MB).
+
+
+
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
index e8b2f472fb4..1721756f056 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
@@ -71,6 +71,8 @@ public class NonAggregatingLogHandler extends AbstractService
implements
private final LocalDirsHandlerService dirsHandler;
private final NMStateStoreService stateStore;
private long deleteDelaySeconds;
+ private boolean enableTriggerDeleteBySize;
+ private long deleteThreshold;
private ScheduledThreadPoolExecutor sched;
public
[hadoop] branch HDFS-2139 deleted (was be4c638e4cc)
This is an automated email from the ASF dual-hosted git repository. zanderxu pushed a change to branch HDFS-2139 in repository https://gitbox.apache.org/repos/asf/hadoop.git was be4c638e4cc HDFS-16748. RBF: DFSClient should uniquely identify writing files by namespace id and iNodeId via RBF (#4813). Contributed by ZanderXu. The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch HDFS-2139 created (now e0a339223a8)
This is an automated email from the ASF dual-hosted git repository. zanderxu pushed a change to branch HDFS-2139 in repository https://gitbox.apache.org/repos/asf/hadoop.git at e0a339223a8 HADOOP-18709. Add curator based ZooKeeper communication support over SSL/TLS into the common library. Contributed by Ferenc Erdelyi No new revisions were added by this update. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-18709. Add curator based ZooKeeper communication support over SSL/TLS into the common library. Contributed by Ferenc Erdelyi
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e0a339223a8 HADOOP-18709. Add curator based ZooKeeper communication
support over SSL/TLS into the common library. Contributed by Ferenc Erdelyi
e0a339223a8 is described below
commit e0a339223a89367d9391758ff62c99923369b5ee
Author: Szilard Nemeth
AuthorDate: Sun Jun 4 14:40:41 2023 -0400
HADOOP-18709. Add curator based ZooKeeper communication support over
SSL/TLS into the common library. Contributed by Ferenc Erdelyi
---
hadoop-common-project/hadoop-common/pom.xml| 8 +
.../apache/hadoop/fs/CommonConfigurationKeys.java | 8 +
.../hadoop/util/curator/ZKCuratorManager.java | 222 ++---
.../src/main/resources/core-default.xml| 29 +++
.../util/curator/TestSecureZKCuratorManager.java | 214
.../util/curator/resources/data/ssl/keystore.jks | Bin 0 -> 4160 bytes
.../util/curator/resources/data/ssl/truststore.jks | Bin 0 -> 994 bytes
7 files changed, 458 insertions(+), 23 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/pom.xml
b/hadoop-common-project/hadoop-common/pom.xml
index 58006c011da..426f7a4af41 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -340,6 +340,14 @@
+
+ io.netty
+ netty-handler
+
+
+ io.netty
+ netty-transport-native-epoll
+
io.dropwizard.metrics
metrics-core
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 9d6224366d1..63f494a20ac 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -417,6 +417,14 @@ public class CommonConfigurationKeys extends
CommonConfigurationKeysPublic {
/** How often to retry a ZooKeeper operation in milliseconds. */
public static final String ZK_RETRY_INTERVAL_MS =
ZK_PREFIX + "retry-interval-ms";
+ /** Keystore location for ZooKeeper client connection over SSL. */
+ public static final String ZK_SSL_KEYSTORE_LOCATION = ZK_PREFIX +
"ssl.keystore.location";
+ /** Keystore password for ZooKeeper client connection over SSL. */
+ public static final String ZK_SSL_KEYSTORE_PASSWORD = ZK_PREFIX +
"ssl.keystore.password";
+ /** Truststore location for ZooKeeper client connection over SSL. */
+ public static final String ZK_SSL_TRUSTSTORE_LOCATION = ZK_PREFIX +
"ssl.truststore.location";
+ /** Truststore password for ZooKeeper client connection over SSL. */
+ public static final String ZK_SSL_TRUSTSTORE_PASSWORD = ZK_PREFIX +
"ssl.truststore.password";
public static final intZK_RETRY_INTERVAL_MS_DEFAULT = 1000;
/** Default domain name resolver for hadoop to use. */
public static final String HADOOP_DOMAINNAME_RESOLVER_IMPL =
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
index c11b8683867..81ee4663e1c 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
+import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.AuthInfo;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
@@ -39,13 +40,17 @@ import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.client.ZKClientConfig;
+import org.apache.zookeeper.common.ClientX509Util;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.util.Preconditions;
+import javax.naming.ConfigurationException;
+
/**
* Helper class that provides utility methods specific to ZK operations.
*/
@@ -122,7 +127,7 @@ public final class ZKCuratorManager {
* Start the connection to the ZooKeeper ensemble.
* @throws IOException If the connection cannot be started.
*/
- public void start() throws IOException {
+ public void start() throws IOException{
this.start(new ArrayList<>());
}
@@ -132,6 +137,20 @@ public final class ZKCuratorManager
