This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 9123f92 HDFS-15720 namenode audit async logger should add some log4j
config (#2532)
9123f92 is described below
commit 9123f9245cc49764d93a5295544a17519191430f
Author: Neil <[email protected]>
AuthorDate: Fri Dec 11 05:47:00 2020 +0800
HDFS-15720 namenode audit async logger should add some log4j config (#2532)
(cherry picked from commit 9bd3c9bc50672ebdd7f12b9a2235aec37b2a07cf)
(cherry picked from commit bc5458bbd4e6829bc02de57671c017fbdac06849)
---
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++++
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 12 ++++++++++--
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 21 +++++++++++++++++++++
3 files changed, 35 insertions(+), 2 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index e0d4306..3164e42 100755
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -610,6 +610,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys
{
public static final boolean DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_DEFAULT
= false;
public static final String DFS_NAMENODE_AUDIT_LOG_ASYNC_KEY =
"dfs.namenode.audit.log.async";
public static final boolean DFS_NAMENODE_AUDIT_LOG_ASYNC_DEFAULT = false;
+ public static final String DFS_NAMENODE_AUDIT_LOG_ASYNC_BLOCKING_KEY =
"dfs.namenode.audit.log.async.blocking";
+ public static final boolean DFS_NAMENODE_AUDIT_LOG_ASYNC_BLOCKING_DEFAULT =
true;
+ public static final String DFS_NAMENODE_AUDIT_LOG_ASYNC_BUFFER_SIZE_KEY =
"dfs.namenode.audit.log.async.buffer.size";
+ public static final int DFS_NAMENODE_AUDIT_LOG_ASYNC_BUFFER_SIZE_DEFAULT
= 128;
public static final String DFS_NAMENODE_AUDIT_LOG_DEBUG_CMDLIST =
"dfs.namenode.audit.log.debug.cmdlist";
public static final String DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_KEY =
"dfs.namenode.metrics.logger.period.seconds";
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 259c6a6..e531686 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -786,7 +786,7 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
if (conf.getBoolean(DFS_NAMENODE_AUDIT_LOG_ASYNC_KEY,
DFS_NAMENODE_AUDIT_LOG_ASYNC_DEFAULT)) {
LOG.info("Enabling async auditlog");
- enableAsyncAuditLog();
+ enableAsyncAuditLog(conf);
}
fsLock = new FSNamesystemLock(conf, detailedLockHoldTimeMetrics);
cond = fsLock.newWriteLockCondition();
@@ -8146,7 +8146,7 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
}
}
- private static void enableAsyncAuditLog() {
+ private static void enableAsyncAuditLog(Configuration conf) {
if (!(auditLog instanceof Log4JLogger)) {
LOG.warn("Log4j is required to enable async auditlog");
return;
@@ -8157,6 +8157,14 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
// failsafe against trying to async it more than once
if (!appenders.isEmpty() && !(appenders.get(0) instanceof AsyncAppender)) {
AsyncAppender asyncAppender = new AsyncAppender();
+ asyncAppender.setBlocking(conf.getBoolean(
+ DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_ASYNC_BLOCKING_KEY,
+ DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_ASYNC_BLOCKING_DEFAULT
+ ));
+ asyncAppender.setBufferSize(conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_ASYNC_BUFFER_SIZE_KEY,
+ DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_ASYNC_BUFFER_SIZE_DEFAULT
+ ));
// change logger to have an async appender containing all the
// previously configured appenders
for (Appender appender : appenders) {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 3a9bf79..8375cff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4368,6 +4368,27 @@
</property>
<property>
+ <name>dfs.namenode.audit.log.async.blocking</name>
+ <value>true</value>
+ <description>
+ Only used when enables asynchronous audit log. Sets whether audit log async
+ appender should wait if there is no space available in the event buffer or
+ immediately return. Default value is true.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.audit.log.async.buffer.size</name>
+ <value>128</value>
+ <description>
+ Only used when enables asynchronous audit log. Sets the number of audit
+ logs allowed in the event buffer before the calling thread is blocked
+ (if dfs.namenode.audit.log.async.blocking is true) or until logs are
+ summarized and discarded. Default value is 128.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.audit.log.token.tracking.id</name>
<value>false</value>
<description>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
