This is an automated email from the ASF dual-hosted git repository.
ZanderXu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3a1fed5a5a7 HDFS-17924. Improve NameNode edit log rolling controls
(#8509)
3a1fed5a5a7 is described below
commit 3a1fed5a5a75489252ad99722105d63cff29cfae
Author: Felix Nguyen <[email protected]>
AuthorDate: Mon Jun 1 10:37:05 2026 +0700
HDFS-17924. Improve NameNode edit log rolling controls (#8509)
---
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 ++
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 67 ++++++++++++++++++----
.../hdfs/server/namenode/ha/EditLogTailer.java | 13 ++++-
.../src/main/resources/hdfs-default.xml | 19 ++++++
.../hdfs/server/namenode/TestEditLogAutoroll.java | 26 ++++++++-
5 files changed, 118 insertions(+), 12 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 9cfc65f3c9b..2fb2de5746f 100755
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -424,6 +424,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD_DEFAULT = 0.5f;
public static final String DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS
= "dfs.namenode.edit.log.autoroll.check.interval.ms";
public static final int
DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS_DEFAULT = 5*60*1000;
+ public static final String DFS_NAMENODE_EDIT_LOG_AUTOROLL_MAX_INTERVAL_MS =
+ "dfs.namenode.edit.log.autoroll.max.interval.ms";
+ public static final long
DFS_NAMENODE_EDIT_LOG_AUTOROLL_MAX_INTERVAL_MS_DEFAULT = 0;
public static final String
DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC =
"dfs.namenode.lazypersist.file.scrub.interval.sec";
public static final int
DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC_DEFAULT = 5 * 60;
@@ -1376,6 +1379,8 @@ public class DFSConfigKeys extends
CommonConfigurationKeys {
public static final boolean DFS_HA_STANDBY_CHECKPOINTS_DEFAULT = true;
public static final String DFS_HA_LOGROLL_PERIOD_KEY =
"dfs.ha.log-roll.period";
public static final int DFS_HA_LOGROLL_PERIOD_DEFAULT = 2 * 60; // 2m
+ public static final String DFS_HA_LOG_ROLL_ENABLED_KEY =
"dfs.ha.log-roll.enabled";
+ public static final boolean DFS_HA_LOG_ROLL_ENABLED_DEFAULT = true;
public static final String DFS_HA_TAILEDITS_PERIOD_KEY =
"dfs.ha.tail-edits.period";
public static final int DFS_HA_TAILEDITS_PERIOD_DEFAULT = 60; // 1m
public static final String DFS_HA_TAILEDITS_PERIOD_BACKOFF_MAX_KEY =
"dfs.ha.tail-edits.period.backoff-max";
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index bf205fc19e9..f225a189185 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -67,6 +67,8 @@
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS_DEFAULT;
+import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_MAX_INTERVAL_MS;
+import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_MAX_INTERVAL_MS_DEFAULT;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD_DEFAULT;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ENABLE_RETRY_CACHE_DEFAULT;
@@ -569,6 +571,7 @@ private boolean isFromProxyUser(CallerContext ctx) {
Daemon nnrmthread = null; // NamenodeResourceMonitor thread
+ private NameNodeEditLogRoller nnEditLogRollerInt;
Daemon nnEditLogRoller = null; // NameNodeEditLogRoller thread
// A daemon to periodically clean up corrupt lazyPersist files
@@ -596,6 +599,12 @@ private boolean isFromProxyUser(CallerContext ctx) {
* Check interval of an active namenode's edit log roller thread
*/
private final int editLogRollerInterval;
+ /**
+ * Max interval between each active edit log roll, regardless of transaction
threshold (as long
+ * as there are transactions). Ensures active NN always rolls at least once
after a certain
+ * duration. Disabled when set to 0 or negative.
+ */
+ private final long editLogRollerMaxIntervalMs;
/**
* How frequently we scan and unlink corrupt lazyPersist files.
@@ -1006,6 +1015,9 @@ static FSNamesystem loadFromDisk(Configuration conf)
throws IOException {
this.editLogRollerInterval = conf.getInt(
DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS,
DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS_DEFAULT);
+ this.editLogRollerMaxIntervalMs = conf.getLong(
+ DFS_NAMENODE_EDIT_LOG_AUTOROLL_MAX_INTERVAL_MS,
+ DFS_NAMENODE_EDIT_LOG_AUTOROLL_MAX_INTERVAL_MS_DEFAULT);
this.lazyPersistFileScrubIntervalSec = conf.getInt(
DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC,
@@ -1476,8 +1488,10 @@ void startActiveServices() throws IOException {
this.nnrmthread = new Daemon(new NameNodeResourceMonitor());
nnrmthread.start();
- nnEditLogRoller = new Daemon(new NameNodeEditLogRoller(
- editLogRollerThreshold, editLogRollerInterval));
+ nnEditLogRollerInt = new NameNodeEditLogRoller(
+ editLogRollerThreshold, editLogRollerInterval,
+ editLogRollerMaxIntervalMs);
+ nnEditLogRoller = new Daemon(nnEditLogRollerInt);
nnEditLogRoller.start();
if (lazyPersistFileScrubIntervalSec > 0) {
@@ -1510,6 +1524,16 @@ void startActiveServices() throws IOException {
}
}
+ @VisibleForTesting
+ public void setLastRollTime(long lastRoll) {
+ nnEditLogRollerInt.lastRollMs = lastRoll;
+ }
+
+ @VisibleForTesting
+ public long getLastRollTime() {
+ return nnEditLogRollerInt.lastRollMs;
+ }
+
private boolean inActiveState() {
return haContext != null &&
haContext.getState().getServiceState() == HAServiceState.ACTIVE;
@@ -4629,22 +4653,29 @@ class NameNodeEditLogRoller implements Runnable {
private boolean shouldRun = true;
private final long rollThreshold;
private final long sleepIntervalMs;
-
- public NameNodeEditLogRoller(long rollThreshold, int sleepIntervalMs) {
- this.rollThreshold = rollThreshold;
+ private final long maxRollIntervalMs;
+ private long lastRollMs;
+
+ NameNodeEditLogRoller(long rollThreshold, int sleepIntervalMs, long
maxRollIntervalMs) {
+ this.rollThreshold = rollThreshold;
+ this.maxRollIntervalMs = maxRollIntervalMs;
+ this.lastRollMs = monotonicNow();
+ if (maxRollIntervalMs > 0) {
+ this.sleepIntervalMs = Math.min(sleepIntervalMs, maxRollIntervalMs);
+ } else {
this.sleepIntervalMs = sleepIntervalMs;
+ }
+ LOG.info("Initializing log roller with parameters rollThreshold={},
maxRollIntervalMs={}, "
+ + "sleepIntervalMs={}", rollThreshold, maxRollIntervalMs,
sleepIntervalMs);
}
@Override
public void run() {
while (fsRunning && shouldRun) {
try {
- long numEdits = getCorrectTransactionsSinceLastLogRoll();
- if (numEdits > rollThreshold) {
- FSNamesystem.LOG.info("NameNode rolling its own edit log because"
- + " number of edits in open segment exceeds threshold of "
- + rollThreshold);
+ if (shouldRoll()) {
rollEditLog();
+ lastRollMs = Time.monotonicNow();
}
} catch (Exception e) {
FSNamesystem.LOG.error("Swallowing exception in "
@@ -4660,6 +4691,22 @@ public void run() {
}
}
+ boolean tooLongSinceLastRoll(long timeSinceLastRollMs) {
+ return maxRollIntervalMs > 0 && timeSinceLastRollMs >= maxRollIntervalMs;
+ }
+
+ private boolean shouldRoll() {
+ long numEdits = getCorrectTransactionsSinceLastLogRoll();
+ long timeSinceLastRollMs = Time.monotonicNow() - lastRollMs;
+ if (numEdits > rollThreshold ||
(tooLongSinceLastRoll(timeSinceLastRollMs) && numEdits > 1)) {
+ FSNamesystem.LOG.info(
+ "Rolling edit logs: numEdits={}, threshold={},
timeSinceLastRollMs={}", numEdits,
+ rollThreshold, timeSinceLastRollMs);
+ return true;
+ }
+ return false;
+ }
+
public void stop() {
shouldRun = false;
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
index 472f1536526..0ac91ecf915 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
@@ -173,6 +173,12 @@ public class EditLogTailer {
*/
private final long maxTxnsPerLock;
+ /**
+ * Directly controls whether {@link #triggerActiveLogRoll} is allowed to run
+ * or not under the current {@link HAState}.
+ */
+ private final boolean triggerRollEnabled;
+
/**
* Timer instance to be set only using constructor.
* Only tests can reassign this by using setTimerForTests().
@@ -253,6 +259,11 @@ public EditLogTailer(FSNamesystem namesystem,
Configuration conf) {
maxRetries = DFSConfigKeys.DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_DEFAULT;
}
+ this.triggerRollEnabled =
conf.getBoolean(DFSConfigKeys.DFS_HA_LOG_ROLL_ENABLED_KEY,
+ DFSConfigKeys.DFS_HA_LOG_ROLL_ENABLED_DEFAULT);
+ LOG.info("Rolling is {} on this NN state {}", triggerRollEnabled ?
"enabled" : "disabled",
+ namesystem.getState());
+
inProgressOk = conf.getBoolean(
DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY,
DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_DEFAULT);
@@ -508,7 +519,7 @@ private void doWork() {
// read any more transactions since the last time a roll was
// triggered.
boolean triggeredLogRoll = false;
- if (tooLongSinceLastLoad() &&
+ if (triggerRollEnabled && tooLongSinceLastLoad() &&
lastRollTriggerTxId < lastLoadedTxnId) {
triggerActiveLogRoll();
triggeredLogRoll = true;
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 5da5e6f0c2f..1c56231fe26 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -1948,6 +1948,16 @@
Support multiple time unit suffix(case insensitive), as described
in dfs.heartbeat.interval.If no time unit is specified then seconds
is assumed.
+ Only works if dfs.ha.log-roll.enabled is set to true.
+ </description>
+</property>
+
+<property>
+ <name>dfs.ha.log-roll.enabled</name>
+ <value>true</value>
+ <description>
+ Whether Standby/Observer Namenodes are allowed to ask Active to roll edit
logs.
+ Takes priority over dfs.ha.log-roll.period.
</description>
</property>
@@ -3160,6 +3170,15 @@
</description>
</property>
+<property>
+ <name>dfs.namenode.edit.log.autoroll.max.interval.ms</name>
+ <value>0</value>
+ <description>
+ How long can an active namenode go before rolling its edit log (if there
is something to roll),
+ in milliseconds. Disabled by default.
+ </description>
+</property>
+
<property>
<name>dfs.webhdfs.user.provider.user.pattern</name>
<value>^[A-Za-z_][A-Za-z0-9._-]*[$]?$</value>
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java
index 375c3810bab..f8887d460d1 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java
@@ -25,7 +25,9 @@
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS;
+import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_MAX_INTERVAL_MS;
import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -73,6 +75,7 @@ public TestEditLogAutoroll(Boolean async) {
private FileSystem fs;
private FSEditLog editLog;
private final Random random = new Random();
+ private final static int SLEEP_TIME = 100;
public static final Logger LOG = LoggerFactory.getLogger(FSEditLog.class);
@@ -84,7 +87,8 @@ public void setUp() throws Exception {
conf.setLong(DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 20);
// Make it autoroll after 10 edits
conf.setFloat(DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD, 0.5f);
- conf.setInt(DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS, 100);
+ conf.setInt(DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS, SLEEP_TIME);
+ conf.setInt(DFS_NAMENODE_EDIT_LOG_AUTOROLL_MAX_INTERVAL_MS, 600000);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
useAsyncEditLog);
@@ -151,4 +155,24 @@ public Boolean get() {
GenericTestUtils.assertNoThreadsMatching(
".*" + NameNodeEditLogRoller.class.getSimpleName() + ".*");
}
+
+ @Test
+ public void testForceRoll() throws Exception {
+ // Write some stuff, manually roll
+ fs.mkdirs(new Path("testForceRollNoEdits1"));
+ FSNamesystem fsn = nn0.getNamesystem();
+ fsn.rollEditLog();
+ long lastRollTime = fsn.getLastRollTime();
+ final long startTxId = editLog.getCurSegmentTxId();
+ // Write some more stuff, force a roll by setting last roll time way back
in the past
+ fs.mkdirs(new Path("testForceRollNoEdits2"));
+ fsn.setLastRollTime(lastRollTime - 9000000);
+ GenericTestUtils.waitFor(() -> editLog.getCurSegmentTxId() > startTxId,
100, 5000);
+ final long newTxId = editLog.getCurSegmentTxId();
+ // Try to force another roll, but there's nothing to roll
+ fsn.setLastRollTime(lastRollTime - 9000000);
+ // Sleep 2 cycles to ensure auto roller has done the check at least once
+ Thread.sleep(SLEEP_TIME * 2);
+ assertEquals(newTxId, editLog.getCurSegmentTxId());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
