Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 27badf6df -> db726c448
HDFS-12603. Enable async edit logging by default. Contributed by Andrew Wang.
(cherry picked from commit d0311dfc5fe92a0e0f3fb8ae92e412edce609740)
(cherry picked from commit f6e768104b0e9a087584c25e28bf8cd29802c719)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db726c44
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db726c44
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db726c44
Branch: refs/heads/branch-2.8
Commit: db726c44833559efb4adb2bd350aa82a638cd02d
Parents: 27badf6
Author: Xiao Chen <[email protected]>
Authored: Mon Oct 16 09:43:39 2017 -0700
Committer: Xiao Chen <[email protected]>
Committed: Mon Oct 16 09:51:34 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 2 +-
.../src/main/resources/hdfs-default.xml | 10 ++++++++
.../namenode/ha/TestFailureToReadEdits.java | 25 ++++++++++++++------
4 files changed, 30 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db726c44/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index bb28d94..d262cb9 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -277,7 +277,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_EDITS_ASYNC_LOGGING =
"dfs.namenode.edits.asynclogging";
- public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = false;
+ public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = true;
public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
public static final int DFS_LIST_LIMIT_DEFAULT = 1000;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db726c44/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 7a392cf..01d2f54 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -126,7 +126,7 @@ public class FSEditLog implements LogsPurgeable {
*
* In a non-HA setup:
*
- * The log starts in UNITIALIZED state upon construction. Once it's
+ * The log starts in UNINITIALIZED state upon construction. Once it's
* initialized, it is usually in IN_SEGMENT state, indicating that edits may
* be written. In the middle of a roll, or while saving the namespace, it
* briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the
previous
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db726c44/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 5bfb1fc..20d1627 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -3061,6 +3061,16 @@
retries or failovers for WebHDFS client.
</description>
</property>
+
+<property>
+ <name>dfs.namenode.edits.asynclogging</name>
+ <value>true</value>
+ <description>
+ If set to true, enables asynchronous edit logs in the Namenode. If set
+ to false, the Namenode uses the traditional synchronous edit logs.
+ </description>
+</property>
+
<property>
<name>dfs.balancer.keytab.enabled</name>
<value>false</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db726c44/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
index 5221ef9..27ed405 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
@@ -75,6 +75,7 @@ public class TestFailureToReadEdits {
private static final Random RANDOM = new Random();
private final TestType clusterType;
+ private final boolean useAsyncEditLogging;
private Configuration conf;
private MiniDFSCluster cluster;
private MiniQJMHACluster miniQjmHaCluster; // for QJM case only
@@ -88,18 +89,26 @@ public class TestFailureToReadEdits {
};
/**
- * Run this suite of tests both for QJM-based HA and for file-based
- * HA.
+ * Run this suite of tests for {QJM-based, file-based HA} x {async
+ * edit logging enabled, disabled}.
+ *
+ * TODO: Enable the test cases with async edit logging on. See HDFS-12603
+ * and HDFS-12660.
*/
@Parameters
public static Iterable<Object[]> data() {
- return Arrays.asList(new Object[][] {
- { TestType.SHARED_DIR_HA },
- { TestType.QJM_HA } });
+ return Arrays.asList(new Object[][]{
+ {TestType.SHARED_DIR_HA, Boolean.FALSE},
+ //{TestType.SHARED_DIR_HA, Boolean.TRUE},
+ {TestType.QJM_HA, Boolean.FALSE},
+ //{TestType.QJM_HA, Boolean.TRUE},
+ });
}
-
- public TestFailureToReadEdits(TestType clusterType) {
+
+ public TestFailureToReadEdits(TestType clusterType, Boolean
+ useAsyncEditLogging) {
this.clusterType = clusterType;
+ this.useAsyncEditLogging = useAsyncEditLogging;
}
@Before
@@ -109,6 +118,8 @@ public class TestFailureToReadEdits {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+ useAsyncEditLogging);
HAUtil.setAllowStandbyReads(conf, true);
if (clusterType == TestType.SHARED_DIR_HA) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
