This is an automated email from the ASF dual-hosted git repository. shv pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push: new 76ca864 HDFS-14806. Bootstrap standby may fail if with in-progress tailing. Contributed by Chen Liang and Erik Krogen. 76ca864 is described below commit 76ca86440672a7067ee582ae851c1686530c7c72 Author: Chen Liang <cli...@apache.org> AuthorDate: Wed Nov 6 09:30:04 2019 -0800 HDFS-14806. Bootstrap standby may fail if with in-progress tailing. Contributed by Chen Liang and Erik Krogen. --- .../hdfs/server/namenode/ha/BootstrapStandby.java | 4 ++ .../TestBootstrapStandbyWithInProgressTailing.java | 46 ++++++++++++++++++++++ .../namenode/ha/TestBootstrapStandbyWithQJM.java | 9 ++++- 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java index 4d6716f..e69baf3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java @@ -37,6 +37,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.HAUtil; @@ -100,6 +101,9 @@ public class BootstrapStandby implements Tool, Configurable { @Override public int run(String[] args) throws Exception { parseArgs(args); + // Disable using the RPC tailing mechanism for bootstrapping the standby + // since it is less efficient in this case; see HDFS-14806 + conf.setBoolean(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, false); parseConfAndFindOtherNN(); NameNode.checkAllowFormat(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithInProgressTailing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithInProgressTailing.java new file mode 100644 index 0000000..a18e8ac --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithInProgressTailing.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.ha; + +import org.apache.hadoop.conf.Configuration; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY; + + +/** + * Test in progress tailing with small txn id per call. + * + * The number of edits that needs to be tailed during + * bootstrapStandby can be large, but the number of edits + * that can be tailed using RPC call can be limited + * (configured by dfs.ha.tail-edits.qjm.rpc.max-txns). + * This is to test that even with small number of configured + * txnid, bootstrapStandby can still work. See HDFS-14806. + */ +public class TestBootstrapStandbyWithInProgressTailing + extends TestBootstrapStandbyWithQJM { + @Override + public Configuration createConfig() { + Configuration conf = super.createConfig(); + + conf.setBoolean(DFS_HA_TAILEDITS_INPROGRESS_KEY, true); + conf.setInt("dfs.ha.tail-edits.qjm.rpc.max-txns", 1); + + return conf; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java index 10194ba..aad01f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java @@ -54,14 +54,19 @@ public class TestBootstrapStandbyWithQJM { private MiniJournalCluster jCluster; private int nnCount = 3; - @Before - public void setup() throws Exception { + protected Configuration createConfig() { Configuration conf = new Configuration(); // Turn off IPC client caching, so that the suite can handle // the restart of the daemons between test cases. conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0); + return conf; + } + + @Before + public void setup() throws Exception { + Configuration conf = createConfig(); MiniQJMHACluster miniQjmHaCluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org