This is an automated email from the ASF dual-hosted git repository.
yqlin pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new de2cb86 HDFS-15410. Add separated config file
hdfs-fedbalance-default.xml for fedbalance tool. Contributed by Jinglun.
de2cb86 is described below
commit de2cb8626016f22b388da7796082b2e160059cf6
Author: Yiqun Lin <yq...@apache.org>
AuthorDate: Wed Jul 1 14:06:27 2020 +0800
HDFS-15410. Add separated config file hdfs-fedbalance-default.xml for
fedbalance tool. Contributed by Jinglun.
---
.../apache/hadoop/tools/fedbalance/FedBalance.java | 48 ++++++++++++++--------
.../hadoop/tools/fedbalance/FedBalanceConfigs.java | 17 ++++----
...pBalanceOptions.java => FedBalanceOptions.java} | 16 ++++----
.../procedure/BalanceProcedureScheduler.java | 7 +---
.../src/main/resources/hdfs-fedbalance-default.xml | 41 ++++++++++++++++++
.../hadoop/tools/fedbalance/TestFedBalance.java | 34 +++++++++++++++
6 files changed, 125 insertions(+), 38 deletions(-)
diff --git
a/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/FedBalance.java
b/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/FedBalance.java
index adfb40b..8252957 100644
---
a/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/FedBalance.java
+++
b/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/FedBalance.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.tools.fedbalance;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
@@ -25,7 +26,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.tools.fedbalance.procedure.BalanceProcedure;
import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
@@ -34,7 +34,6 @@ import
org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
import org.apache.hadoop.tools.fedbalance.procedure.BalanceJob;
import org.apache.hadoop.tools.fedbalance.procedure.BalanceProcedureScheduler;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.slf4j.Logger;
@@ -45,14 +44,13 @@ import java.net.InetSocketAddress;
import java.util.Collection;
import java.util.concurrent.TimeUnit;
-import static org.apache.hadoop.tools.fedbalance.DistCpBalanceOptions.ROUTER;
-import static
org.apache.hadoop.tools.fedbalance.DistCpBalanceOptions.FORCE_CLOSE_OPEN;
-import static org.apache.hadoop.tools.fedbalance.DistCpBalanceOptions.MAP;
-import static
org.apache.hadoop.tools.fedbalance.DistCpBalanceOptions.BANDWIDTH;
-import static org.apache.hadoop.tools.fedbalance.DistCpBalanceOptions.TRASH;
-import static
org.apache.hadoop.tools.fedbalance.DistCpBalanceOptions.DELAY_DURATION;
-import static
org.apache.hadoop.tools.fedbalance.DistCpBalanceOptions.CLI_OPTIONS;
-import static
org.apache.hadoop.tools.fedbalance.FedBalanceConfigs.FEDERATION_BALANCE_CLASS;
+import static org.apache.hadoop.tools.fedbalance.FedBalanceOptions.ROUTER;
+import static
org.apache.hadoop.tools.fedbalance.FedBalanceOptions.FORCE_CLOSE_OPEN;
+import static org.apache.hadoop.tools.fedbalance.FedBalanceOptions.MAP;
+import static org.apache.hadoop.tools.fedbalance.FedBalanceOptions.BANDWIDTH;
+import static org.apache.hadoop.tools.fedbalance.FedBalanceOptions.TRASH;
+import static
org.apache.hadoop.tools.fedbalance.FedBalanceOptions.DELAY_DURATION;
+import static org.apache.hadoop.tools.fedbalance.FedBalanceOptions.CLI_OPTIONS;
import static org.apache.hadoop.tools.fedbalance.FedBalanceConfigs.TrashOption;
/**
@@ -73,6 +71,10 @@ public class FedBalance extends Configured implements Tool {
private static final String MOUNT_TABLE_PROCEDURE = "mount-table-procedure";
private static final String TRASH_PROCEDURE = "trash-procedure";
+ private static final String FED_BALANCE_DEFAULT_XML =
+ "hdfs-fedbalance-default.xml";
+ private static final String FED_BALANCE_SITE_XML =
"hdfs-fedbalance-site.xml";
+
/**
* This class helps building the balance job.
*/
@@ -210,7 +212,7 @@ public class FedBalance extends Configured implements Tool {
public int run(String[] args) throws Exception {
CommandLineParser parser = new GnuParser();
CommandLine command =
- parser.parse(DistCpBalanceOptions.CLI_OPTIONS, args, true);
+ parser.parse(FedBalanceOptions.CLI_OPTIONS, args, true);
String[] leftOverArgs = command.getArgs();
if (leftOverArgs == null || leftOverArgs.length < 1) {
printUsage();
@@ -356,18 +358,32 @@ public class FedBalance extends Configured implements
Tool {
}
/**
+ * Loads properties from hdfs-fedbalance-default.xml into configuration
+ * object.
+ *
+ * @return Configuration which includes properties from
+ * hdfs-fedbalance-default.xml and hdfs-fedbalance-site.xml
+ */
+ @VisibleForTesting
+ static Configuration getDefaultConf() {
+ Configuration config = new Configuration();
+ config.addResource(FED_BALANCE_DEFAULT_XML);
+ config.addResource(FED_BALANCE_SITE_XML);
+ return config;
+ }
+
+ /**
* Main function of the FedBalance program. Parses the input arguments and
* invokes the FedBalance::run() method, via the ToolRunner.
* @param argv Command-line arguments sent to FedBalance.
*/
public static void main(String[] argv) {
- Configuration conf = new HdfsConfiguration();
- Class<Tool> balanceClazz = (Class<Tool>) conf
- .getClass(FEDERATION_BALANCE_CLASS, FedBalance.class);
- Tool balancer = ReflectionUtils.newInstance(balanceClazz, conf);
+ Configuration conf = getDefaultConf();
+ FedBalance fedBalance = new FedBalance();
+ fedBalance.setConf(conf);
int exitCode;
try {
- exitCode = ToolRunner.run(balancer, argv);
+ exitCode = ToolRunner.run(fedBalance, argv);
} catch (Exception e) {
LOG.warn("Couldn't complete FedBalance operation.", e);
exitCode = -1;
diff --git
a/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/FedBalanceConfigs.java
b/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/FedBalanceConfigs.java
index 952aef2..efe906b 100644
---
a/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/FedBalanceConfigs.java
+++
b/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/FedBalanceConfigs.java
@@ -24,9 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
*/
@InterfaceAudience.Private
public final class FedBalanceConfigs {
- /* The class used for federation balance */
- public static final String FEDERATION_BALANCE_CLASS =
- "federation.balance.class";
+
public static final String LAST_SNAPSHOT_NAME = "DISTCP-BALANCE-CURRENT";
public static final String CURRENT_SNAPSHOT_NAME = "DISTCP-BALANCE-NEXT";
/* Specify the behaviour of trash. */
@@ -34,17 +32,18 @@ public final class FedBalanceConfigs {
TRASH, DELETE, SKIP
}
- /* The worker threads number of the BalanceProcedureScheduler */
+ /* The worker threads number of the BalanceProcedureScheduler.
+ BalanceProcedureScheduler is responsible for scheduling a balance job,
+ including submit, run, delay and recover. */
public static final String WORK_THREAD_NUM =
- "hadoop.hdfs.procedure.work.thread.num";
+ "hdfs.fedbalance.procedure.work.thread.num";
public static final int WORK_THREAD_NUM_DEFAULT = 10;
- /* The uri of the journal */
+ /* The uri of the journal, the journal file is used for handling the job
+ persistence and recover. */
public static final String SCHEDULER_JOURNAL_URI =
- "hadoop.hdfs.procedure.scheduler.journal.uri";
+ "hdfs.fedbalance.procedure.scheduler.journal.uri";
public static final String JOB_PREFIX = "JOB-";
public static final String TMP_TAIL = ".tmp";
- public static final String JOURNAL_CLASS =
- "hadoop.hdfs.procedure.journal.class";
private FedBalanceConfigs(){}
}
diff --git
a/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/DistCpBalanceOptions.java
b/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/FedBalanceOptions.java
similarity index 86%
rename from
hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/DistCpBalanceOptions.java
rename to
hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/FedBalanceOptions.java
index 704ffd9..71a7d9d 100644
---
a/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/DistCpBalanceOptions.java
+++
b/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/FedBalanceOptions.java
@@ -23,23 +23,23 @@ import org.apache.commons.cli.Options;
/**
* Command line options of FedBalance.
*/
-public final class DistCpBalanceOptions {
+public final class FedBalanceOptions {
/**
* The private construct protects this class from being instantiated.
*/
- private DistCpBalanceOptions() {}
+ private FedBalanceOptions() {}
/**
* Run in router-based federation mode.
*/
final static Option ROUTER = new Option("router", false,
- "If `true` the command runs in router mode. The source path is "
- + "taken as a mount point. It will disable write by setting the
mount"
- + " point readonly. Otherwise the command works in normal federation"
- + " mode. The source path is taken as the full path. It will disable"
- + " write by cancelling all permissions of the source path. The"
- + " default value is `true`.");
+ "If this option is set then the command runs in router mode."
+ + " The source path is taken as a mount point. It will disable write"
+ + " by setting the mount point readonly. Otherwise the command works"
+ + " in normal federation mode. The source path is taken as the full"
+ + " path. It will disable write by cancelling all permissions of the"
+ + " source path.");
/**
* If true, in DIFF_DISTCP stage it will force close all open files when
diff --git
a/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/procedure/BalanceProcedureScheduler.java
b/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/procedure/BalanceProcedureScheduler.java
index a27db10..d7e1c7c 100644
---
a/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/procedure/BalanceProcedureScheduler.java
+++
b/hadoop-tools/hadoop-federation-balance/src/main/java/org/apache/hadoop/tools/fedbalance/procedure/BalanceProcedureScheduler.java
@@ -35,14 +35,12 @@ import org.apache.commons.lang3.builder.CompareToBuilder;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static
org.apache.hadoop.tools.fedbalance.FedBalanceConfigs.WORK_THREAD_NUM;
import static
org.apache.hadoop.tools.fedbalance.FedBalanceConfigs.WORK_THREAD_NUM_DEFAULT;
-import static
org.apache.hadoop.tools.fedbalance.FedBalanceConfigs.JOURNAL_CLASS;
/**
* <pre>
* The state machine framework consist of:
@@ -115,9 +113,8 @@ public class BalanceProcedureScheduler {
this.readerThread.start();
// init journal.
- Class<BalanceJournal> clazz = (Class<BalanceJournal>) conf
- .getClass(JOURNAL_CLASS, BalanceJournalInfoHDFS.class);
- journal = ReflectionUtils.newInstance(clazz, conf);
+ journal = new BalanceJournalInfoHDFS();
+ journal.setConf(conf);
if (recoverJobs) {
recoverAllJobs();
diff --git
a/hadoop-tools/hadoop-federation-balance/src/main/resources/hdfs-fedbalance-default.xml
b/hadoop-tools/hadoop-federation-balance/src/main/resources/hdfs-fedbalance-default.xml
new file mode 100644
index 0000000..d769832
--- /dev/null
+++
b/hadoop-tools/hadoop-federation-balance/src/main/resources/hdfs-fedbalance-default.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Do not modify this file directly. Anything that need to be overwritten,
+ need to be done so, through -D switches or customized conf -->
+
+<configuration>
+
+ <property>
+ <name>hdfs.fedbalance.procedure.scheduler.journal.uri</name>
+ <value>hdfs://localhost:8020/tmp/procedure</value>
+ <description>
+ The uri of the journal, the journal file is used for handling the
+ job persistence and recover.
+ </description>
+ </property>
+
+ <property>
+ <name>hdfs.fedbalance.procedure.work.thread.num</name>
+ <value>10</value>
+ <description>
+ The worker threads number of the BalanceProcedureScheduler.
+ BalanceProcedureScheduler is responsible for scheduling a balance
+ job, including submit, run, delay and recover.
+ </description>
+ </property>
+
+</configuration>
diff --git
a/hadoop-tools/hadoop-federation-balance/src/test/java/org/apache/hadoop/tools/fedbalance/TestFedBalance.java
b/hadoop-tools/hadoop-federation-balance/src/test/java/org/apache/hadoop/tools/fedbalance/TestFedBalance.java
new file mode 100644
index 0000000..0d29e6f
--- /dev/null
+++
b/hadoop-tools/hadoop-federation-balance/src/test/java/org/apache/hadoop/tools/fedbalance/TestFedBalance.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools.fedbalance;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertNotNull;
+import static
org.apache.hadoop.tools.fedbalance.FedBalanceConfigs.SCHEDULER_JOURNAL_URI;
+import static
org.apache.hadoop.tools.fedbalance.FedBalanceConfigs.WORK_THREAD_NUM;
+
+public class TestFedBalance {
+ @Test
+ public void testLoadFedBalanceDefaultConf() {
+ Configuration conf = FedBalance.getDefaultConf();
+ assertNotNull(conf.get(SCHEDULER_JOURNAL_URI));
+ assertNotNull(conf.get(WORK_THREAD_NUM));
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
