[1/2] hadoop git commit: HDFS-13653. Make dfs.client.failover.random.order a per nameservice configuration. Contributed by Ekanth Sethuramalingam.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 4049fa37a -> 805939079
refs/heads/branch-2.9 0aa4067e9 -> 88141548d
HDFS-13653. Make dfs.client.failover.random.order a per nameservice
configuration. Contributed by Ekanth Sethuramalingam.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80593907
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80593907
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80593907
Branch: refs/heads/branch-2
Commit: 8059390798b63589c313415a4bc175293edac529
Parents: 4049fa3
Author: Inigo Goiri
Authored: Mon Jun 11 13:47:08 2018 -0700
Committer: Inigo Goiri
Committed: Mon Jun 11 13:47:08 2018 -0700
--
.../ha/ConfiguredFailoverProxyProvider.java | 29 +-
.../ha/TestConfiguredFailoverProxyProvider.java | 264 +++
.../src/main/resources/hdfs-default.xml | 12 +
.../hadoop/tools/TestHdfsConfigFields.java | 1 +
4 files changed, 303 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80593907/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
index 58f4943..96722fc 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
@@ -94,9 +94,7 @@ public class ConfiguredFailoverProxyProvider extends
proxies.add(new AddressRpcProxyPair(address));
}
// Randomize the list to prevent all clients pointing to the same one
- boolean randomized = conf.getBoolean(
- HdfsClientConfigKeys.Failover.RANDOM_ORDER,
- HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT);
+ boolean randomized = getRandomOrder(conf, uri);
if (randomized) {
Collections.shuffle(proxies);
}
@@ -111,6 +109,31 @@ public class ConfiguredFailoverProxyProvider extends
}
}
+ /**
+ * Check whether random order is configured for failover proxy provider
+ * for the namenode/nameservice.
+ *
+ * @param conf Configuration
+ * @param nameNodeUri The URI of namenode/nameservice
+ * @return random order configuration
+ */
+ private static boolean getRandomOrder(
+ Configuration conf, URI nameNodeUri) {
+String host = nameNodeUri.getHost();
+String configKeyWithHost = HdfsClientConfigKeys.Failover.RANDOM_ORDER
++ "." + host;
+
+if (conf.get(configKeyWithHost) != null) {
+ return conf.getBoolean(
+ configKeyWithHost,
+ HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT);
+}
+
+return conf.getBoolean(
+HdfsClientConfigKeys.Failover.RANDOM_ORDER,
+HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT);
+ }
+
@Override
public Class getInterface() {
return xface;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80593907/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java
new file mode 100644
index 000..d7a5db6
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java
@@ -0,0 +1,264 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
[1/2] hadoop git commit: HDFS-13653. Make dfs.client.failover.random.order a per nameservice configuration. Contributed by Ekanth Sethuramalingam.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0 b87411027 -> 6eea6d28d
refs/heads/branch-3.1 e3c96354a -> f0150f024
HDFS-13653. Make dfs.client.failover.random.order a per nameservice
configuration. Contributed by Ekanth Sethuramalingam.
(cherry picked from commit 784dbbd2d1fc0c361fee851d3f9b3aa50be5481d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0150f02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0150f02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0150f02
Branch: refs/heads/branch-3.1
Commit: f0150f024f7e18e782666cede0b7e3c5881badd2
Parents: e3c9635
Author: Inigo Goiri
Authored: Mon Jun 11 11:16:52 2018 -0700
Committer: Inigo Goiri
Committed: Mon Jun 11 11:17:37 2018 -0700
--
.../ha/ConfiguredFailoverProxyProvider.java | 29 +-
.../ha/TestConfiguredFailoverProxyProvider.java | 264 +++
.../src/main/resources/hdfs-default.xml | 12 +
.../hadoop/tools/TestHdfsConfigFields.java | 1 +
4 files changed, 303 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0150f02/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
index 58f4943..96722fc 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java
@@ -94,9 +94,7 @@ public class ConfiguredFailoverProxyProvider extends
proxies.add(new AddressRpcProxyPair(address));
}
// Randomize the list to prevent all clients pointing to the same one
- boolean randomized = conf.getBoolean(
- HdfsClientConfigKeys.Failover.RANDOM_ORDER,
- HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT);
+ boolean randomized = getRandomOrder(conf, uri);
if (randomized) {
Collections.shuffle(proxies);
}
@@ -111,6 +109,31 @@ public class ConfiguredFailoverProxyProvider extends
}
}
+ /**
+ * Check whether random order is configured for failover proxy provider
+ * for the namenode/nameservice.
+ *
+ * @param conf Configuration
+ * @param nameNodeUri The URI of namenode/nameservice
+ * @return random order configuration
+ */
+ private static boolean getRandomOrder(
+ Configuration conf, URI nameNodeUri) {
+String host = nameNodeUri.getHost();
+String configKeyWithHost = HdfsClientConfigKeys.Failover.RANDOM_ORDER
++ "." + host;
+
+if (conf.get(configKeyWithHost) != null) {
+ return conf.getBoolean(
+ configKeyWithHost,
+ HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT);
+}
+
+return conf.getBoolean(
+HdfsClientConfigKeys.Failover.RANDOM_ORDER,
+HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT);
+ }
+
@Override
public Class getInterface() {
return xface;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0150f02/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java
new file mode 100644
index 000..d7a5db6
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java
@@ -0,0 +1,264 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS"
