This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1189af4 HDFS-15263. Fix the logic of scope and excluded scope in
Network Topology. Contributed by Ayush Saxena.
1189af4 is described below
commit 1189af4746919774035f5d64ccb4d2ce21905aaa
Author: Ayush Saxena <[email protected]>
AuthorDate: Wed Apr 8 15:26:03 2020 +0530
HDFS-15263. Fix the logic of scope and excluded scope in Network Topology.
Contributed by Ayush Saxena.
---
.../org/apache/hadoop/net/NetworkTopology.java | 36 +++++++++++++++++++---
.../org/apache/hadoop/net/TestClusterTopology.java | 5 +++
.../apache/hadoop/hdfs/net/DFSNetworkTopology.java | 13 ++------
3 files changed, 39 insertions(+), 15 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index aae56dd..199c8f0 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -487,10 +487,10 @@ public class NetworkTopology {
protected Node chooseRandom(final String scope, String excludedScope,
final Collection<Node> excludedNodes) {
if (excludedScope != null) {
- if (scope.startsWith(excludedScope)) {
+ if (isChildScope(scope, excludedScope)) {
return null;
}
- if (!excludedScope.startsWith(scope)) {
+ if (!isChildScope(excludedScope, scope)) {
excludedScope = null;
}
}
@@ -668,8 +668,7 @@ public class NetworkTopology {
if (node == null) {
continue;
}
- if ((NodeBase.getPath(node) + NodeBase.PATH_SEPARATOR_STR)
- .startsWith(scope + NodeBase.PATH_SEPARATOR_STR)) {
+ if (isNodeInScope(node, scope)) {
if (node instanceof InnerNode) {
excludedCountInScope += ((InnerNode) node).getNumOfLeaves();
} else {
@@ -994,4 +993,33 @@ public class NetworkTopology {
Preconditions.checkState(idx == activeLen,
"Sorted the wrong number of nodes!");
}
+
+ /**
+ * Checks whether one scope is contained in the other scope.
+ * @param parentScope the parent scope to check
+ * @param childScope the child scope which needs to be checked.
+ * @return true if childScope is contained within the parentScope
+ */
+ protected static boolean isChildScope(final String parentScope,
+ final String childScope) {
+ String pScope = parentScope.endsWith(NodeBase.PATH_SEPARATOR_STR) ?
+ parentScope : parentScope + NodeBase.PATH_SEPARATOR_STR;
+ String cScope = childScope.endsWith(NodeBase.PATH_SEPARATOR_STR) ?
+ childScope : childScope + NodeBase.PATH_SEPARATOR_STR;
+ return pScope.startsWith(cScope);
+ }
+
+ /**
+ * Checks whether a node belongs to the scope.
+ * @param node the node to check.
+ * @param scope scope to check.
+ * @return true if node lies within the scope
+ */
+ protected static boolean isNodeInScope(Node node, String scope) {
+ if (!scope.endsWith(NodeBase.PATH_SEPARATOR_STR)) {
+ scope += NodeBase.PATH_SEPARATOR_STR;
+ }
+ String nodeLocation = NodeBase.getPath(node) + NodeBase.PATH_SEPARATOR_STR;
+ return nodeLocation.startsWith(scope);
+ }
}
\ No newline at end of file
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
index fbed605..6d9dc77 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.net;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Arrays;
@@ -193,6 +194,10 @@ public class TestClusterTopology extends Assert {
}
assertEquals("Random is not selecting the nodes it should",
2, histogram.size());
+
+ Node val = cluster.chooseRandom("/d1", "/d", Collections.emptyList());
+ assertNotNull(val);
+
}
@Test
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
index 9082b91..e4e4350 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
@@ -180,10 +180,10 @@ public class DFSNetworkTopology extends NetworkTopology {
String excludedScope, final Collection<Node> excludedNodes,
StorageType type) {
if (excludedScope != null) {
- if (scope.startsWith(excludedScope)) {
+ if (isChildScope(scope, excludedScope)) {
return null;
}
- if (!excludedScope.startsWith(scope)) {
+ if (!isChildScope(excludedScope, scope)) {
excludedScope = null;
}
}
@@ -264,15 +264,6 @@ public class DFSNetworkTopology extends NetworkTopology {
return chosen;
}
- private boolean isNodeInScope(Node node, String scope) {
- if (!scope.endsWith(NodeBase.PATH_SEPARATOR_STR)) {
- scope += NodeBase.PATH_SEPARATOR_STR;
- }
- String nodeLocation =
- node.getNetworkLocation() + NodeBase.PATH_SEPARATOR_STR;
- return nodeLocation.startsWith(scope);
- }
-
/**
* Choose a random node that has the required storage type, under the given
* root, with an excluded subtree root (could also just be a leaf node).
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
