This is an automated email from the ASF dual-hosted git repository.
ZanderXu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new bcef4eeee3d HDFS-17913. Dead DataNode in Host2NodesMap can break block
location sorting (#8462)
bcef4eeee3d is described below
commit bcef4eeee3da3042aa1dbe0dc385c39c277eae3b
Author: CapMoon <[email protected]>
AuthorDate: Wed May 6 10:31:15 2026 +0800
HDFS-17913. Dead DataNode in Host2NodesMap can break block location sorting
(#8462)
---
.../server/blockmanagement/DatanodeManager.java | 3 ++
.../blockmanagement/TestDatanodeManager.java | 51 ++++++++++++++++++++++
2 files changed, 54 insertions(+)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 01f1af9624d..27722bc1d8a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -637,6 +637,9 @@ private void sortLocatedBlock(final LocatedBlock lb, String
targetHost,
// here we should get node but not datanode only .
boolean nonDatanodeReader = false;
Node client = getDatanodeByHost(targetHost);
+ if (client != null && client.getParent() == null) {
+ client = null;
+ }
if (client == null) {
nonDatanodeReader = true;
List<String> hosts = new ArrayList<>(1);
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
index fd0a2c33011..386d5b424b5 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
@@ -667,6 +667,57 @@ public void testGetBlockLocationConsiderLoad()
assertEquals(locs[0].getIpAddr(), sortedLocs2[4].getIpAddr());
}
+ @Test
+ public void testGetBlockLocationWithDeadDatanodeReader()
+ throws IOException, URISyntaxException {
+ Configuration conf = new Configuration();
+ conf.setBoolean(
+ DFSConfigKeys.DFS_NAMENODE_READ_CONSIDERLOAD_KEY, true);
+ conf.setBoolean(
+ DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY, true);
+ conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 0);
+ conf.setInt(
+ DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 0);
+ FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
+ Mockito.when(fsn.hasWriteLock()).thenReturn(true);
+ Mockito.when(fsn.hasWriteLock(RwLockMode.BM)).thenReturn(true);
+ URL shellScript = getClass().getResource(
+ "/" + Shell.appendScriptExtension("topology-script"));
+ Path resourcePath = Paths.get(shellScript.toURI());
+ FileUtil.setExecutable(resourcePath.toFile(), true);
+ conf.set(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY,
+ resourcePath.toString());
+ DatanodeManager dm = mockDatanodeManager(fsn, conf);
+
+ int totalDNs = 5;
+ DatanodeInfo[] locs = new DatanodeInfo[totalDNs];
+ for (int i = 0; i < totalDNs; i++) {
+ String uuid = "UUID-" + i;
+ String ip = "IP-" + i / 2 + "-" + i;
+ DatanodeRegistration dr = Mockito.mock(DatanodeRegistration.class);
+ Mockito.when(dr.getDatanodeUuid()).thenReturn(uuid);
+ Mockito.when(dr.getIpAddr()).thenReturn(ip);
+ dm.registerDatanode(dr);
+ locs[i] = dm.getDatanode(uuid);
+ locs[i].setXceiverCount(i);
+ }
+
+ locs[3].setLastUpdateMonotonic(0);
+ dm.removeDeadDatanode(locs[3], true);
+ assertThat(locs[3].getParent()).isNull();
+ assertThat(dm.getDatanodeByHost(locs[3].getIpAddr())).isSameAs(locs[3]);
+
+ ExtendedBlock b = new ExtendedBlock("somePoolID", 1234);
+ LocatedBlock block = new LocatedBlock(b,
+ new DatanodeInfo[] {locs[1], locs[2], locs[4]});
+ List<LocatedBlock> blocks = new ArrayList<>();
+ blocks.add(block);
+
+ dm.sortLocatedBlocks(locs[3].getIpAddr(), blocks);
+ DatanodeInfo[] sortedLocs = block.getLocations();
+ assertEquals(locs[2].getIpAddr(), sortedLocs[0].getIpAddr());
+ }
+
@Test
public void testGetBlockLocationConsiderLoadWithNodesOfSameDistance()
throws IOException, URISyntaxException {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
