Repository: hadoop Updated Branches: refs/heads/trunk 78a7e8d3a -> db6606223
HDFS-7795. Show warning if not all favored nodes were chosen by namenode. Contributed by Kihwal Lee. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db660622 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db660622 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db660622 Branch: refs/heads/trunk Commit: db6606223ca2e17aa7e1b2e2be13c1a19d8e7465 Parents: 78a7e8d Author: Kihwal Lee <kih...@apache.org> Authored: Tue Feb 17 13:05:43 2015 -0600 Committer: Kihwal Lee <kih...@apache.org> Committed: Tue Feb 17 13:05:43 2015 -0600 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/DFSOutputStream.java | 26 ++++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/db660622/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5e54731..48eb61c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -644,6 +644,9 @@ Release 2.7.0 - UNRELEASED HDFS-4266. BKJM: Separate write and ack quorum (Rakesh R via umamahesh) + HDFS-7795. Show warning if not all favored nodes were chosen by namenode + (kihwal) + OPTIMIZATIONS HDFS-7454. Reduce memory footprint for AclEntries in NameNode. http://git-wip-us.apache.org/repos/asf/hadoop/blob/db660622/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 7d29b3d..85d3410 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -35,6 +35,7 @@ import java.nio.channels.ClosedChannelException; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -1443,7 +1444,7 @@ public class DFSOutputStream extends FSOutputSummer ExtendedBlock blockCopy = new ExtendedBlock(block); blockCopy.setNumBytes(blockSize); - boolean[] targetPinnings = getPinnings(nodes); + boolean[] targetPinnings = getPinnings(nodes, true); // send the request new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken, dfsClient.clientName, nodes, nodeStorageTypes, null, bcs, @@ -1537,20 +1538,29 @@ public class DFSOutputStream extends FSOutputSummer } } - private boolean[] getPinnings(DatanodeInfo[] nodes) { + private boolean[] getPinnings(DatanodeInfo[] nodes, boolean shouldLog) { if (favoredNodes == null) { return null; } else { boolean[] pinnings = new boolean[nodes.length]; + HashSet<String> favoredSet = + new HashSet<String>(Arrays.asList(favoredNodes)); for (int i = 0; i < nodes.length; i++) { - pinnings[i] = false; - for (int j = 0; j < favoredNodes.length; j++) { - if (nodes[i].getXferAddrWithHostname().equals(favoredNodes[j])) { - pinnings[i] = true; - break; - } + pinnings[i] = favoredSet.remove(nodes[i].getXferAddrWithHostname()); + if (DFSClient.LOG.isDebugEnabled()) { + DFSClient.LOG.debug(nodes[i].getXferAddrWithHostname() + + " was chosen by name node (favored=" + pinnings[i] + + ")."); } } + if (shouldLog && !favoredSet.isEmpty()) { + // There is one or more favored nodes that were not allocated. + DFSClient.LOG.warn( + "These favored nodes were specified but not chosen: " + + favoredSet + + " Specified favored nodes: " + Arrays.toString(favoredNodes)); + + } return pinnings; } }