Repository: hadoop Updated Branches: refs/heads/HDFS-7285 a42ffa8b7 -> edbe633c0 (forced update)
HDFS-7829. Code clean up for LocatedBlock. Contributed by Takanobu Asanuma. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a6a5aae4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a6a5aae4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a6a5aae4 Branch: refs/heads/HDFS-7285 Commit: a6a5aae472d015d2ea5cd746719485dff93873a8 Parents: 6bc7710 Author: Jing Zhao <ji...@apache.org> Authored: Fri Mar 20 10:50:03 2015 -0700 Committer: Jing Zhao <ji...@apache.org> Committed: Fri Mar 20 10:50:19 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hadoop/hdfs/protocol/LocatedBlock.java | 20 ++++++-------------- .../hadoop/hdfs/protocol/LocatedBlocks.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 2 +- .../server/protocol/BlockRecoveryCommand.java | 4 ++-- .../hadoop/hdfs/TestDFSClientRetries.java | 5 ++--- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 8 ++++++-- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 3 ++- 8 files changed, 22 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6a5aae4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 52fbeff..418eee6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -326,6 +326,8 @@ Release 2.8.0 - UNRELEASED HDFS-7835. make initial sleeptime in locateFollowingBlock configurable for DFSClient. (Zhihai Xu via Yongjun Zhang) + HDFS-7829. Code clean up for LocatedBlock. (Takanobu Asanuma via jing9) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6a5aae4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java index 0d52191..e729869 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java @@ -44,9 +44,9 @@ public class LocatedBlock { private long offset; // offset of the first byte of the block in the file private final DatanodeInfoWithStorage[] locs; /** Cached storage ID for each replica */ - private String[] storageIDs; + private final String[] storageIDs; /** Cached storage type for each replica, if reported. */ - private StorageType[] storageTypes; + private final StorageType[] storageTypes; // corrupt flag is true if all of the replicas of a block are corrupt. // else false. If block has few corrupt replicas, they are filtered and // their locations are not part of this object @@ -62,16 +62,8 @@ public class LocatedBlock { new DatanodeInfoWithStorage[0]; public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs) { - this(b, locs, -1, false); // startOffset is unknown - } - - public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, long startOffset, - boolean corrupt) { - this(b, locs, null, null, startOffset, corrupt, EMPTY_LOCS); - } - - public LocatedBlock(ExtendedBlock b, DatanodeStorageInfo[] storages) { - this(b, storages, -1, false); // startOffset is unknown + // By default, startOffset is unknown(-1) and corrupt is false. + this(b, locs, null, null, -1, false, EMPTY_LOCS); } public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, @@ -170,11 +162,11 @@ public class LocatedBlock { return b.getNumBytes(); } - void setStartOffset(long value) { + public void setStartOffset(long value) { this.offset = value; } - void setCorrupt(boolean corrupt) { + public void setCorrupt(boolean corrupt) { this.corrupt = corrupt; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6a5aae4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java index fc739cf..e35a431 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java @@ -119,7 +119,7 @@ public class LocatedBlocks { public int findBlock(long offset) { // create fake block of size 0 as a key LocatedBlock key = new LocatedBlock( - new ExtendedBlock(), new DatanodeInfo[0], 0L, false); + new ExtendedBlock(), new DatanodeInfo[0]); key.setStartOffset(offset); key.getBlock().setNumBytes(1); Comparator<LocatedBlock> comp = http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6a5aae4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 3aefbf6..45b0190 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3292,7 +3292,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, final DatanodeStorageInfo[] targets = blockManager.chooseTarget4AdditionalDatanode( src, numAdditionalNodes, clientnode, chosen, excludes, preferredblocksize, storagePolicyID); - final LocatedBlock lb = new LocatedBlock(blk, targets); + final LocatedBlock lb = new LocatedBlock(blk, targets, -1, false); blockManager.setBlockToken(lb, AccessMode.COPY); return lb; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6a5aae4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java index ced3296..3adc85c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java @@ -61,7 +61,7 @@ public class BlockRecoveryCommand extends DatanodeCommand { * Create RecoveringBlock. */ public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs, long newGS) { - super(b, locs, -1, false); // startOffset is unknown + super(b, locs); // startOffset is unknown this.newGenerationStamp = newGS; this.recoveryBlock = null; } @@ -71,7 +71,7 @@ public class BlockRecoveryCommand extends DatanodeCommand { */ public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs, Block recoveryBlock) { - super(b, locs, -1, false); // startOffset is unknown + super(b, locs); // startOffset is unknown this.newGenerationStamp = recoveryBlock.getGenerationStamp(); this.recoveryBlock = recoveryBlock; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6a5aae4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index 73b9fbd..3912774 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@ -489,9 +489,8 @@ public class TestDFSClientRetries { goodLocatedBlock.getBlock(), new DatanodeInfo[] { DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234) - }, - goodLocatedBlock.getStartOffset(), - false); + }); + badLocatedBlock.setStartOffset(goodLocatedBlock.getStartOffset()); List<LocatedBlock> badBlocks = new ArrayList<LocatedBlock>(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6a5aae4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index 046265f..7de121f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -99,11 +99,15 @@ public class TestDFSUtil { // ok ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 1, 1); - LocatedBlock l1 = new LocatedBlock(b1, ds, 0, false); + LocatedBlock l1 = new LocatedBlock(b1, ds); + l1.setStartOffset(0); + l1.setCorrupt(false); // corrupt ExtendedBlock b2 = new ExtendedBlock("bpid", 2, 1, 1); - LocatedBlock l2 = new LocatedBlock(b2, ds, 0, true); + LocatedBlock l2 = new LocatedBlock(b2, ds); + l2.setStartOffset(0); + l2.setCorrupt(true); List<LocatedBlock> ls = Arrays.asList(l1, l2); LocatedBlocks lbs = new LocatedBlocks(10, false, ls, l2, true, null); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6a5aae4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index 0236288..c7233bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -478,10 +478,11 @@ public class TestPBHelper { AdminStates.NORMAL) }; LocatedBlock lb = new LocatedBlock( - new ExtendedBlock("bp12", 12345, 10, 53), dnInfos, 5, false); + new ExtendedBlock("bp12", 12345, 10, 53), dnInfos); lb.setBlockToken(new Token<BlockTokenIdentifier>( "identifier".getBytes(), "password".getBytes(), new Text("kind"), new Text("service"))); + lb.setStartOffset(5); return lb; }
