HDFS-8482. Rename BlockInfoContiguous to BlockInfo. Contributed by Zhe Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b88df72 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b88df72 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b88df72 Branch: refs/heads/YARN-2928 Commit: 8b88df723b75e8a81ed7db7599baeb4098059f46 Parents: 00c4b9d Author: Andrew Wang <w...@apache.org> Authored: Wed May 27 15:37:01 2015 -0700 Committer: Zhijie Shen <zjs...@apache.org> Committed: Tue Jun 2 15:46:59 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../server/blockmanagement/BlockCollection.java | 8 +- .../hdfs/server/blockmanagement/BlockInfo.java | 399 +++++++++++++++++++ .../blockmanagement/BlockInfoContiguous.java | 399 ------------------- .../BlockInfoContiguousUnderConstruction.java | 6 +- .../server/blockmanagement/BlockManager.java | 124 +++--- .../hdfs/server/blockmanagement/BlocksMap.java | 32 +- .../CacheReplicationMonitor.java | 14 +- .../blockmanagement/DatanodeDescriptor.java | 16 +- .../blockmanagement/DatanodeStorageInfo.java | 25 +- .../blockmanagement/DecommissionManager.java | 22 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 14 +- .../hdfs/server/namenode/FSDirectory.java | 6 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 8 +- .../hdfs/server/namenode/FSEditLogLoader.java | 14 +- .../hdfs/server/namenode/FSImageFormat.java | 14 +- .../server/namenode/FSImageFormatPBINode.java | 11 +- .../server/namenode/FSImageSerialization.java | 6 +- .../hdfs/server/namenode/FSNamesystem.java | 42 +- .../namenode/FileUnderConstructionFeature.java | 6 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 74 ++-- .../hdfs/server/namenode/LeaseManager.java | 6 +- .../hdfs/server/namenode/NamenodeFsck.java | 4 +- .../snapshot/FSImageFormatPBSnapshot.java | 8 +- .../hdfs/server/namenode/snapshot/FileDiff.java | 10 +- .../server/namenode/snapshot/FileDiffList.java | 20 +- .../snapshot/FileWithSnapshotFeature.java | 4 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 4 +- .../apache/hadoop/hdfs/TestDecommission.java | 4 +- .../server/blockmanagement/TestBlockInfo.java | 20 +- .../blockmanagement/TestBlockManager.java | 36 +- .../blockmanagement/TestDatanodeDescriptor.java | 4 +- .../blockmanagement/TestPendingReplication.java | 4 +- .../blockmanagement/TestReplicationPolicy.java | 9 +- .../hdfs/server/namenode/CreateEditsLog.java | 8 +- .../hdfs/server/namenode/TestAddBlock.java | 12 +- .../namenode/TestBlockUnderConstruction.java | 6 +- .../TestCommitBlockSynchronization.java | 8 +- .../hdfs/server/namenode/TestEditLog.java | 4 +- .../hdfs/server/namenode/TestFSImage.java | 4 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 4 +- .../server/namenode/TestGetBlockLocations.java | 4 +- .../hdfs/server/namenode/TestINodeFile.java | 4 +- .../namenode/TestTruncateQuotaUpdate.java | 16 +- .../namenode/snapshot/SnapshotTestHelper.java | 4 +- .../snapshot/TestFileWithSnapshotFeature.java | 6 +- .../snapshot/TestSnapshotBlocksMap.java | 24 +- .../namenode/snapshot/TestSnapshotDeletion.java | 16 +- 48 files changed, 745 insertions(+), 750 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ce97552..948a516 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -580,6 +580,8 @@ Release 2.8.0 - UNRELEASED HDFS-8377. Support HTTP/2 in datanode. (Duo Zhang via wheat9) + HDFS-8482. Rename BlockInfoContiguous to BlockInfo. (Zhe Zhang via wang) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java index c0a959c..02a1d05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java @@ -31,7 +31,7 @@ public interface BlockCollection { /** * Get the last block of the collection. */ - public BlockInfoContiguous getLastBlock(); + public BlockInfo getLastBlock(); /** * Get content summary. @@ -46,7 +46,7 @@ public interface BlockCollection { /** * Get the blocks. */ - public BlockInfoContiguous[] getBlocks(); + public BlockInfo[] getBlocks(); /** * Get preferred block size for the collection @@ -73,13 +73,13 @@ public interface BlockCollection { /** * Set the block at the given index. */ - public void setBlock(int index, BlockInfoContiguous blk); + public void setBlock(int index, BlockInfo blk); /** * Convert the last block of the collection to an under-construction block * and set the locations. */ - public BlockInfoContiguousUnderConstruction setLastBlock(BlockInfoContiguous lastBlock, + public BlockInfoContiguousUnderConstruction setLastBlock(BlockInfo lastBlock, DatanodeStorageInfo[] targets) throws IOException; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java new file mode 100644 index 0000000..0370c57 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -0,0 +1,399 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.blockmanagement; + +import java.util.LinkedList; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; +import org.apache.hadoop.util.LightWeightGSet; + +/** + * BlockInfo class maintains for a given block + * the {@link BlockCollection} it is part of and datanodes where the replicas of + * the block are stored. + */ +@InterfaceAudience.Private +public class BlockInfo extends Block + implements LightWeightGSet.LinkedElement { + public static final BlockInfo[] EMPTY_ARRAY = {}; + + private BlockCollection bc; + + /** For implementing {@link LightWeightGSet.LinkedElement} interface */ + private LightWeightGSet.LinkedElement nextLinkedElement; + + /** + * This array contains triplets of references. For each i-th storage, the + * block belongs to triplets[3*i] is the reference to the + * {@link DatanodeStorageInfo} and triplets[3*i+1] and triplets[3*i+2] are + * references to the previous and the next blocks, respectively, in the list + * of blocks belonging to this storage. + * + * Using previous and next in Object triplets is done instead of a + * {@link LinkedList} list to efficiently use memory. With LinkedList the cost + * per replica is 42 bytes (LinkedList#Entry object per replica) versus 16 + * bytes using the triplets. + */ + private Object[] triplets; + + /** + * Construct an entry for blocksmap + * @param replication the block's replication factor + */ + public BlockInfo(short replication) { + this.triplets = new Object[3*replication]; + this.bc = null; + } + + public BlockInfo(Block blk, short replication) { + super(blk); + this.triplets = new Object[3*replication]; + this.bc = null; + } + + /** + * Copy construction. + * This is used to convert BlockInfoUnderConstruction + * @param from BlockInfo to copy from. + */ + protected BlockInfo(BlockInfo from) { + super(from); + this.triplets = new Object[from.triplets.length]; + this.bc = from.bc; + } + + public BlockCollection getBlockCollection() { + return bc; + } + + public void setBlockCollection(BlockCollection bc) { + this.bc = bc; + } + + public boolean isDeleted() { + return (bc == null); + } + + public DatanodeDescriptor getDatanode(int index) { + DatanodeStorageInfo storage = getStorageInfo(index); + return storage == null ? null : storage.getDatanodeDescriptor(); + } + + DatanodeStorageInfo getStorageInfo(int index) { + assert this.triplets != null : "BlockInfo is not initialized"; + assert index >= 0 && index*3 < triplets.length : "Index is out of bound"; + return (DatanodeStorageInfo)triplets[index*3]; + } + + private BlockInfo getPrevious(int index) { + assert this.triplets != null : "BlockInfo is not initialized"; + assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound"; + BlockInfo info = (BlockInfo)triplets[index*3+1]; + assert info == null || + info.getClass().getName().startsWith(BlockInfo.class.getName()) : + "BlockInfo is expected at " + index*3; + return info; + } + + BlockInfo getNext(int index) { + assert this.triplets != null : "BlockInfo is not initialized"; + assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound"; + BlockInfo info = (BlockInfo)triplets[index*3+2]; + assert info == null || info.getClass().getName().startsWith( + BlockInfo.class.getName()) : + "BlockInfo is expected at " + index*3; + return info; + } + + private void setStorageInfo(int index, DatanodeStorageInfo storage) { + assert this.triplets != null : "BlockInfo is not initialized"; + assert index >= 0 && index*3 < triplets.length : "Index is out of bound"; + triplets[index*3] = storage; + } + + /** + * Return the previous block on the block list for the datanode at + * position index. Set the previous block on the list to "to". + * + * @param index - the datanode index + * @param to - block to be set to previous on the list of blocks + * @return current previous block on the list of blocks + */ + private BlockInfo setPrevious(int index, BlockInfo to) { + assert this.triplets != null : "BlockInfo is not initialized"; + assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound"; + BlockInfo info = (BlockInfo)triplets[index*3+1]; + triplets[index*3+1] = to; + return info; + } + + /** + * Return the next block on the block list for the datanode at + * position index. Set the next block on the list to "to". + * + * @param index - the datanode index + * @param to - block to be set to next on the list of blocks + * * @return current next block on the list of blocks + */ + private BlockInfo setNext(int index, BlockInfo to) { + assert this.triplets != null : "BlockInfo is not initialized"; + assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound"; + BlockInfo info = (BlockInfo)triplets[index*3+2]; + triplets[index*3+2] = to; + return info; + } + + public int getCapacity() { + assert this.triplets != null : "BlockInfo is not initialized"; + assert triplets.length % 3 == 0 : "Malformed BlockInfo"; + return triplets.length / 3; + } + + /** + * Ensure that there is enough space to include num more triplets. + * @return first free triplet index. + */ + private int ensureCapacity(int num) { + assert this.triplets != null : "BlockInfo is not initialized"; + int last = numNodes(); + if(triplets.length >= (last+num)*3) + return last; + /* Not enough space left. Create a new array. Should normally + * happen only when replication is manually increased by the user. */ + Object[] old = triplets; + triplets = new Object[(last+num)*3]; + System.arraycopy(old, 0, triplets, 0, last*3); + return last; + } + + /** + * Count the number of data-nodes the block belongs to. + */ + public int numNodes() { + assert this.triplets != null : "BlockInfo is not initialized"; + assert triplets.length % 3 == 0 : "Malformed BlockInfo"; + for(int idx = getCapacity()-1; idx >= 0; idx--) { + if(getDatanode(idx) != null) + return idx+1; + } + return 0; + } + + /** + * Add a {@link DatanodeStorageInfo} location for a block + */ + boolean addStorage(DatanodeStorageInfo storage) { + // find the last null node + int lastNode = ensureCapacity(1); + setStorageInfo(lastNode, storage); + setNext(lastNode, null); + setPrevious(lastNode, null); + return true; + } + + /** + * Remove {@link DatanodeStorageInfo} location for a block + */ + boolean removeStorage(DatanodeStorageInfo storage) { + int dnIndex = findStorageInfo(storage); + if(dnIndex < 0) // the node is not found + return false; + assert getPrevious(dnIndex) == null && getNext(dnIndex) == null : + "Block is still in the list and must be removed first."; + // find the last not null node + int lastNode = numNodes()-1; + // replace current node triplet by the lastNode one + setStorageInfo(dnIndex, getStorageInfo(lastNode)); + setNext(dnIndex, getNext(lastNode)); + setPrevious(dnIndex, getPrevious(lastNode)); + // set the last triplet to null + setStorageInfo(lastNode, null); + setNext(lastNode, null); + setPrevious(lastNode, null); + return true; + } + + /** + * Find specified DatanodeStorageInfo. + * @return DatanodeStorageInfo or null if not found. + */ + DatanodeStorageInfo findStorageInfo(DatanodeDescriptor dn) { + int len = getCapacity(); + for(int idx = 0; idx < len; idx++) { + DatanodeStorageInfo cur = getStorageInfo(idx); + if(cur == null) + break; + if(cur.getDatanodeDescriptor() == dn) + return cur; + } + return null; + } + + /** + * Find specified DatanodeStorageInfo. + * @return index or -1 if not found. + */ + int findStorageInfo(DatanodeStorageInfo storageInfo) { + int len = getCapacity(); + for(int idx = 0; idx < len; idx++) { + DatanodeStorageInfo cur = getStorageInfo(idx); + if (cur == storageInfo) { + return idx; + } + if (cur == null) { + break; + } + } + return -1; + } + + /** + * Insert this block into the head of the list of blocks + * related to the specified DatanodeStorageInfo. + * If the head is null then form a new list. + * @return current block as the new head of the list. + */ + BlockInfo listInsert(BlockInfo head, + DatanodeStorageInfo storage) { + int dnIndex = this.findStorageInfo(storage); + assert dnIndex >= 0 : "Data node is not found: current"; + assert getPrevious(dnIndex) == null && getNext(dnIndex) == null : + "Block is already in the list and cannot be inserted."; + this.setPrevious(dnIndex, null); + this.setNext(dnIndex, head); + if(head != null) + head.setPrevious(head.findStorageInfo(storage), this); + return this; + } + + /** + * Remove this block from the list of blocks + * related to the specified DatanodeStorageInfo. + * If this block is the head of the list then return the next block as + * the new head. + * @return the new head of the list or null if the list becomes + * empy after deletion. + */ + BlockInfo listRemove(BlockInfo head, + DatanodeStorageInfo storage) { + if(head == null) + return null; + int dnIndex = this.findStorageInfo(storage); + if(dnIndex < 0) // this block is not on the data-node list + return head; + + BlockInfo next = this.getNext(dnIndex); + BlockInfo prev = this.getPrevious(dnIndex); + this.setNext(dnIndex, null); + this.setPrevious(dnIndex, null); + if(prev != null) + prev.setNext(prev.findStorageInfo(storage), next); + if(next != null) + next.setPrevious(next.findStorageInfo(storage), prev); + if(this == head) // removing the head + head = next; + return head; + } + + /** + * Remove this block from the list of blocks related to the specified + * DatanodeDescriptor. Insert it into the head of the list of blocks. + * + * @return the new head of the list. + */ + public BlockInfo moveBlockToHead(BlockInfo head, + DatanodeStorageInfo storage, int curIndex, int headIndex) { + if (head == this) { + return this; + } + BlockInfo next = this.setNext(curIndex, head); + BlockInfo prev = this.setPrevious(curIndex, null); + + head.setPrevious(headIndex, this); + prev.setNext(prev.findStorageInfo(storage), next); + if (next != null) { + next.setPrevious(next.findStorageInfo(storage), prev); + } + return this; + } + + /** + * BlockInfo represents a block that is not being constructed. + * In order to start modifying the block, the BlockInfo should be converted + * to {@link BlockInfoContiguousUnderConstruction}. + * @return {@link BlockUCState#COMPLETE} + */ + public BlockUCState getBlockUCState() { + return BlockUCState.COMPLETE; + } + + /** + * Is this block complete? + * + * @return true if the state of the block is {@link BlockUCState#COMPLETE} + */ + public boolean isComplete() { + return getBlockUCState().equals(BlockUCState.COMPLETE); + } + + /** + * Convert a complete block to an under construction block. + * @return BlockInfoUnderConstruction - an under construction block. + */ + public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction( + BlockUCState s, DatanodeStorageInfo[] targets) { + if(isComplete()) { + BlockInfoContiguousUnderConstruction ucBlock = + new BlockInfoContiguousUnderConstruction(this, + getBlockCollection().getPreferredBlockReplication(), s, targets); + ucBlock.setBlockCollection(getBlockCollection()); + return ucBlock; + } + // the block is already under construction + BlockInfoContiguousUnderConstruction ucBlock = + (BlockInfoContiguousUnderConstruction)this; + ucBlock.setBlockUCState(s); + ucBlock.setExpectedLocations(targets); + ucBlock.setBlockCollection(getBlockCollection()); + return ucBlock; + } + + @Override + public int hashCode() { + // Super implementation is sufficient + return super.hashCode(); + } + + @Override + public boolean equals(Object obj) { + // Sufficient to rely on super's implementation + return (this == obj) || super.equals(obj); + } + + @Override + public LightWeightGSet.LinkedElement getNext() { + return nextLinkedElement; + } + + @Override + public void setNext(LightWeightGSet.LinkedElement next) { + this.nextLinkedElement = next; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java deleted file mode 100644 index 769046b..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java +++ /dev/null @@ -1,399 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.blockmanagement; - -import java.util.LinkedList; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; -import org.apache.hadoop.util.LightWeightGSet; - -/** - * BlockInfo class maintains for a given block - * the {@link BlockCollection} it is part of and datanodes where the replicas of - * the block are stored. - */ -@InterfaceAudience.Private -public class BlockInfoContiguous extends Block - implements LightWeightGSet.LinkedElement { - public static final BlockInfoContiguous[] EMPTY_ARRAY = {}; - - private BlockCollection bc; - - /** For implementing {@link LightWeightGSet.LinkedElement} interface */ - private LightWeightGSet.LinkedElement nextLinkedElement; - - /** - * This array contains triplets of references. For each i-th storage, the - * block belongs to triplets[3*i] is the reference to the - * {@link DatanodeStorageInfo} and triplets[3*i+1] and triplets[3*i+2] are - * references to the previous and the next blocks, respectively, in the list - * of blocks belonging to this storage. - * - * Using previous and next in Object triplets is done instead of a - * {@link LinkedList} list to efficiently use memory. With LinkedList the cost - * per replica is 42 bytes (LinkedList#Entry object per replica) versus 16 - * bytes using the triplets. - */ - private Object[] triplets; - - /** - * Construct an entry for blocksmap - * @param replication the block's replication factor - */ - public BlockInfoContiguous(short replication) { - this.triplets = new Object[3*replication]; - this.bc = null; - } - - public BlockInfoContiguous(Block blk, short replication) { - super(blk); - this.triplets = new Object[3*replication]; - this.bc = null; - } - - /** - * Copy construction. - * This is used to convert BlockInfoUnderConstruction - * @param from BlockInfo to copy from. - */ - protected BlockInfoContiguous(BlockInfoContiguous from) { - super(from); - this.triplets = new Object[from.triplets.length]; - this.bc = from.bc; - } - - public BlockCollection getBlockCollection() { - return bc; - } - - public void setBlockCollection(BlockCollection bc) { - this.bc = bc; - } - - public boolean isDeleted() { - return (bc == null); - } - - public DatanodeDescriptor getDatanode(int index) { - DatanodeStorageInfo storage = getStorageInfo(index); - return storage == null ? null : storage.getDatanodeDescriptor(); - } - - DatanodeStorageInfo getStorageInfo(int index) { - assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3 < triplets.length : "Index is out of bound"; - return (DatanodeStorageInfo)triplets[index*3]; - } - - private BlockInfoContiguous getPrevious(int index) { - assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound"; - BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+1]; - assert info == null || - info.getClass().getName().startsWith(BlockInfoContiguous.class.getName()) : - "BlockInfo is expected at " + index*3; - return info; - } - - BlockInfoContiguous getNext(int index) { - assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound"; - BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+2]; - assert info == null || info.getClass().getName().startsWith( - BlockInfoContiguous.class.getName()) : - "BlockInfo is expected at " + index*3; - return info; - } - - private void setStorageInfo(int index, DatanodeStorageInfo storage) { - assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3 < triplets.length : "Index is out of bound"; - triplets[index*3] = storage; - } - - /** - * Return the previous block on the block list for the datanode at - * position index. Set the previous block on the list to "to". - * - * @param index - the datanode index - * @param to - block to be set to previous on the list of blocks - * @return current previous block on the list of blocks - */ - private BlockInfoContiguous setPrevious(int index, BlockInfoContiguous to) { - assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound"; - BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+1]; - triplets[index*3+1] = to; - return info; - } - - /** - * Return the next block on the block list for the datanode at - * position index. Set the next block on the list to "to". - * - * @param index - the datanode index - * @param to - block to be set to next on the list of blocks - * * @return current next block on the list of blocks - */ - private BlockInfoContiguous setNext(int index, BlockInfoContiguous to) { - assert this.triplets != null : "BlockInfo is not initialized"; - assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound"; - BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+2]; - triplets[index*3+2] = to; - return info; - } - - public int getCapacity() { - assert this.triplets != null : "BlockInfo is not initialized"; - assert triplets.length % 3 == 0 : "Malformed BlockInfo"; - return triplets.length / 3; - } - - /** - * Ensure that there is enough space to include num more triplets. - * @return first free triplet index. - */ - private int ensureCapacity(int num) { - assert this.triplets != null : "BlockInfo is not initialized"; - int last = numNodes(); - if(triplets.length >= (last+num)*3) - return last; - /* Not enough space left. Create a new array. Should normally - * happen only when replication is manually increased by the user. */ - Object[] old = triplets; - triplets = new Object[(last+num)*3]; - System.arraycopy(old, 0, triplets, 0, last*3); - return last; - } - - /** - * Count the number of data-nodes the block belongs to. - */ - public int numNodes() { - assert this.triplets != null : "BlockInfo is not initialized"; - assert triplets.length % 3 == 0 : "Malformed BlockInfo"; - for(int idx = getCapacity()-1; idx >= 0; idx--) { - if(getDatanode(idx) != null) - return idx+1; - } - return 0; - } - - /** - * Add a {@link DatanodeStorageInfo} location for a block - */ - boolean addStorage(DatanodeStorageInfo storage) { - // find the last null node - int lastNode = ensureCapacity(1); - setStorageInfo(lastNode, storage); - setNext(lastNode, null); - setPrevious(lastNode, null); - return true; - } - - /** - * Remove {@link DatanodeStorageInfo} location for a block - */ - boolean removeStorage(DatanodeStorageInfo storage) { - int dnIndex = findStorageInfo(storage); - if(dnIndex < 0) // the node is not found - return false; - assert getPrevious(dnIndex) == null && getNext(dnIndex) == null : - "Block is still in the list and must be removed first."; - // find the last not null node - int lastNode = numNodes()-1; - // replace current node triplet by the lastNode one - setStorageInfo(dnIndex, getStorageInfo(lastNode)); - setNext(dnIndex, getNext(lastNode)); - setPrevious(dnIndex, getPrevious(lastNode)); - // set the last triplet to null - setStorageInfo(lastNode, null); - setNext(lastNode, null); - setPrevious(lastNode, null); - return true; - } - - /** - * Find specified DatanodeStorageInfo. - * @return DatanodeStorageInfo or null if not found. - */ - DatanodeStorageInfo findStorageInfo(DatanodeDescriptor dn) { - int len = getCapacity(); - for(int idx = 0; idx < len; idx++) { - DatanodeStorageInfo cur = getStorageInfo(idx); - if(cur == null) - break; - if(cur.getDatanodeDescriptor() == dn) - return cur; - } - return null; - } - - /** - * Find specified DatanodeStorageInfo. - * @return index or -1 if not found. - */ - int findStorageInfo(DatanodeStorageInfo storageInfo) { - int len = getCapacity(); - for(int idx = 0; idx < len; idx++) { - DatanodeStorageInfo cur = getStorageInfo(idx); - if (cur == storageInfo) { - return idx; - } - if (cur == null) { - break; - } - } - return -1; - } - - /** - * Insert this block into the head of the list of blocks - * related to the specified DatanodeStorageInfo. - * If the head is null then form a new list. - * @return current block as the new head of the list. - */ - BlockInfoContiguous listInsert(BlockInfoContiguous head, - DatanodeStorageInfo storage) { - int dnIndex = this.findStorageInfo(storage); - assert dnIndex >= 0 : "Data node is not found: current"; - assert getPrevious(dnIndex) == null && getNext(dnIndex) == null : - "Block is already in the list and cannot be inserted."; - this.setPrevious(dnIndex, null); - this.setNext(dnIndex, head); - if(head != null) - head.setPrevious(head.findStorageInfo(storage), this); - return this; - } - - /** - * Remove this block from the list of blocks - * related to the specified DatanodeStorageInfo. - * If this block is the head of the list then return the next block as - * the new head. - * @return the new head of the list or null if the list becomes - * empy after deletion. - */ - BlockInfoContiguous listRemove(BlockInfoContiguous head, - DatanodeStorageInfo storage) { - if(head == null) - return null; - int dnIndex = this.findStorageInfo(storage); - if(dnIndex < 0) // this block is not on the data-node list - return head; - - BlockInfoContiguous next = this.getNext(dnIndex); - BlockInfoContiguous prev = this.getPrevious(dnIndex); - this.setNext(dnIndex, null); - this.setPrevious(dnIndex, null); - if(prev != null) - prev.setNext(prev.findStorageInfo(storage), next); - if(next != null) - next.setPrevious(next.findStorageInfo(storage), prev); - if(this == head) // removing the head - head = next; - return head; - } - - /** - * Remove this block from the list of blocks related to the specified - * DatanodeDescriptor. Insert it into the head of the list of blocks. - * - * @return the new head of the list. - */ - public BlockInfoContiguous moveBlockToHead(BlockInfoContiguous head, - DatanodeStorageInfo storage, int curIndex, int headIndex) { - if (head == this) { - return this; - } - BlockInfoContiguous next = this.setNext(curIndex, head); - BlockInfoContiguous prev = this.setPrevious(curIndex, null); - - head.setPrevious(headIndex, this); - prev.setNext(prev.findStorageInfo(storage), next); - if (next != null) { - next.setPrevious(next.findStorageInfo(storage), prev); - } - return this; - } - - /** - * BlockInfo represents a block that is not being constructed. - * In order to start modifying the block, the BlockInfo should be converted - * to {@link BlockInfoContiguousUnderConstruction}. - * @return {@link BlockUCState#COMPLETE} - */ - public BlockUCState getBlockUCState() { - return BlockUCState.COMPLETE; - } - - /** - * Is this block complete? - * - * @return true if the state of the block is {@link BlockUCState#COMPLETE} - */ - public boolean isComplete() { - return getBlockUCState().equals(BlockUCState.COMPLETE); - } - - /** - * Convert a complete block to an under construction block. - * @return BlockInfoUnderConstruction - an under construction block. - */ - public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction( - BlockUCState s, DatanodeStorageInfo[] targets) { - if(isComplete()) { - BlockInfoContiguousUnderConstruction ucBlock = - new BlockInfoContiguousUnderConstruction(this, - getBlockCollection().getPreferredBlockReplication(), s, targets); - ucBlock.setBlockCollection(getBlockCollection()); - return ucBlock; - } - // the block is already under construction - BlockInfoContiguousUnderConstruction ucBlock = - (BlockInfoContiguousUnderConstruction)this; - ucBlock.setBlockUCState(s); - ucBlock.setExpectedLocations(targets); - ucBlock.setBlockCollection(getBlockCollection()); - return ucBlock; - } - - @Override - public int hashCode() { - // Super implementation is sufficient - return super.hashCode(); - } - - @Override - public boolean equals(Object obj) { - // Sufficient to rely on super's implementation - return (this == obj) || super.equals(obj); - } - - @Override - public LightWeightGSet.LinkedElement getNext() { - return nextLinkedElement; - } - - @Override - public void setNext(LightWeightGSet.LinkedElement next) { - this.nextLinkedElement = next; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java index 92153ab..58020a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; * Represents a block that is currently being constructed.<br> * This is usually the last block of a file opened for write or append. */ -public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous { +public class BlockInfoContiguousUnderConstruction extends BlockInfo { /** Block state. See {@link BlockUCState} */ private BlockUCState blockUCState; @@ -182,10 +182,10 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous { * the client or it does not have at least a minimal number of replicas * reported from data-nodes. */ - BlockInfoContiguous convertToCompleteBlock() throws IOException { + BlockInfo convertToCompleteBlock() throws IOException { assert getBlockUCState() != BlockUCState.COMPLETE : "Trying to convert a COMPLETE block"; - return new BlockInfoContiguous(this); + return new BlockInfo(this); } /** Set expected locations */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 54981fb..45a485b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -543,8 +543,8 @@ public class BlockManager { int usableReplicas = numReplicas.liveReplicas() + numReplicas.decommissionedAndDecommissioning(); - if (block instanceof BlockInfoContiguous) { - BlockCollection bc = ((BlockInfoContiguous) block).getBlockCollection(); + if (block instanceof BlockInfo) { + BlockCollection bc = ((BlockInfo) block).getBlockCollection(); String fileName = (bc == null) ? "[orphaned]" : bc.getName(); out.print(fileName + ": "); } @@ -624,7 +624,7 @@ public class BlockManager { Block commitBlock) throws IOException { if(commitBlock == null) return false; // not committing, this is a block allocation retry - BlockInfoContiguous lastBlock = bc.getLastBlock(); + BlockInfo lastBlock = bc.getLastBlock(); if(lastBlock == null) return false; // no blocks in file yet if(lastBlock.isComplete()) @@ -644,11 +644,11 @@ public class BlockManager { * @throws IOException if the block does not have at least a minimal number * of replicas reported from data-nodes. */ - private BlockInfoContiguous completeBlock(final BlockCollection bc, + private BlockInfo completeBlock(final BlockCollection bc, final int blkIndex, boolean force) throws IOException { if(blkIndex < 0) return null; - BlockInfoContiguous curBlock = bc.getBlocks()[blkIndex]; + BlockInfo curBlock = bc.getBlocks()[blkIndex]; if(curBlock.isComplete()) return curBlock; BlockInfoContiguousUnderConstruction ucBlock = @@ -660,7 +660,7 @@ public class BlockManager { if(!force && ucBlock.getBlockUCState() != BlockUCState.COMMITTED) throw new IOException( "Cannot complete block: block has not been COMMITTED by the client"); - BlockInfoContiguous completeBlock = ucBlock.convertToCompleteBlock(); + BlockInfo completeBlock = ucBlock.convertToCompleteBlock(); // replace penultimate block in file bc.setBlock(blkIndex, completeBlock); @@ -678,9 +678,9 @@ public class BlockManager { return blocksMap.replaceBlock(completeBlock); } - private BlockInfoContiguous completeBlock(final BlockCollection bc, - final BlockInfoContiguous block, boolean force) throws IOException { - BlockInfoContiguous[] fileBlocks = bc.getBlocks(); + private BlockInfo completeBlock(final BlockCollection bc, + final BlockInfo block, boolean force) throws IOException { + BlockInfo[] fileBlocks = bc.getBlocks(); for(int idx = 0; idx < fileBlocks.length; idx++) if(fileBlocks[idx] == block) { return completeBlock(bc, idx, force); @@ -693,7 +693,7 @@ public class BlockManager { * regardless of whether enough replicas are present. This is necessary * when tailing edit logs as a Standby. */ - public BlockInfoContiguous forceCompleteBlock(final BlockCollection bc, + public BlockInfo forceCompleteBlock(final BlockCollection bc, final BlockInfoContiguousUnderConstruction block) throws IOException { block.commitBlock(block); return completeBlock(bc, block, true); @@ -716,7 +716,7 @@ public class BlockManager { */ public LocatedBlock convertLastBlockToUnderConstruction( BlockCollection bc, long bytesToRemove) throws IOException { - BlockInfoContiguous oldBlock = bc.getLastBlock(); + BlockInfo oldBlock = bc.getLastBlock(); if(oldBlock == null || bc.getPreferredBlockSize() == oldBlock.getNumBytes() - bytesToRemove) return null; @@ -769,7 +769,7 @@ public class BlockManager { } private List<LocatedBlock> createLocatedBlockList( - final BlockInfoContiguous[] blocks, + final BlockInfo[] blocks, final long offset, final long length, final int nrBlocksToReturn, final AccessMode mode) throws IOException { int curBlk = 0; @@ -799,7 +799,7 @@ public class BlockManager { return results; } - private LocatedBlock createLocatedBlock(final BlockInfoContiguous[] blocks, + private LocatedBlock createLocatedBlock(final BlockInfo[] blocks, final long endPos, final AccessMode mode) throws IOException { int curBlk = 0; long curPos = 0; @@ -815,7 +815,7 @@ public class BlockManager { return createLocatedBlock(blocks[curBlk], curPos, mode); } - private LocatedBlock createLocatedBlock(final BlockInfoContiguous blk, final long pos, + private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos, final AccessMode mode) throws IOException { final LocatedBlock lb = createLocatedBlock(blk, pos); if (mode != null) { @@ -825,7 +825,7 @@ public class BlockManager { } /** @return a LocatedBlock for the given block */ - private LocatedBlock createLocatedBlock(final BlockInfoContiguous blk, final long pos + private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos ) throws IOException { if (blk instanceof BlockInfoContiguousUnderConstruction) { if (blk.isComplete()) { @@ -874,7 +874,7 @@ public class BlockManager { } /** Create a LocatedBlocks. */ - public LocatedBlocks createLocatedBlocks(final BlockInfoContiguous[] blocks, + public LocatedBlocks createLocatedBlocks(final BlockInfo[] blocks, final long fileSizeExcludeBlocksUnderConstruction, final boolean isFileUnderConstruction, final long offset, final long length, final boolean needBlockToken, @@ -897,7 +897,7 @@ public class BlockManager { final LocatedBlock lastlb; final boolean isComplete; if (!inSnapshot) { - final BlockInfoContiguous last = blocks[blocks.length - 1]; + final BlockInfo last = blocks[blocks.length - 1]; final long lastPos = last.isComplete()? fileSizeExcludeBlocksUnderConstruction - last.getNumBytes() : fileSizeExcludeBlocksUnderConstruction; @@ -986,7 +986,7 @@ public class BlockManager { /** * Check if a block is replicated to at least the minimum replication. */ - public boolean isSufficientlyReplicated(BlockInfoContiguous b) { + public boolean isSufficientlyReplicated(BlockInfo b) { // Compare against the lesser of the minReplication and number of live DNs. final int replication = Math.min(minReplication, getDatanodeManager().getNumLiveDataNodes()); @@ -1027,7 +1027,7 @@ public class BlockManager { if(numBlocks == 0) { return new BlocksWithLocations(new BlockWithLocations[0]); } - Iterator<BlockInfoContiguous> iter = node.getBlockIterator(); + Iterator<BlockInfo> iter = node.getBlockIterator(); // starting from a random block int startBlock = ThreadLocalRandom.current().nextInt(numBlocks); // skip blocks @@ -1036,7 +1036,7 @@ public class BlockManager { } List<BlockWithLocations> results = new ArrayList<BlockWithLocations>(); long totalSize = 0; - BlockInfoContiguous curBlock; + BlockInfo curBlock; while(totalSize<size && iter.hasNext()) { curBlock = iter.next(); if(!curBlock.isComplete()) continue; @@ -1135,7 +1135,7 @@ public class BlockManager { public void findAndMarkBlockAsCorrupt(final ExtendedBlock blk, final DatanodeInfo dn, String storageID, String reason) throws IOException { assert namesystem.hasWriteLock(); - final BlockInfoContiguous storedBlock = getStoredBlock(blk.getLocalBlock()); + final BlockInfo storedBlock = getStoredBlock(blk.getLocalBlock()); if (storedBlock == null) { // Check if the replica is in the blockMap, if not // ignore the request for now. This could happen when BlockScanner @@ -1694,7 +1694,7 @@ public class BlockManager { * Use the blockinfo from the blocksmap to be certain we're working * with the most up-to-date block information (e.g. genstamp). */ - BlockInfoContiguous bi = blocksMap.getStoredBlock(timedOutItems[i]); + BlockInfo bi = blocksMap.getStoredBlock(timedOutItems[i]); if (bi == null) { continue; } @@ -1738,16 +1738,16 @@ public class BlockManager { */ private static class BlockToMarkCorrupt { /** The corrupted block in a datanode. */ - final BlockInfoContiguous corrupted; + final BlockInfo corrupted; /** The corresponding block stored in the BlockManager. */ - final BlockInfoContiguous stored; + final BlockInfo stored; /** The reason to mark corrupt. */ final String reason; /** The reason code to be stored */ final Reason reasonCode; - BlockToMarkCorrupt(BlockInfoContiguous corrupted, - BlockInfoContiguous stored, String reason, + BlockToMarkCorrupt(BlockInfo corrupted, + BlockInfo stored, String reason, Reason reasonCode) { Preconditions.checkNotNull(corrupted, "corrupted is null"); Preconditions.checkNotNull(stored, "stored is null"); @@ -1758,14 +1758,14 @@ public class BlockManager { this.reasonCode = reasonCode; } - BlockToMarkCorrupt(BlockInfoContiguous stored, String reason, + BlockToMarkCorrupt(BlockInfo stored, String reason, Reason reasonCode) { this(stored, stored, reason, reasonCode); } - BlockToMarkCorrupt(BlockInfoContiguous stored, long gs, String reason, + BlockToMarkCorrupt(BlockInfo stored, long gs, String reason, Reason reasonCode) { - this(new BlockInfoContiguous(stored), stored, reason, reasonCode); + this(new BlockInfo(stored), stored, reason, reasonCode); //the corrupted block in datanode has a different generation stamp corrupted.setGenerationStamp(gs); } @@ -1882,10 +1882,10 @@ public class BlockManager { "longer exists on the DataNode.", Long.toHexString(context.getReportId()), zombie.getStorageID()); assert(namesystem.hasWriteLock()); - Iterator<BlockInfoContiguous> iter = zombie.getBlockIterator(); + Iterator<BlockInfo> iter = zombie.getBlockIterator(); int prevBlocks = zombie.numBlocks(); while (iter.hasNext()) { - BlockInfoContiguous block = iter.next(); + BlockInfo block = iter.next(); // We assume that a block can be on only one storage in a DataNode. // That's why we pass in the DatanodeDescriptor rather than the // DatanodeStorageInfo. @@ -1943,7 +1943,7 @@ public class BlockManager { break; } - BlockInfoContiguous bi = blocksMap.getStoredBlock(b); + BlockInfo bi = blocksMap.getStoredBlock(b); if (bi == null) { if (LOG.isDebugEnabled()) { LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " + @@ -1983,7 +1983,7 @@ public class BlockManager { // Modify the (block-->datanode) map, according to the difference // between the old and new block report. // - Collection<BlockInfoContiguous> toAdd = new LinkedList<BlockInfoContiguous>(); + Collection<BlockInfo> toAdd = new LinkedList<BlockInfo>(); Collection<Block> toRemove = new TreeSet<Block>(); Collection<Block> toInvalidate = new LinkedList<Block>(); Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<BlockToMarkCorrupt>(); @@ -2000,7 +2000,7 @@ public class BlockManager { removeStoredBlock(b, node); } int numBlocksLogged = 0; - for (BlockInfoContiguous b : toAdd) { + for (BlockInfo b : toAdd) { addStoredBlock(b, storageInfo, null, numBlocksLogged < maxNumBlocksToLog); numBlocksLogged++; } @@ -2022,7 +2022,7 @@ public class BlockManager { * Mark block replicas as corrupt except those on the storages in * newStorages list. */ - public void markBlockReplicasAsCorrupt(BlockInfoContiguous block, + public void markBlockReplicasAsCorrupt(BlockInfo block, long oldGenerationStamp, long oldNumBytes, DatanodeStorageInfo[] newStorages) throws IOException { assert namesystem.hasWriteLock(); @@ -2091,7 +2091,7 @@ public class BlockManager { continue; } - BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk); + BlockInfo storedBlock = blocksMap.getStoredBlock(iblk); // If block does not belong to any file, we are done. if (storedBlock == null) continue; @@ -2136,7 +2136,7 @@ public class BlockManager { private void reportDiff(DatanodeStorageInfo storageInfo, BlockListAsLongs newReport, - Collection<BlockInfoContiguous> toAdd, // add to DatanodeDescriptor + Collection<BlockInfo> toAdd, // add to DatanodeDescriptor Collection<Block> toRemove, // remove from DatanodeDescriptor Collection<Block> toInvalidate, // should be removed from DN Collection<BlockToMarkCorrupt> toCorrupt, // add to corrupt replicas list @@ -2144,7 +2144,7 @@ public class BlockManager { // place a delimiter in the list which separates blocks // that have been reported from those that have not - BlockInfoContiguous delimiter = new BlockInfoContiguous(new Block(), (short) 1); + BlockInfo delimiter = new BlockInfo(new Block(), (short) 1); AddBlockResult result = storageInfo.addBlock(delimiter); assert result == AddBlockResult.ADDED : "Delimiting block cannot be present in the node"; @@ -2157,7 +2157,7 @@ public class BlockManager { // scan the report and process newly reported blocks for (BlockReportReplica iblk : newReport) { ReplicaState iState = iblk.getState(); - BlockInfoContiguous storedBlock = processReportedBlock(storageInfo, + BlockInfo storedBlock = processReportedBlock(storageInfo, iblk, iState, toAdd, toInvalidate, toCorrupt, toUC); // move block to the head of the list @@ -2169,7 +2169,7 @@ public class BlockManager { // collect blocks that have not been reported // all of them are next to the delimiter - Iterator<BlockInfoContiguous> it = + Iterator<BlockInfo> it = storageInfo.new BlockIterator(delimiter.getNext(0)); while(it.hasNext()) toRemove.add(it.next()); @@ -2207,10 +2207,10 @@ public class BlockManager { * @return the up-to-date stored block, if it should be kept. * Otherwise, null. */ - private BlockInfoContiguous processReportedBlock( + private BlockInfo processReportedBlock( final DatanodeStorageInfo storageInfo, final Block block, final ReplicaState reportedState, - final Collection<BlockInfoContiguous> toAdd, + final Collection<BlockInfo> toAdd, final Collection<Block> toInvalidate, final Collection<BlockToMarkCorrupt> toCorrupt, final Collection<StatefulBlockInfo> toUC) { @@ -2231,7 +2231,7 @@ public class BlockManager { } // find block by blockId - BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block); + BlockInfo storedBlock = blocksMap.getStoredBlock(block); if(storedBlock == null) { // If blocksMap does not contain reported block id, // the replica should be removed from the data-node. @@ -2370,7 +2370,7 @@ public class BlockManager { */ private BlockToMarkCorrupt checkReplicaCorrupt( Block reported, ReplicaState reportedState, - BlockInfoContiguous storedBlock, BlockUCState ucState, + BlockInfo storedBlock, BlockUCState ucState, DatanodeDescriptor dn) { switch(reportedState) { case FINALIZED: @@ -2443,7 +2443,7 @@ public class BlockManager { } } - private boolean isBlockUnderConstruction(BlockInfoContiguous storedBlock, + private boolean isBlockUnderConstruction(BlockInfo storedBlock, BlockUCState ucState, ReplicaState reportedState) { switch(reportedState) { case FINALIZED: @@ -2487,7 +2487,7 @@ public class BlockManager { * * @throws IOException */ - private void addStoredBlockImmediate(BlockInfoContiguous storedBlock, + private void addStoredBlockImmediate(BlockInfo storedBlock, DatanodeStorageInfo storageInfo) throws IOException { assert (storedBlock != null && namesystem.hasWriteLock()); @@ -2519,13 +2519,13 @@ public class BlockManager { * needed replications if this takes care of the problem. * @return the block that is stored in blockMap. */ - private Block addStoredBlock(final BlockInfoContiguous block, + private Block addStoredBlock(final BlockInfo block, DatanodeStorageInfo storageInfo, DatanodeDescriptor delNodeHint, boolean logEveryBlock) throws IOException { assert block != null && namesystem.hasWriteLock(); - BlockInfoContiguous storedBlock; + BlockInfo storedBlock; DatanodeDescriptor node = storageInfo.getDatanodeDescriptor(); if (block instanceof BlockInfoContiguousUnderConstruction) { //refresh our copy in case the block got completed in another thread @@ -2623,7 +2623,7 @@ public class BlockManager { return storedBlock; } - private void logAddStoredBlock(BlockInfoContiguous storedBlock, + private void logAddStoredBlock(BlockInfo storedBlock, DatanodeDescriptor node) { if (!blockLog.isInfoEnabled()) { return; @@ -2651,7 +2651,7 @@ public class BlockManager { * * @param blk Block whose corrupt replicas need to be invalidated */ - private void invalidateCorruptReplicas(BlockInfoContiguous blk) { + private void invalidateCorruptReplicas(BlockInfo blk) { Collection<DatanodeDescriptor> nodes = corruptReplicas.getNodes(blk); boolean removedFromBlocksMap = true; if (nodes == null) @@ -2730,7 +2730,7 @@ public class BlockManager { long nrInvalid = 0, nrOverReplicated = 0; long nrUnderReplicated = 0, nrPostponed = 0, nrUnderConstruction = 0; long startTimeMisReplicatedScan = Time.monotonicNow(); - Iterator<BlockInfoContiguous> blocksItr = blocksMap.getBlocks().iterator(); + Iterator<BlockInfo> blocksItr = blocksMap.getBlocks().iterator(); long totalBlocks = blocksMap.size(); replicationQueuesInitProgress = 0; long totalProcessed = 0; @@ -2742,7 +2742,7 @@ public class BlockManager { namesystem.writeLockInterruptibly(); try { while (processed < numBlocksPerIteration && blocksItr.hasNext()) { - BlockInfoContiguous block = blocksItr.next(); + BlockInfo block = blocksItr.next(); MisReplicationResult res = processMisReplicatedBlock(block); if (LOG.isTraceEnabled()) { LOG.trace("block " + block + ": " + res); @@ -2817,7 +2817,7 @@ public class BlockManager { * appropriate queues if necessary, and returns a result code indicating * what happened with it. */ - private MisReplicationResult processMisReplicatedBlock(BlockInfoContiguous block) { + private MisReplicationResult processMisReplicatedBlock(BlockInfo block) { if (block.isDeleted()) { // block does not belong to any file addToInvalidates(block); @@ -3157,7 +3157,7 @@ public class BlockManager { ReplicaState reportedState, DatanodeDescriptor delHintNode) throws IOException { // blockReceived reports a finalized block - Collection<BlockInfoContiguous> toAdd = new LinkedList<BlockInfoContiguous>(); + Collection<BlockInfo> toAdd = new LinkedList<BlockInfo>(); Collection<Block> toInvalidate = new LinkedList<Block>(); Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<BlockToMarkCorrupt>(); Collection<StatefulBlockInfo> toUC = new LinkedList<StatefulBlockInfo>(); @@ -3174,7 +3174,7 @@ public class BlockManager { addStoredBlockUnderConstruction(b, storageInfo); } long numBlocksLogged = 0; - for (BlockInfoContiguous b : toAdd) { + for (BlockInfo b : toAdd) { addStoredBlock(b, storageInfo, delHintNode, numBlocksLogged < maxNumBlocksToLog); numBlocksLogged++; } @@ -3301,7 +3301,7 @@ public class BlockManager { * @param b - the block being tested * @return count of live nodes for this block */ - int countLiveNodes(BlockInfoContiguous b) { + int countLiveNodes(BlockInfo b) { if (!namesystem.isInStartupSafeMode()) { return countNodes(b).liveReplicas(); } @@ -3380,7 +3380,7 @@ public class BlockManager { return blocksMap.size(); } - public DatanodeStorageInfo[] getStorages(BlockInfoContiguous block) { + public DatanodeStorageInfo[] getStorages(BlockInfo block) { final DatanodeStorageInfo[] storages = new DatanodeStorageInfo[block.numNodes()]; int i = 0; for(DatanodeStorageInfo s : blocksMap.getStorages(block)) { @@ -3409,7 +3409,7 @@ public class BlockManager { } } - public BlockInfoContiguous getStoredBlock(Block block) { + public BlockInfo getStoredBlock(Block block) { return blocksMap.getStoredBlock(block); } @@ -3462,8 +3462,8 @@ public class BlockManager { * replicated. */ public boolean checkBlocksProperlyReplicated( - String src, BlockInfoContiguous[] blocks) { - for (BlockInfoContiguous b: blocks) { + String src, BlockInfo[] blocks) { + for (BlockInfo b: blocks) { if (!b.isComplete()) { final BlockInfoContiguousUnderConstruction uc = (BlockInfoContiguousUnderConstruction)b; @@ -3578,7 +3578,7 @@ public class BlockManager { return this.neededReplications.getCorruptReplOneBlockSize(); } - public BlockInfoContiguous addBlockCollection(BlockInfoContiguous block, + public BlockInfo addBlockCollection(BlockInfo block, BlockCollection bc) { return blocksMap.addBlockCollection(block, bc); } @@ -3818,7 +3818,7 @@ public class BlockManager { /** * A simple result enum for the result of - * {@link BlockManager#processMisReplicatedBlock(BlockInfoContiguous)}. + * {@link BlockManager#processMisReplicatedBlock(BlockInfo)}. */ enum MisReplicationResult { /** The block should be invalidated since it belongs to a deleted file. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java index 5e7d34f..d06d503 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java @@ -36,10 +36,10 @@ import com.google.common.collect.Iterables; */ class BlocksMap { private static class StorageIterator implements Iterator<DatanodeStorageInfo> { - private final BlockInfoContiguous blockInfo; + private final BlockInfo blockInfo; private int nextIdx = 0; - StorageIterator(BlockInfoContiguous blkInfo) { + StorageIterator(BlockInfo blkInfo) { this.blockInfo = blkInfo; } @@ -63,14 +63,14 @@ class BlocksMap { /** Constant {@link LightWeightGSet} capacity. */ private final int capacity; - private GSet<Block, BlockInfoContiguous> blocks; + private GSet<Block, BlockInfo> blocks; BlocksMap(int capacity) { // Use 2% of total memory to size the GSet capacity this.capacity = capacity; - this.blocks = new LightWeightGSet<Block, BlockInfoContiguous>(capacity) { + this.blocks = new LightWeightGSet<Block, BlockInfo>(capacity) { @Override - public Iterator<BlockInfoContiguous> iterator() { + public Iterator<BlockInfo> iterator() { SetIterator iterator = new SetIterator(); /* * Not tracking any modifications to set. As this set will be used @@ -97,15 +97,15 @@ class BlocksMap { } BlockCollection getBlockCollection(Block b) { - BlockInfoContiguous info = blocks.get(b); + BlockInfo info = blocks.get(b); return (info != null) ? info.getBlockCollection() : null; } /** * Add block b belonging to the specified block collection to the map. */ - BlockInfoContiguous addBlockCollection(BlockInfoContiguous b, BlockCollection bc) { - BlockInfoContiguous info = blocks.get(b); + BlockInfo addBlockCollection(BlockInfo b, BlockCollection bc) { + BlockInfo info = blocks.get(b); if (info != b) { info = b; blocks.put(info); @@ -120,7 +120,7 @@ class BlocksMap { * and remove all data-node locations associated with the block. */ void removeBlock(Block block) { - BlockInfoContiguous blockInfo = blocks.remove(block); + BlockInfo blockInfo = blocks.remove(block); if (blockInfo == null) return; @@ -132,7 +132,7 @@ class BlocksMap { } /** Returns the block object it it exists in the map. */ - BlockInfoContiguous getStoredBlock(Block b) { + BlockInfo getStoredBlock(Block b) { return blocks.get(b); } @@ -164,7 +164,7 @@ class BlocksMap { * For a block that has already been retrieved from the BlocksMap * returns {@link Iterable} of the storages the block belongs to. */ - Iterable<DatanodeStorageInfo> getStorages(final BlockInfoContiguous storedBlock) { + Iterable<DatanodeStorageInfo> getStorages(final BlockInfo storedBlock) { return new Iterable<DatanodeStorageInfo>() { @Override public Iterator<DatanodeStorageInfo> iterator() { @@ -175,7 +175,7 @@ class BlocksMap { /** counts number of containing nodes. Better than using iterator. */ int numNodes(Block b) { - BlockInfoContiguous info = blocks.get(b); + BlockInfo info = blocks.get(b); return info == null ? 0 : info.numNodes(); } @@ -185,7 +185,7 @@ class BlocksMap { * only if it does not belong to any file and data-nodes. */ boolean removeNode(Block b, DatanodeDescriptor node) { - BlockInfoContiguous info = blocks.get(b); + BlockInfo info = blocks.get(b); if (info == null) return false; @@ -203,7 +203,7 @@ class BlocksMap { return blocks.size(); } - Iterable<BlockInfoContiguous> getBlocks() { + Iterable<BlockInfo> getBlocks() { return blocks; } @@ -218,8 +218,8 @@ class BlocksMap { * @param newBlock - block for replacement * @return new block */ - BlockInfoContiguous replaceBlock(BlockInfoContiguous newBlock) { - BlockInfoContiguous currentBlock = blocks.get(newBlock); + BlockInfo replaceBlock(BlockInfo newBlock) { + BlockInfo currentBlock = blocks.get(newBlock); assert currentBlock != null : "the block if not in blocksMap"; // replace block in data-node lists for (int i = currentBlock.numNodes() - 1; i >= 0; i--) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java index bf5ece9..a0f3503 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java @@ -369,7 +369,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable { * @param file The file. */ private void rescanFile(CacheDirective directive, INodeFile file) { - BlockInfoContiguous[] blockInfos = file.getBlocks(); + BlockInfo[] blockInfos = file.getBlocks(); // Increment the "needed" statistics directive.addFilesNeeded(1); @@ -394,7 +394,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable { } long cachedTotal = 0; - for (BlockInfoContiguous blockInfo : blockInfos) { + for (BlockInfo blockInfo : blockInfos) { if (!blockInfo.getBlockUCState().equals(BlockUCState.COMPLETE)) { // We don't try to cache blocks that are under construction. LOG.trace("Directive {}: can't cache block {} because it is in state " @@ -453,7 +453,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable { } private String findReasonForNotCaching(CachedBlock cblock, - BlockInfoContiguous blockInfo) { + BlockInfo blockInfo) { if (blockInfo == null) { // Somehow, a cache report with the block arrived, but the block // reports from the DataNode haven't (yet?) described such a block. @@ -513,7 +513,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable { iter.remove(); } } - BlockInfoContiguous blockInfo = blockManager. + BlockInfo blockInfo = blockManager. getStoredBlock(new Block(cblock.getBlockId())); String reason = findReasonForNotCaching(cblock, blockInfo); int neededCached = 0; @@ -628,7 +628,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable { List<DatanodeDescriptor> pendingCached) { // To figure out which replicas can be cached, we consult the // blocksMap. We don't want to try to cache a corrupt replica, though. - BlockInfoContiguous blockInfo = blockManager. + BlockInfo blockInfo = blockManager. getStoredBlock(new Block(cachedBlock.getBlockId())); if (blockInfo == null) { LOG.debug("Block {}: can't add new cached replicas," + @@ -667,7 +667,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable { Iterator<CachedBlock> it = datanode.getPendingCached().iterator(); while (it.hasNext()) { CachedBlock cBlock = it.next(); - BlockInfoContiguous info = + BlockInfo info = blockManager.getStoredBlock(new Block(cBlock.getBlockId())); if (info != null) { pendingBytes -= info.getNumBytes(); @@ -677,7 +677,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable { // Add pending uncached blocks from effective capacity while (it.hasNext()) { CachedBlock cBlock = it.next(); - BlockInfoContiguous info = + BlockInfo info = blockManager.getStoredBlock(new Block(cBlock.getBlockId())); if (info != null) { pendingBytes += info.getNumBytes(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 4731ad4..02abc1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -335,7 +335,7 @@ public class DatanodeDescriptor extends DatanodeInfo { * Remove block from the list of blocks belonging to the data-node. Remove * data-node from the block. */ - boolean removeBlock(BlockInfoContiguous b) { + boolean removeBlock(BlockInfo b) { final DatanodeStorageInfo s = b.findStorageInfo(this); // if block exists on this datanode if (s != null) { @@ -348,7 +348,7 @@ public class DatanodeDescriptor extends DatanodeInfo { * Remove block from the list of blocks belonging to the data-node. Remove * data-node from the block. */ - boolean removeBlock(String storageID, BlockInfoContiguous b) { + boolean removeBlock(String storageID, BlockInfo b) { DatanodeStorageInfo s = getStorageInfo(storageID); if (s != null) { return s.removeBlock(b); @@ -537,12 +537,12 @@ public class DatanodeDescriptor extends DatanodeInfo { } } - private static class BlockIterator implements Iterator<BlockInfoContiguous> { + private static class BlockIterator implements Iterator<BlockInfo> { private int index = 0; - private final List<Iterator<BlockInfoContiguous>> iterators; + private final List<Iterator<BlockInfo>> iterators; private BlockIterator(final DatanodeStorageInfo... storages) { - List<Iterator<BlockInfoContiguous>> iterators = new ArrayList<Iterator<BlockInfoContiguous>>(); + List<Iterator<BlockInfo>> iterators = new ArrayList<Iterator<BlockInfo>>(); for (DatanodeStorageInfo e : storages) { iterators.add(e.getBlockIterator()); } @@ -556,7 +556,7 @@ public class DatanodeDescriptor extends DatanodeInfo { } @Override - public BlockInfoContiguous next() { + public BlockInfo next() { update(); return iterators.get(index).next(); } @@ -573,10 +573,10 @@ public class DatanodeDescriptor extends DatanodeInfo { } } - Iterator<BlockInfoContiguous> getBlockIterator() { + Iterator<BlockInfo> getBlockIterator() { return new BlockIterator(getStorageInfos()); } - Iterator<BlockInfoContiguous> getBlockIterator(final String storageID) { + Iterator<BlockInfo> getBlockIterator(final String storageID) { return new BlockIterator(getStorageInfo(storageID)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index c6c9001..57e81b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -25,9 +25,6 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State; import org.apache.hadoop.hdfs.server.protocol.StorageReport; @@ -83,10 +80,10 @@ public class DatanodeStorageInfo { /** * Iterates over the list of blocks belonging to the data-node. */ - class BlockIterator implements Iterator<BlockInfoContiguous> { - private BlockInfoContiguous current; + class BlockIterator implements Iterator<BlockInfo> { + private BlockInfo current; - BlockIterator(BlockInfoContiguous head) { + BlockIterator(BlockInfo head) { this.current = head; } @@ -94,8 +91,8 @@ public class DatanodeStorageInfo { return current != null; } - public BlockInfoContiguous next() { - BlockInfoContiguous res = current; + public BlockInfo next() { + BlockInfo res = current; current = current.getNext(current.findStorageInfo(DatanodeStorageInfo.this)); return res; } @@ -115,7 +112,7 @@ public class DatanodeStorageInfo { private volatile long remaining; private long blockPoolUsed; - private volatile BlockInfoContiguous blockList = null; + private volatile BlockInfo blockList = null; private int numBlocks = 0; // The ID of the last full block report which updated this storage. @@ -229,7 +226,7 @@ public class DatanodeStorageInfo { return blockPoolUsed; } - public AddBlockResult addBlock(BlockInfoContiguous b) { + public AddBlockResult addBlock(BlockInfo b) { // First check whether the block belongs to a different storage // on the same DN. AddBlockResult result = AddBlockResult.ADDED; @@ -254,7 +251,7 @@ public class DatanodeStorageInfo { return result; } - public boolean removeBlock(BlockInfoContiguous b) { + public boolean removeBlock(BlockInfo b) { blockList = b.listRemove(blockList, this); if (b.removeStorage(this)) { numBlocks--; @@ -268,7 +265,7 @@ public class DatanodeStorageInfo { return numBlocks; } - Iterator<BlockInfoContiguous> getBlockIterator() { + Iterator<BlockInfo> getBlockIterator() { return new BlockIterator(blockList); } @@ -277,7 +274,7 @@ public class DatanodeStorageInfo { * Move block to the head of the list of blocks belonging to the data-node. * @return the index of the head of the blockList */ - int moveBlockToHead(BlockInfoContiguous b, int curIndex, int headIndex) { + int moveBlockToHead(BlockInfo b, int curIndex, int headIndex) { blockList = b.moveBlockToHead(blockList, this, curIndex, headIndex); return curIndex; } @@ -287,7 +284,7 @@ public class DatanodeStorageInfo { * @return the head of the blockList */ @VisibleForTesting - BlockInfoContiguous getBlockListHeadForTesting(){ + BlockInfo getBlockListHeadForTesting(){ return blockList; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java index 5f7366e..639d2b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java @@ -100,7 +100,7 @@ public class DecommissionManager { * reports or other events. Before being finally marking as decommissioned, * another check is done with the actual block map. */ - private final TreeMap<DatanodeDescriptor, AbstractList<BlockInfoContiguous>> + private final TreeMap<DatanodeDescriptor, AbstractList<BlockInfo>> decomNodeBlocks; /** @@ -248,7 +248,7 @@ public class DecommissionManager { * Full-strength replication is not always necessary, hence "sufficient". * @return true if sufficient, else false. */ - private boolean isSufficientlyReplicated(BlockInfoContiguous block, + private boolean isSufficientlyReplicated(BlockInfo block, BlockCollection bc, NumberReplicas numberReplicas) { final int numExpected = bc.getPreferredBlockReplication(); @@ -412,7 +412,7 @@ public class DecommissionManager { } private void check() { - final Iterator<Map.Entry<DatanodeDescriptor, AbstractList<BlockInfoContiguous>>> + final Iterator<Map.Entry<DatanodeDescriptor, AbstractList<BlockInfo>>> it = new CyclicIteration<>(decomNodeBlocks, iterkey).iterator(); final LinkedList<DatanodeDescriptor> toRemove = new LinkedList<>(); @@ -420,10 +420,10 @@ public class DecommissionManager { && !exceededNumBlocksPerCheck() && !exceededNumNodesPerCheck()) { numNodesChecked++; - final Map.Entry<DatanodeDescriptor, AbstractList<BlockInfoContiguous>> + final Map.Entry<DatanodeDescriptor, AbstractList<BlockInfo>> entry = it.next(); final DatanodeDescriptor dn = entry.getKey(); - AbstractList<BlockInfoContiguous> blocks = entry.getValue(); + AbstractList<BlockInfo> blocks = entry.getValue(); boolean fullScan = false; if (blocks == null) { // This is a newly added datanode, run through its list to schedule @@ -495,7 +495,7 @@ public class DecommissionManager { * datanode. */ private void pruneSufficientlyReplicated(final DatanodeDescriptor datanode, - AbstractList<BlockInfoContiguous> blocks) { + AbstractList<BlockInfo> blocks) { processBlocksForDecomInternal(datanode, blocks.iterator(), null, true); } @@ -509,9 +509,9 @@ public class DecommissionManager { * @param datanode * @return List of insufficiently replicated blocks */ - private AbstractList<BlockInfoContiguous> handleInsufficientlyReplicated( + private AbstractList<BlockInfo> handleInsufficientlyReplicated( final DatanodeDescriptor datanode) { - AbstractList<BlockInfoContiguous> insufficient = new ChunkedArrayList<>(); + AbstractList<BlockInfo> insufficient = new ChunkedArrayList<>(); processBlocksForDecomInternal(datanode, datanode.getBlockIterator(), insufficient, false); return insufficient; @@ -535,8 +535,8 @@ public class DecommissionManager { */ private void processBlocksForDecomInternal( final DatanodeDescriptor datanode, - final Iterator<BlockInfoContiguous> it, - final List<BlockInfoContiguous> insufficientlyReplicated, + final Iterator<BlockInfo> it, + final List<BlockInfo> insufficientlyReplicated, boolean pruneSufficientlyReplicated) { boolean firstReplicationLog = true; int underReplicatedBlocks = 0; @@ -544,7 +544,7 @@ public class DecommissionManager { int underReplicatedInOpenFiles = 0; while (it.hasNext()) { numBlocksChecked++; - final BlockInfoContiguous block = it.next(); + final BlockInfo block = it.next(); // Remove the block from the list if it's no longer in the block map, // e.g. the containing file has been deleted if (blockManager.blocksMap.getStoredBlock(block) == null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index 307bd59..3b5f973 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; @@ -235,7 +235,7 @@ class FSDirWriteFileOp { return onRetryBlock[0]; } else { // add new chosen targets to already allocated block and return - BlockInfoContiguous lastBlockInFile = pendingFile.getLastBlock(); + BlockInfo lastBlockInFile = pendingFile.getLastBlock(); ((BlockInfoContiguousUnderConstruction) lastBlockInFile) .setExpectedLocations(targets); offset = pendingFile.computeFileSize(); @@ -507,7 +507,7 @@ class FSDirWriteFileOp { /** * Add a block to the file. Returns a reference to the added block. */ - private static BlockInfoContiguous addBlock( + private static BlockInfo addBlock( FSDirectory fsd, String path, INodesInPath inodesInPath, Block block, DatanodeStorageInfo[] targets) throws IOException { fsd.writeLock(); @@ -608,7 +608,7 @@ class FSDirWriteFileOp { } } final INodeFile file = fsn.checkLease(src, clientName, inode, fileId); - BlockInfoContiguous lastBlockInFile = file.getLastBlock(); + BlockInfo lastBlockInFile = file.getLastBlock(); if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) { // The block that the client claims is the current last block // doesn't match up with what we think is the last block. There are @@ -636,7 +636,7 @@ class FSDirWriteFileOp { // changed the namesystem state yet. // We run this analysis again in Part II where case 4 is impossible. - BlockInfoContiguous penultimateBlock = file.getPenultimateBlock(); + BlockInfo penultimateBlock = file.getPenultimateBlock(); if (previous == null && lastBlockInFile != null && lastBlockInFile.getNumBytes() >= file.getPreferredBlockSize() && @@ -762,7 +762,7 @@ class FSDirWriteFileOp { long id, PermissionStatus permissions, long mtime, long atime, short replication, long preferredBlockSize, byte storagePolicyId) { return new INodeFile(id, null, permissions, mtime, atime, - BlockInfoContiguous.EMPTY_ARRAY, replication, preferredBlockSize, + BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize, storagePolicyId); } @@ -802,7 +802,7 @@ class FSDirWriteFileOp { DatanodeStorageInfo[] targets) throws IOException { assert fsn.hasWriteLock(); - BlockInfoContiguous b = addBlock(fsn.dir, src, inodesInPath, newBlock, + BlockInfo b = addBlock(fsn.dir, src, inodesInPath, newBlock, targets); NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src); DatanodeStorageInfo.incrementBlocksScheduled(targets); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 8fdd2d7..5b8650e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -33,10 +33,8 @@ import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.XAttrSetFlag; -import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.XAttrHelper; @@ -51,7 +49,7 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; import org.apache.hadoop.hdfs.protocolPB.PBHelper; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; @@ -919,7 +917,7 @@ public class FSDirectory implements Closeable { unprotectedTruncate(iip, newLength, collectedBlocks, mtime, null); if(! onBlockBoundary) { - BlockInfoContiguous oldBlock = file.getLastBlock(); + BlockInfo oldBlock = file.getLastBlock(); Block tBlk = getFSNamesystem().prepareFileForTruncate(iip, clientName, clientMachine, file.computeFileSize() - newLength, http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index 83e52bc..dec9709 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -44,7 +44,7 @@ import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.Storage.FormatConfirmable; @@ -773,10 +773,10 @@ public class FSEditLog implements LogsPurgeable { public void logAddBlock(String path, INodeFile file) { Preconditions.checkArgument(file.isUnderConstruction()); - BlockInfoContiguous[] blocks = file.getBlocks(); + BlockInfo[] blocks = file.getBlocks(); Preconditions.checkState(blocks != null && blocks.length > 0); - BlockInfoContiguous pBlock = blocks.length > 1 ? blocks[blocks.length - 2] : null; - BlockInfoContiguous lastBlock = blocks[blocks.length - 1]; + BlockInfo pBlock = blocks.length > 1 ? blocks[blocks.length - 2] : null; + BlockInfo lastBlock = blocks[blocks.length - 1]; AddBlockOp op = AddBlockOp.getInstance(cache.get()).setPath(path) .setPenultimateBlock(pBlock).setLastBlock(lastBlock); logEdit(op);