Repository: hadoop Updated Branches: refs/heads/HDFS-7285 96ab5bd27 -> 4e528be3b
HDFS-8372. Erasure coding: compute storage type quotas for striped files, to be consistent with HDFS-8327. Contributed by Zhe Zhang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e528be3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e528be3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e528be3 Branch: refs/heads/HDFS-7285 Commit: 4e528be3b6dab99df4ebbb062e3cec68bc5f6571 Parents: 96ab5bd Author: Jing Zhao <ji...@apache.org> Authored: Tue May 12 11:43:04 2015 -0700 Committer: Jing Zhao <ji...@apache.org> Committed: Tue May 12 11:43:04 2015 -0700 ---------------------------------------------------------------------- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 ++ .../hdfs/server/namenode/FSDirectory.java | 3 +- .../namenode/FileWithStripedBlocksFeature.java | 12 +++-- .../hadoop/hdfs/server/namenode/INodeFile.java | 53 +++++++++++++++----- .../server/namenode/TestStripedINodeFile.java | 22 ++++---- 5 files changed, 66 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e528be3/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 0acf746..f026a5c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -198,3 +198,6 @@ HDFS-7678. Erasure coding: DFSInputStream with decode functionality (pread). (Zhe Zhang) + + HDFS-8372. Erasure coding: compute storage type quotas for striped files, + to be consistent with HDFS-8327. (Zhe Zhang via jing9) http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e528be3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 3f1b182..5cf103c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -59,6 +59,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; @@ -556,7 +557,7 @@ public class FSDirectory implements Closeable { INodeFile fileNode, Block block) throws IOException { // modify file-> block and blocksMap // fileNode should be under construction - BlockInfoContiguousUnderConstruction uc = fileNode.removeLastBlock(block); + BlockInfoUnderConstruction uc = fileNode.removeLastBlock(block); if (uc == null) { return false; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e528be3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileWithStripedBlocksFeature.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileWithStripedBlocksFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileWithStripedBlocksFeature.java index 47445be..94ab527 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileWithStripedBlocksFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileWithStripedBlocksFeature.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction; /** * Feature for file with striped blocks @@ -78,20 +79,23 @@ class FileWithStripedBlocksFeature implements INode.Feature { } } - boolean removeLastBlock(Block oldblock) { + BlockInfoStripedUnderConstruction removeLastBlock( + Block oldblock) { if (blocks == null || blocks.length == 0) { - return false; + return null; } int newSize = blocks.length - 1; if (!blocks[newSize].equals(oldblock)) { - return false; + return null; } + BlockInfoStripedUnderConstruction uc = + (BlockInfoStripedUnderConstruction) blocks[newSize]; //copy to a new list BlockInfoStriped[] newlist = new BlockInfoStriped[newSize]; System.arraycopy(blocks, 0, newlist, 0, newSize); setBlocks(newlist); - return true; + return uc; } void truncateStripedBlocks(int n) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e528be3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 79d4a29..2c23424 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff; @@ -295,7 +296,7 @@ public class INodeFile extends INodeWithAdditionalFields * Remove a block from the block list. This block should be * the last one on the list. */ - BlockInfoContiguousUnderConstruction removeLastBlock(Block oldblock) { + BlockInfoUnderConstruction removeLastBlock(Block oldblock) { Preconditions.checkState(isUnderConstruction(), "file is no longer under construction"); FileWithStripedBlocksFeature sb = getStripedBlocksFeature(); @@ -317,7 +318,7 @@ public class INodeFile extends INodeWithAdditionalFields return uc; } else { assert hasNoContiguousBlock(); - return null; + return sb.removeLastBlock(oldblock); } } @@ -678,7 +679,7 @@ public class INodeFile extends INodeWithAdditionalFields final long ssDeltaNoReplication; short replication; if (isStriped()) { - return computeQuotaUsageWithStriped(bsps, counts); + return computeQuotaUsageWithStriped(bsp, counts); } if (last < lastSnapshotId) { @@ -704,11 +705,15 @@ public class INodeFile extends INodeWithAdditionalFields } /** - * Compute quota of striped file + * Compute quota of striped file. Note that currently EC files do not support + * append/hflush/hsync, thus the file length recorded in snapshots should be + * the same with the current file length. */ public final QuotaCounts computeQuotaUsageWithStriped( - BlockStoragePolicySuite bsps, QuotaCounts counts) { - return null; + BlockStoragePolicy bsp, QuotaCounts counts) { + counts.addNameSpace(1); + counts.add(storagespaceConsumed(bsp)); + return counts; } @Override @@ -830,21 +835,44 @@ public class INodeFile extends INodeWithAdditionalFields * Use preferred block size for the last block if it is under construction. */ public final QuotaCounts storagespaceConsumed(BlockStoragePolicy bsp) { - QuotaCounts counts = new QuotaCounts.Builder().build(); if (isStriped()) { - return storagespaceConsumedWithStriped(bsp); + return storagespaceConsumedWithStriped(); } else { return storagespaceConsumedWithReplication(bsp); } } - public final QuotaCounts storagespaceConsumedWithStriped( - BlockStoragePolicy bsp) { - return null; + // TODO: support EC with heterogeneous storage + public final QuotaCounts storagespaceConsumedWithStriped() { + QuotaCounts counts = new QuotaCounts.Builder().build(); + BlockInfo[] blockInfos = getBlocks(); + if (blockInfos == null || blockInfos.length == 0) { + return counts; + } + + long size; + final int last = blockInfos.length - 1; + if (blockInfos[last] instanceof BlockInfoStripedUnderConstruction) { + BlockInfoStripedUnderConstruction blockInfoStripedUC + =(BlockInfoStripedUnderConstruction)blockInfos[last]; + size = getPreferredBlockSize() * blockInfoStripedUC.getTotalBlockNum(); + } else { + // In case of last block is complete + BlockInfoStriped blockInfoStriped = (BlockInfoStriped)blockInfos[last]; + size = blockInfoStriped.spaceConsumed(); + } + for (int i = 0; i < last; i++) { + BlockInfoStriped blockInfoStriped = (BlockInfoStriped)blockInfos[i]; + size += blockInfoStriped.spaceConsumed(); + } + + counts.addStorageSpace(size); + return counts; } public final QuotaCounts storagespaceConsumedWithReplication( - BlockStoragePolicy bsp) { QuotaCounts counts = new QuotaCounts.Builder().build(); + BlockStoragePolicy bsp) { + QuotaCounts counts = new QuotaCounts.Builder().build(); final Iterable<BlockInfo> blocks; FileWithSnapshotFeature sf = getFileWithSnapshotFeature(); if (sf == null) { @@ -967,6 +995,7 @@ public class INodeFile extends INodeWithAdditionalFields /** * compute the quota usage change for a truncate op * @param newLength the length for truncation + * TODO: properly handle striped blocks (HDFS-7622) **/ void computeQuotaDeltaForTruncate( long newLength, BlockStoragePolicy bsps, http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e528be3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java index a725e6b..e24bc54 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction; @@ -45,6 +46,11 @@ public class TestStripedINodeFile { private static final PermissionStatus perm = new PermissionStatus( "userName", null, FsPermission.getDefault()); + private final BlockStoragePolicySuite defaultSuite = + BlockStoragePolicySuite.createDefaultSuite(); + private final BlockStoragePolicy defaultPolicy = + defaultSuite.getDefaultPolicy(); + private static INodeFile createStripedINodeFile() { return new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L, null, (short)0, 1024L, HdfsServerConstants.COLD_STORAGE_POLICY_ID); @@ -109,8 +115,8 @@ public class TestStripedINodeFile { // a. <Cell Size> * (<Num Stripes> - 1) * <Total Block Num> = 0 // b. <Num Bytes> % <Num Bytes per Stripes> = 1 // c. <Last Stripe Length> * <Parity Block Num> = 1 * 3 - assertEquals(4, inf.storagespaceConsumedWithStriped(null)); - assertEquals(4, inf.storagespaceConsumed(null)); + assertEquals(4, inf.storagespaceConsumedWithStriped().getStorageSpace()); + assertEquals(4, inf.storagespaceConsumed(defaultPolicy).getStorageSpace()); } @Test @@ -134,8 +140,8 @@ public class TestStripedINodeFile { inf.addBlock(blockInfoStriped1); inf.addBlock(blockInfoStriped2); // This is the double size of one block in above case. - assertEquals(4 * 2, inf.storagespaceConsumedWithStriped(null)); - assertEquals(4 * 2, inf.storagespaceConsumed(null)); + assertEquals(4 * 2, inf.storagespaceConsumedWithStriped().getStorageSpace()); + assertEquals(4 * 2, inf.storagespaceConsumed(defaultPolicy).getStorageSpace()); } @Test @@ -188,10 +194,8 @@ public class TestStripedINodeFile { blockInfoStriped.setNumBytes(100); inf.addBlock(blockInfoStriped); - BlockStoragePolicySuite suite = - BlockStoragePolicySuite.createDefaultSuite(); QuotaCounts counts = - inf.computeQuotaUsageWithStriped(suite, + inf.computeQuotaUsageWithStriped(defaultPolicy, new QuotaCounts.Builder().build()); assertEquals(1, counts.getNameSpace()); // The total consumed space is the sum of @@ -215,10 +219,8 @@ public class TestStripedINodeFile { bInfoStripedUC.setNumBytes(100); inf.addBlock(bInfoStripedUC); - BlockStoragePolicySuite suite - = BlockStoragePolicySuite.createDefaultSuite(); QuotaCounts counts - = inf.computeQuotaUsageWithStriped(suite, + = inf.computeQuotaUsageWithStriped(defaultPolicy, new QuotaCounts.Builder().build()); assertEquals(1024, inf.getPreferredBlockSize()); assertEquals(1, counts.getNameSpace());