HDFS-8375. Add cellSize as an XAttr to ECZone. Contributed by Vinayakumar B.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c36df15 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c36df15 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c36df15 Branch: refs/heads/HDFS-7285 Commit: 3c36df1540e3483e34acd1995a39d9960b06eff6 Parents: 3676277 Author: Zhe Zhang <z...@apache.org> Authored: Tue May 19 13:58:50 2015 -0700 Committer: Zhe Zhang <z...@apache.org> Committed: Tue May 19 13:58:50 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hdfs/protocol/HdfsFileStatus.java | 14 ++++-- .../protocol/SnapshottableDirectoryStatus.java | 2 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 2 +- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 8 ++-- .../java/org/apache/hadoop/hdfs/DFSClient.java | 7 +-- .../hadoop/hdfs/DFSStripedInputStream.java | 10 ++-- .../hadoop/hdfs/DFSStripedOutputStream.java | 2 +- .../hadoop/hdfs/DistributedFileSystem.java | 9 ++-- .../hadoop/hdfs/protocol/ClientProtocol.java | 2 +- .../hdfs/protocol/ErasureCodingZoneInfo.java | 14 +++++- .../hdfs/protocol/HdfsLocatedFileStatus.java | 5 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 3 +- .../ClientNamenodeProtocolTranslatorPB.java | 5 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 15 ++++-- .../server/blockmanagement/BlockManager.java | 12 +++-- .../blockmanagement/DatanodeDescriptor.java | 4 +- .../erasurecode/ErasureCodingWorker.java | 2 +- .../namenode/ErasureCodingZoneManager.java | 40 +++++++++++++--- .../server/namenode/FSDirStatAndListingOp.java | 21 ++++++--- .../hdfs/server/namenode/FSDirectory.java | 4 +- .../hdfs/server/namenode/FSNamesystem.java | 49 +++++++------------- .../hdfs/server/namenode/NameNodeRpcServer.java | 5 +- .../hadoop/hdfs/server/namenode/Namesystem.java | 8 ++-- .../server/protocol/BlockECRecoveryCommand.java | 26 ++++++----- .../hdfs/tools/erasurecode/ECCommand.java | 14 +++++- .../hadoop/hdfs/util/StripedBlockUtil.java | 49 +++++++++----------- .../src/main/proto/erasurecoding.proto | 3 ++ .../hadoop-hdfs/src/main/proto/hdfs.proto | 3 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 2 +- .../hadoop/hdfs/TestDFSClientRetries.java | 4 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 10 ++-- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 +- .../TestDFSStripedOutputStreamWithFailure.java | 2 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 2 +- .../hadoop/hdfs/TestErasureCodingZones.java | 18 +++---- .../hadoop/hdfs/TestFileStatusWithECschema.java | 2 +- .../java/org/apache/hadoop/hdfs/TestLease.java | 4 +- .../hadoop/hdfs/TestRecoverStripedFile.java | 2 +- .../hadoop/hdfs/TestWriteReadStripedFile.java | 2 +- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 6 ++- .../server/namenode/TestAddStripedBlocks.java | 2 +- .../server/namenode/TestFSEditLogLoader.java | 4 +- .../hdfs/server/namenode/TestFSImage.java | 4 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 4 +- .../namenode/TestQuotaWithStripedBlocks.java | 2 +- ...TestOfflineImageViewerWithStripedBlocks.java | 2 +- .../hadoop/hdfs/util/TestStripedBlockUtil.java | 4 +- .../apache/hadoop/hdfs/web/TestJsonUtil.java | 2 +- 48 files changed, 244 insertions(+), 174 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java index f07973a..8c902b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java @@ -49,7 +49,8 @@ public class HdfsFileStatus { private final FileEncryptionInfo feInfo; - private final ECSchema schema; + private final ECSchema ecSchema; + private final int stripeCellSize; // Used by dir, not including dot and dotdot. Always zero for a regular file. private final int childrenNum; @@ -76,7 +77,7 @@ public class HdfsFileStatus { long blocksize, long modification_time, long access_time, FsPermission permission, String owner, String group, byte[] symlink, byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo, - byte storagePolicy, ECSchema schema) { + byte storagePolicy, ECSchema ecSchema, int stripeCellSize) { this.length = length; this.isdir = isdir; this.block_replication = (short)block_replication; @@ -96,7 +97,8 @@ public class HdfsFileStatus { this.childrenNum = childrenNum; this.feInfo = feInfo; this.storagePolicy = storagePolicy; - this.schema = schema; + this.ecSchema = ecSchema; + this.stripeCellSize = stripeCellSize; } /** @@ -255,7 +257,11 @@ public class HdfsFileStatus { } public ECSchema getECSchema() { - return schema; + return ecSchema; + } + + public int getStripeCellSize() { + return stripeCellSize; } public final int getChildrenNum() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java index 813ea26..a6c7b10 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java @@ -61,7 +61,7 @@ public class SnapshottableDirectoryStatus { int snapshotNumber, int snapshotQuota, byte[] parentFullPath) { this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time, access_time, permission, owner, group, null, localName, inodeId, - childrenNum, null, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null); + childrenNum, null, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null, 0); this.snapshotNumber = snapshotNumber; this.snapshotQuota = snapshotQuota; this.parentFullPath = parentFullPath; http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index 62f679b..5635c1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -132,7 +132,7 @@ class JsonUtilClient { blockSize, mTime, aTime, permission, owner, group, symlink, DFSUtilClient.string2Bytes(localName), fileId, childrenNum, null, - storagePolicy, null); + storagePolicy, null, 0); } /** Convert a Json map to an ExtendedBlock object. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 1e7dbea..d6c9dba 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -227,9 +227,11 @@ (Yi Liu via jing9) HDFS-8320. Erasure coding: consolidate striping-related terminologies. (zhz) - - HDFS-8366. Erasure Coding: Make the timeout parameter of polling blocking queue - configurable in DFSStripedOutputStream. (Li Bo) + + HDFS-8366. Erasure Coding: Make the timeout parameter of polling blocking queue + configurable in DFSStripedOutputStream. (Li Bo) HDFS-8378. Erasure Coding: Few improvements for the erasure coding worker. (Rakesh R via waltersu4549) + + HDFS-8375. Add cellSize as an XAttr to ECZone. ( Vinayakumar B via zhz). http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index cde1fc8..b77ae9c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -1195,7 +1195,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, if (fileInfo != null) { ECSchema schema = fileInfo.getECSchema(); if (schema != null) { - return new DFSStripedInputStream(this, src, verifyChecksum, schema); + return new DFSStripedInputStream(this, src, verifyChecksum, schema, + fileInfo.getStripeCellSize()); } } return new DFSInputStream(this, src, verifyChecksum); @@ -3007,12 +3008,12 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, return new EncryptionZoneIterator(namenode, traceSampler); } - public void createErasureCodingZone(String src, ECSchema schema) + public void createErasureCodingZone(String src, ECSchema schema, int cellSize) throws IOException { checkOpen(); TraceScope scope = getPathTraceScope("createErasureCodingZone", src); try { - namenode.createErasureCodingZone(src, schema); + namenode.createErasureCodingZone(src, schema, cellSize); } catch (RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, SafeModeException.class, http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java index 744d586..3b7eb58 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java @@ -125,12 +125,12 @@ public class DFSStripedInputStream extends DFSInputStream { private final CompletionService<Integer> readingService; DFSStripedInputStream(DFSClient dfsClient, String src, boolean verifyChecksum, - ECSchema schema) throws IOException { + ECSchema schema, int cellSize) throws IOException { super(dfsClient, src, verifyChecksum); assert schema != null; this.schema = schema; - cellSize = schema.getChunkSize(); + this.cellSize = cellSize; dataBlkNum = (short) schema.getNumDataUnits(); parityBlkNum = (short) schema.getNumParityUnits(); groupSize = dataBlkNum; @@ -189,7 +189,7 @@ public class DFSStripedInputStream extends DFSInputStream { targetBlockGroup, cellSize, dataBlkNum, parityBlkNum); // The purpose is to get start offset into each block. long[] offsetsForInternalBlocks = getStartOffsetsForInternalBlocks(schema, - targetBlockGroup, offsetIntoBlockGroup); + cellSize, targetBlockGroup, offsetIntoBlockGroup); Preconditions.checkNotNull(offsetsForInternalBlocks); final ReaderRetryPolicy retry = new ReaderRetryPolicy(); @@ -514,8 +514,8 @@ public class DFSStripedInputStream extends DFSInputStream { // Refresh the striped block group LocatedStripedBlock blockGroup = getBlockGroupAt(blockStartOffset); - AlignedStripe[] stripes = divideByteRangeIntoStripes(schema, blockGroup, - start, end, buf, offset); + AlignedStripe[] stripes = divideByteRangeIntoStripes(schema, cellSize, + blockGroup, start, end, buf, offset); for (AlignedStripe stripe : stripes) { fetchOneStripe(blockGroup, buf, stripe, corruptedBlockMap); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index a648023..4399a37 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -230,7 +230,7 @@ public class DFSStripedOutputStream extends DFSOutputStream { final ECSchema schema = stat.getECSchema(); final int numParityBlocks = schema.getNumParityUnits(); - cellSize = schema.getChunkSize(); + cellSize = stat.getStripeCellSize(); numDataBlocks = schema.getNumDataUnits(); numAllBlocks = numDataBlocks + numParityBlocks; http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 0cd8334..2e21372 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -2281,16 +2281,17 @@ public class DistributedFileSystem extends FileSystem { * * @param path Directory to create the ec zone * @param schema ECSchema for the zone. If not specified default will be used. + * @param cellSize Cellsize for the striped erasure coding * @throws IOException */ - public void createErasureCodingZone(final Path path, final ECSchema schema) - throws IOException { + public void createErasureCodingZone(final Path path, final ECSchema schema, + final int cellSize) throws IOException { Path absF = fixRelativePart(path); new FileSystemLinkResolver<Void>() { @Override public Void doCall(final Path p) throws IOException, UnresolvedLinkException { - dfs.createErasureCodingZone(getPathName(p), schema); + dfs.createErasureCodingZone(getPathName(p), schema, cellSize); return null; } @@ -2298,7 +2299,7 @@ public class DistributedFileSystem extends FileSystem { public Void next(final FileSystem fs, final Path p) throws IOException { if (fs instanceof DistributedFileSystem) { DistributedFileSystem myDfs = (DistributedFileSystem) fs; - myDfs.createErasureCodingZone(p, schema); + myDfs.createErasureCodingZone(p, schema, cellSize); return null; } throw new UnsupportedOperationException( http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index b0b457c..4f985ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -1463,7 +1463,7 @@ public interface ClientProtocol { * default */ @AtMostOnce - public void createErasureCodingZone(String src, ECSchema schema) + public void createErasureCodingZone(String src, ECSchema schema, int cellSize) throws IOException; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingZoneInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingZoneInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingZoneInfo.java index ec0efbd..282eeaf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingZoneInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingZoneInfo.java @@ -25,10 +25,12 @@ public class ErasureCodingZoneInfo { private String dir; private ECSchema schema; + private int cellSize; - public ErasureCodingZoneInfo(String dir, ECSchema schema) { + public ErasureCodingZoneInfo(String dir, ECSchema schema, int cellSize) { this.dir = dir; this.schema = schema; + this.cellSize = cellSize; } /** @@ -49,8 +51,16 @@ public class ErasureCodingZoneInfo { return schema; } + /** + * Get cellSize for the EC Zone + */ + public int getCellSize() { + return cellSize; + } + @Override public String toString() { - return "Dir: " + getDir() + ", Schema: " + schema; + return "Dir: " + getDir() + ", Schema: " + schema + ", cellSize: " + + cellSize; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java index 9194d26..4701538 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java @@ -59,10 +59,11 @@ public class HdfsLocatedFileStatus extends HdfsFileStatus { int block_replication, long blocksize, long modification_time, long access_time, FsPermission permission, String owner, String group, byte[] symlink, byte[] path, long fileId, LocatedBlocks locations, - int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy, ECSchema schema) { + int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy, + ECSchema schema, int stripeCellSize) { super(length, isdir, block_replication, blocksize, modification_time, access_time, permission, owner, group, symlink, path, fileId, - childrenNum, feInfo, storagePolicy, schema); + childrenNum, feInfo, storagePolicy, schema, stripeCellSize); this.locations = locations; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index bae753b..863b217 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -1408,7 +1408,8 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements try { ECSchema schema = req.hasSchema() ? PBHelper.convertECSchema(req .getSchema()) : null; - server.createErasureCodingZone(req.getSrc(), schema); + int cellSize = req.hasCellSize() ? req.getCellSize() : 0; + server.createErasureCodingZone(req.getSrc(), schema, cellSize); return CreateErasureCodingZoneResponseProto.newBuilder().build(); } catch (IOException e) { throw new ServiceException(e); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 67b1457..336e3a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -1422,7 +1422,7 @@ public class ClientNamenodeProtocolTranslatorPB implements } @Override - public void createErasureCodingZone(String src, ECSchema schema) + public void createErasureCodingZone(String src, ECSchema schema, int cellSize) throws IOException { final CreateErasureCodingZoneRequestProto.Builder builder = CreateErasureCodingZoneRequestProto.newBuilder(); @@ -1430,6 +1430,9 @@ public class ClientNamenodeProtocolTranslatorPB implements if (schema != null) { builder.setSchema(PBHelper.convertECSchema(schema)); } + if (cellSize > 0) { + builder.setCellSize(cellSize); + } CreateErasureCodingZoneRequestProto req = builder.build(); try { rpcProxy.createErasureCodingZone(null, req); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index a6a356c..4d0f871 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -1506,7 +1506,8 @@ public class PBHelper { fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : null, fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy() : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, - fs.hasEcSchema() ? PBHelper.convertECSchema(fs.getEcSchema()) : null); + fs.hasEcSchema() ? PBHelper.convertECSchema(fs.getEcSchema()) : null, + fs.hasStripeCellSize() ? fs.getStripeCellSize() : 0); } public static SnapshottableDirectoryStatus convert( @@ -1570,6 +1571,7 @@ public class PBHelper { if(fs.getECSchema() != null) { builder.setEcSchema(PBHelper.convertECSchema(fs.getECSchema())); } + builder.setStripeCellSize(fs.getStripeCellSize()); return builder.build(); } @@ -3157,12 +3159,14 @@ public class PBHelper { public static ErasureCodingZoneInfoProto convertECZoneInfo(ErasureCodingZoneInfo ecZoneInfo) { return ErasureCodingZoneInfoProto.newBuilder().setDir(ecZoneInfo.getDir()) - .setSchema(convertECSchema(ecZoneInfo.getSchema())).build(); + .setSchema(convertECSchema(ecZoneInfo.getSchema())) + .setCellSize(ecZoneInfo.getCellSize()).build(); } public static ErasureCodingZoneInfo convertECZoneInfo(ErasureCodingZoneInfoProto ecZoneInfoProto) { return new ErasureCodingZoneInfo(ecZoneInfoProto.getDir(), - convertECSchema(ecZoneInfoProto.getSchema())); + convertECSchema(ecZoneInfoProto.getSchema()), + ecZoneInfoProto.getCellSize()); } public static BlockECRecoveryInfo convertBlockECRecoveryInfo( @@ -3196,9 +3200,11 @@ public class PBHelper { } ECSchema ecSchema = convertECSchema(blockEcRecoveryInfoProto.getEcSchema()); + int cellSize = blockEcRecoveryInfoProto.getCellSize(); return new BlockECRecoveryInfo(block, sourceDnInfos, targetDnInfos, - targetStorageUuids, convertStorageTypes, liveBlkIndices, ecSchema); + targetStorageUuids, convertStorageTypes, liveBlkIndices, ecSchema, + cellSize); } public static BlockECRecoveryInfoProto convertBlockECRecoveryInfo( @@ -3224,6 +3230,7 @@ public class PBHelper { builder.addAllLiveBlockIndices(convertIntArray(liveBlockIndices)); builder.setEcSchema(convertECSchema(blockEcRecoveryInfo.getECSchema())); + builder.setCellSize(blockEcRecoveryInfo.getCellSize()); return builder.build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 2215b65..de3b50f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs.BlockReportReplica; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingZoneInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlock; @@ -1558,14 +1559,14 @@ public class BlockManager { assert rw instanceof ErasureCodingWork; assert rw.targets.length > 0; String src = block.getBlockCollection().getName(); - ECSchema ecSchema = null; + ErasureCodingZoneInfo ecZoneInfo = null; try { - ecSchema = namesystem.getECSchemaForPath(src); + ecZoneInfo = namesystem.getErasureCodingZoneInfoForPath(src); } catch (IOException e) { blockLog - .warn("Failed to get the EC schema for the file {} ", src); + .warn("Failed to get the EC zone info for the file {} ", src); } - if (ecSchema == null) { + if (ecZoneInfo == null) { blockLog.warn("No EC schema found for the file {}. " + "So cannot proceed for recovery", src); // TODO: we may have to revisit later for what we can do better to @@ -1575,7 +1576,8 @@ public class BlockManager { rw.targets[0].getDatanodeDescriptor().addBlockToBeErasureCoded( new ExtendedBlock(namesystem.getBlockPoolId(), block), rw.srcNodes, rw.targets, - ((ErasureCodingWork) rw).liveBlockIndicies, ecSchema); + ((ErasureCodingWork) rw).liveBlockIndicies, + ecZoneInfo.getSchema(), ecZoneInfo.getCellSize()); } else { rw.srcNodes[0].addBlockToBeReplicated(block, targets); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 83d3303..47bc765 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -610,10 +610,10 @@ public class DatanodeDescriptor extends DatanodeInfo { */ void addBlockToBeErasureCoded(ExtendedBlock block, DatanodeDescriptor[] sources, DatanodeStorageInfo[] targets, - short[] liveBlockIndices, ECSchema ecSchema) { + short[] liveBlockIndices, ECSchema ecSchema, int cellSize) { assert (block != null && sources != null && sources.length > 0); BlockECRecoveryInfo task = new BlockECRecoveryInfo(block, sources, targets, - liveBlockIndices, ecSchema); + liveBlockIndices, ecSchema, cellSize); erasurecodeBlocks.offer(task); BlockManager.LOG.debug("Adding block recovery task " + task + "to " + getName() + ", current queue size is " + erasurecodeBlocks.size()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java index 4723e9f..d227de8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java @@ -269,7 +269,7 @@ public final class ErasureCodingWorker { ECSchema schema = recoveryInfo.getECSchema(); dataBlkNum = schema.getNumDataUnits(); parityBlkNum = schema.getNumParityUnits(); - cellSize = schema.getChunkSize(); + cellSize = recoveryInfo.getCellSize(); blockGroup = recoveryInfo.getExtendedBlock(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java index 6b6add7..371b8ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java @@ -19,12 +19,20 @@ package org.apache.hadoop.hdfs.server.namenode; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; + import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.ErasureCodingZoneInfo; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.io.erasurecode.ECSchema; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.EnumSet; @@ -78,17 +86,21 @@ public class ErasureCodingZoneManager { : inode.getXAttrFeature().getXAttrs(); for (XAttr xAttr : xAttrs) { if (XATTR_ERASURECODING_ZONE.equals(XAttrHelper.getPrefixName(xAttr))) { - String schemaName = new String(xAttr.getValue()); + ByteArrayInputStream bIn=new ByteArrayInputStream(xAttr.getValue()); + DataInputStream dIn=new DataInputStream(bIn); + int cellSize = WritableUtils.readVInt(dIn); + String schemaName = WritableUtils.readString(dIn); ECSchema schema = dir.getFSNamesystem().getECSchemaManager() .getSchema(schemaName); - return new ErasureCodingZoneInfo(inode.getFullPathName(), schema); + return new ErasureCodingZoneInfo(inode.getFullPathName(), schema, + cellSize); } } } return null; } - XAttr createErasureCodingZone(String src, ECSchema schema) + XAttr createErasureCodingZone(String src, ECSchema schema, int cellSize) throws IOException { assert dir.hasWriteLock(); final INodesInPath srcIIP = dir.getINodesInPath4Write(src, false); @@ -113,10 +125,24 @@ public class ErasureCodingZoneManager { schema = ErasureCodingSchemaManager.getSystemDefaultSchema(); } - // Now persist the schema name in xattr - byte[] schemaBytes = schema.getSchemaName().getBytes(); - final XAttr ecXAttr = XAttrHelper.buildXAttr(XATTR_ERASURECODING_ZONE, - schemaBytes); + if (cellSize <= 0) { + cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + } + + // Write the cellsize first and then schema name + final XAttr ecXAttr; + DataOutputStream dOut = null; + try { + ByteArrayOutputStream bOut = new ByteArrayOutputStream(); + dOut = new DataOutputStream(bOut); + WritableUtils.writeVInt(dOut, cellSize); + // Now persist the schema name in xattr + WritableUtils.writeString(dOut, schema.getSchemaName()); + ecXAttr = XAttrHelper.buildXAttr(XATTR_ERASURECODING_ZONE, + bOut.toByteArray()); + } finally { + IOUtils.closeStream(dOut); + } final List<XAttr> xattrs = Lists.newArrayListWithCapacity(1); xattrs.add(ecXAttr); FSDirXAttrOp.unprotectedSetXAttrs(dir, src, xattrs, http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index 7133cf1..eba5013 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.ErasureCodingZoneInfo; import org.apache.hadoop.hdfs.protocol.FsPermissionExtension; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -317,7 +318,7 @@ class FSDirStatAndListingOp { if (fsd.getINode4DotSnapshot(srcs) != null) { return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null, HdfsFileStatus.EMPTY_NAME, -1L, 0, null, - HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null); + HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null, 0); } return null; } @@ -385,8 +386,10 @@ class FSDirStatAndListingOp { final FileEncryptionInfo feInfo = isRawPath ? null : fsd.getFileEncryptionInfo(node, snapshot, iip); - final ECSchema schema = fsd.getECSchema(iip); - + final ErasureCodingZoneInfo ecZoneInfo = fsd.getECZoneInfo(iip); + final ECSchema schema = ecZoneInfo != null ? ecZoneInfo.getSchema() : null; + final int cellSize = ecZoneInfo != null ? ecZoneInfo.getCellSize() : 0; + if (node.isFile()) { final INodeFile fileNode = node.asFile(); size = fileNode.computeFileSize(snapshot); @@ -417,7 +420,8 @@ class FSDirStatAndListingOp { childrenNum, feInfo, storagePolicy, - schema); + schema, + cellSize); } private static INodeAttributes getINodeAttributes( @@ -464,8 +468,10 @@ class FSDirStatAndListingOp { } int childrenNum = node.isDirectory() ? node.asDirectory().getChildrenNum(snapshot) : 0; - final ECSchema schema = fsd.getECSchema(iip); - + final ErasureCodingZoneInfo ecZoneInfo = fsd.getECZoneInfo(iip); + final ECSchema schema = ecZoneInfo != null ? ecZoneInfo.getSchema() : null; + final int cellSize = ecZoneInfo != null ? ecZoneInfo.getCellSize() : 0; + HdfsLocatedFileStatus status = new HdfsLocatedFileStatus(size, node.isDirectory(), replication, blocksize, node.getModificationTime(snapshot), @@ -473,7 +479,8 @@ class FSDirStatAndListingOp { getPermissionForFileStatus(nodeAttrs, isEncrypted), nodeAttrs.getUserName(), nodeAttrs.getGroupName(), node.isSymlink() ? node.asSymlink().getSymlink() : null, path, - node.getId(), loc, childrenNum, feInfo, storagePolicy, schema); + node.getId(), loc, childrenNum, feInfo, storagePolicy, schema, + cellSize); // Set caching information for the located blocks. if (loc != null) { CacheManager cacheManager = fsd.getFSNamesystem().getCacheManager(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 68fa8b6..9a2b291 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -1329,11 +1329,11 @@ public class FSDirectory implements Closeable { } } - XAttr createErasureCodingZone(String src, ECSchema schema) + XAttr createErasureCodingZone(String src, ECSchema schema, int cellSize) throws IOException { writeLock(); try { - return ecZoneManager.createErasureCodingZone(src, schema); + return ecZoneManager.createErasureCodingZone(src, schema, cellSize); } finally { writeUnlock(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index fd4039e..16212e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -7755,14 +7755,14 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, * @param srcArg the path of a directory which will be the root of the * erasure coding zone. The directory must be empty. * @param schema ECSchema for the erasure coding zone - * + * @param cellSize Cell size of stripe * @throws AccessControlException if the caller is not the superuser. * @throws UnresolvedLinkException if the path can't be resolved. * @throws SafeModeException if the Namenode is in safe mode. */ void createErasureCodingZone(final String srcArg, final ECSchema schema, - final boolean logRetryCache) throws IOException, UnresolvedLinkException, - SafeModeException, AccessControlException { + int cellSize, final boolean logRetryCache) throws IOException, + UnresolvedLinkException, SafeModeException, AccessControlException { String src = srcArg; HdfsFileStatus resultingStat = null; FSPermissionChecker pc = null; @@ -7785,7 +7785,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, checkNameNodeSafeMode("Cannot create erasure coding zone on " + src); src = dir.resolvePath(pc, src, pathComponents); - final XAttr ecXAttr = dir.createErasureCodingZone(src, schema); + final XAttr ecXAttr = dir.createErasureCodingZone(src, schema, cellSize); List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1); xAttrs.add(ecXAttr); getEditLog().logSetXAttrs(src, xAttrs, logRetryCache); @@ -7804,9 +7804,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, */ ErasureCodingInfo getErasureCodingInfo(String src) throws AccessControlException, UnresolvedLinkException, IOException { - ECSchema schema = getECSchemaForPath(src); - if (schema != null) { - return new ErasureCodingInfo(src, schema); + ErasureCodingZoneInfo zoneInfo = getErasureCodingZoneInfo(src); + if (zoneInfo != null) { + return new ErasureCodingInfo(src, zoneInfo.getSchema()); } return null; } @@ -7814,21 +7814,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, /** * Get the erasure coding zone information for specified path */ - ErasureCodingZoneInfo getErasureCodingZoneInfo(String src) throws AccessControlException, - UnresolvedLinkException, IOException { + ErasureCodingZoneInfo getErasureCodingZoneInfo(String src) + throws AccessControlException, UnresolvedLinkException, IOException { checkOperation(OperationCategory.READ); - final byte[][] pathComponents = FSDirectory - .getPathComponentsForReservedPath(src); - final FSPermissionChecker pc = getPermissionChecker(); readLock(); try { checkOperation(OperationCategory.READ); - src = dir.resolvePath(pc, src, pathComponents); - final INodesInPath iip = dir.getINodesInPath(src, true); - if (isPermissionEnabled) { - dir.checkPathAccess(pc, iip, FsAction.READ); - } - return dir.getECZoneInfo(iip); + return getErasureCodingZoneInfoForPath(src); } finally { readUnlock(); } @@ -8049,24 +8041,17 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, } @Override - public ECSchema getECSchemaForPath(String src) throws IOException { - checkOperation(OperationCategory.READ); + public ErasureCodingZoneInfo getErasureCodingZoneInfoForPath(String src) + throws IOException { final byte[][] pathComponents = FSDirectory .getPathComponentsForReservedPath(src); final FSPermissionChecker pc = getPermissionChecker(); - readLock(); - try { - checkOperation(OperationCategory.READ); - src = dir.resolvePath(pc, src, pathComponents); - final INodesInPath iip = dir.getINodesInPath(src, true); - if (isPermissionEnabled) { - dir.checkPathAccess(pc, iip, FsAction.READ); - } - // Get schema set for the zone - return dir.getECSchema(iip); - } finally { - readUnlock(); + src = dir.resolvePath(pc, src, pathComponents); + final INodesInPath iip = dir.getINodesInPath(src, true); + if (isPermissionEnabled) { + dir.checkPathAccess(pc, iip, FsAction.READ); } + return dir.getECZoneInfo(iip); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 5a69b2f..747f528 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -1824,7 +1824,7 @@ class NameNodeRpcServer implements NamenodeProtocols { } @Override // ClientProtocol - public void createErasureCodingZone(String src, ECSchema schema) + public void createErasureCodingZone(String src, ECSchema schema, int cellSize) throws IOException { checkNNStartup(); final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); @@ -1833,7 +1833,8 @@ class NameNodeRpcServer implements NamenodeProtocols { } boolean success = false; try { - namesystem.createErasureCodingZone(src, schema, cacheEntry != null); + namesystem.createErasureCodingZone(src, schema, cellSize, + cacheEntry != null); success = true; } finally { RetryCache.setState(cacheEntry, success); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java index e6c7fc0..a32e800 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java @@ -21,6 +21,7 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.ErasureCodingZoneInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection; import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; import org.apache.hadoop.hdfs.util.RwLock; @@ -51,12 +52,13 @@ public interface Namesystem extends RwLock, SafeMode { public boolean isInSnapshot(BlockCollection bc); /** - * Gets the ECSchema for the specified path + * Gets the ECZone info for path * * @param src * - path - * @return ECSchema + * @return {@link ErasureCodingZoneInfo} * @throws IOException */ - public ECSchema getECSchemaForPath(String src) throws IOException; + public ErasureCodingZoneInfo getErasureCodingZoneInfoForPath(String src) + throws IOException; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java index 61e49e9..56a1546 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java @@ -78,25 +78,22 @@ public class BlockECRecoveryCommand extends DatanodeCommand { private StorageType[] targetStorageTypes; private final short[] liveBlockIndices; private final ECSchema ecSchema; + private final int cellSize; public BlockECRecoveryInfo(ExtendedBlock block, DatanodeInfo[] sources, DatanodeStorageInfo[] targetDnStorageInfo, short[] liveBlockIndices, - ECSchema ecSchema) { - this.block = block; - this.sources = sources; - this.targets = DatanodeStorageInfo.toDatanodeInfos(targetDnStorageInfo); - this.targetStorageIDs = DatanodeStorageInfo - .toStorageIDs(targetDnStorageInfo); - this.targetStorageTypes = DatanodeStorageInfo - .toStorageTypes(targetDnStorageInfo); - this.liveBlockIndices = liveBlockIndices; - this.ecSchema = ecSchema; + ECSchema ecSchema, int cellSize) { + this(block, sources, DatanodeStorageInfo + .toDatanodeInfos(targetDnStorageInfo), DatanodeStorageInfo + .toStorageIDs(targetDnStorageInfo), DatanodeStorageInfo + .toStorageTypes(targetDnStorageInfo), liveBlockIndices, ecSchema, + cellSize); } - + public BlockECRecoveryInfo(ExtendedBlock block, DatanodeInfo[] sources, DatanodeInfo[] targets, String[] targetStorageIDs, StorageType[] targetStorageTypes, short[] liveBlockIndices, - ECSchema ecSchema) { + ECSchema ecSchema, int cellSize) { this.block = block; this.sources = sources; this.targets = targets; @@ -104,6 +101,7 @@ public class BlockECRecoveryCommand extends DatanodeCommand { this.targetStorageTypes = targetStorageTypes; this.liveBlockIndices = liveBlockIndices; this.ecSchema = ecSchema; + this.cellSize = cellSize; } public ExtendedBlock getExtendedBlock() { @@ -134,6 +132,10 @@ public class BlockECRecoveryCommand extends DatanodeCommand { return ecSchema; } + public int getCellSize() { + return cellSize; + } + @Override public String toString() { return new StringBuilder().append("BlockECRecoveryInfo(\n ") http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java index 2b6a6a5..2d82208 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.shell.CommandFactory; import org.apache.hadoop.fs.shell.PathData; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.ErasureCodingZoneInfo; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException; import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.util.StringUtils; @@ -88,14 +89,23 @@ public abstract class ECCommand extends Command { + "Options :\n" + " -s <schemaName> : EC schema name to encode files. " + "If not passed default schema will be used\n" + + " -c <cellSize> : cell size to use for striped encoding files." + + " If not passed default cellsize of " + + HdfsConstants.BLOCK_STRIPED_CELL_SIZE + " will be used\n" + " <path> : Path to an empty directory. Under this directory " + "files will be encoded using specified schema"; private String schemaName; + private int cellSize = 0; private ECSchema schema = null; @Override protected void processOptions(LinkedList<String> args) throws IOException { schemaName = StringUtils.popOptionWithArgument("-s", args); + String cellSizeStr = StringUtils.popOptionWithArgument("-c", args); + if (cellSizeStr != null) { + cellSize = (int) StringUtils.TraditionalBinaryPrefix + .string2long(cellSizeStr); + } if (args.isEmpty()) { throw new HadoopIllegalArgumentException("<path> is missing"); } @@ -131,7 +141,7 @@ public abstract class ECCommand extends Command { throw new HadoopIllegalArgumentException(sb.toString()); } } - dfs.createErasureCodingZone(item.path, schema); + dfs.createErasureCodingZone(item.path, schema, cellSize); out.println("EC Zone created successfully at " + item.path); } catch (IOException e) { throw new IOException("Unable to create EC zone for the path " @@ -213,4 +223,4 @@ public abstract class ECCommand extends Command { out.println(sb.toString()); } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java index 2fa3fdf..6f7dcb1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java @@ -306,6 +306,7 @@ public class StripedBlockUtil { * {@link AlignedStripe}. * @param ecSchema The codec schema for the file, which carries the numbers * of data / parity blocks, as well as cell size + * @param cellSize Cell size of stripe * @param blockGroup The striped block group * @param rangeStartInBlockGroup The byte range's start offset in block group * @param rangeEndInBlockGroup The byte range's end offset in block group @@ -315,28 +316,29 @@ public class StripedBlockUtil { * At most 5 stripes will be generated from each logical range, as * demonstrated in the header of {@link AlignedStripe}. */ - public static AlignedStripe[] divideByteRangeIntoStripes ( - ECSchema ecSchema, LocatedStripedBlock blockGroup, + public static AlignedStripe[] divideByteRangeIntoStripes(ECSchema ecSchema, + int cellSize, LocatedStripedBlock blockGroup, long rangeStartInBlockGroup, long rangeEndInBlockGroup, byte[] buf, int offsetInBuf) { // TODO: change ECSchema naming to use cell size instead of chunk size // Step 0: analyze range and calculate basic parameters - int cellSize = ecSchema.getChunkSize(); int dataBlkNum = ecSchema.getNumDataUnits(); // Step 1: map the byte range to StripingCells - StripingCell[] cells = getStripingCellsOfByteRange(ecSchema, blockGroup, - rangeStartInBlockGroup, rangeEndInBlockGroup); + StripingCell[] cells = getStripingCellsOfByteRange(ecSchema, cellSize, + blockGroup, rangeStartInBlockGroup, rangeEndInBlockGroup); // Step 2: get the unmerged ranges on each internal block - VerticalRange[] ranges = getRangesForInternalBlocks(ecSchema, cells); + VerticalRange[] ranges = getRangesForInternalBlocks(ecSchema, cellSize, + cells); // Step 3: merge into at most 5 stripes AlignedStripe[] stripes = mergeRangesForInternalBlocks(ecSchema, ranges); // Step 4: calculate each chunk's position in destination buffer - calcualteChunkPositionsInBuf(ecSchema, stripes, cells, buf, offsetInBuf); + calcualteChunkPositionsInBuf(ecSchema, cellSize, stripes, cells, buf, + offsetInBuf); // Step 5: prepare ALLZERO blocks prepareAllZeroChunks(blockGroup, buf, stripes, cellSize, dataBlkNum); @@ -351,19 +353,18 @@ public class StripedBlockUtil { */ @VisibleForTesting private static StripingCell[] getStripingCellsOfByteRange(ECSchema ecSchema, - LocatedStripedBlock blockGroup, + int cellSize, LocatedStripedBlock blockGroup, long rangeStartInBlockGroup, long rangeEndInBlockGroup) { Preconditions.checkArgument( rangeStartInBlockGroup <= rangeEndInBlockGroup && rangeEndInBlockGroup < blockGroup.getBlockSize()); - int cellSize = ecSchema.getChunkSize(); int len = (int) (rangeEndInBlockGroup - rangeStartInBlockGroup + 1); int firstCellIdxInBG = (int) (rangeStartInBlockGroup / cellSize); int lastCellIdxInBG = (int) (rangeEndInBlockGroup / cellSize); int numCells = lastCellIdxInBG - firstCellIdxInBG + 1; StripingCell[] cells = new StripingCell[numCells]; - cells[0] = new StripingCell(ecSchema, firstCellIdxInBG); - cells[numCells - 1] = new StripingCell(ecSchema, lastCellIdxInBG); + cells[0] = new StripingCell(ecSchema, cellSize, firstCellIdxInBG); + cells[numCells - 1] = new StripingCell(ecSchema, cellSize, lastCellIdxInBG); cells[0].offset = (int) (rangeStartInBlockGroup % cellSize); cells[0].size = @@ -373,7 +374,7 @@ public class StripedBlockUtil { } for (int i = 1; i < numCells - 1; i++) { - cells[i] = new StripingCell(ecSchema, i + firstCellIdxInBG); + cells[i] = new StripingCell(ecSchema, cellSize, i + firstCellIdxInBG); } return cells; @@ -383,18 +384,16 @@ public class StripedBlockUtil { * Given a logical start offset in a block group, calculate the physical * start offset into each stored internal block. */ - public static long[] getStartOffsetsForInternalBlocks( - ECSchema ecSchema, LocatedStripedBlock blockGroup, - long rangeStartInBlockGroup) { + public static long[] getStartOffsetsForInternalBlocks(ECSchema ecSchema, + int cellSize, LocatedStripedBlock blockGroup, long rangeStartInBlockGroup) { Preconditions.checkArgument( rangeStartInBlockGroup < blockGroup.getBlockSize()); int dataBlkNum = ecSchema.getNumDataUnits(); int parityBlkNum = ecSchema.getNumParityUnits(); - int cellSize = ecSchema.getChunkSize(); long[] startOffsets = new long[dataBlkNum + parityBlkNum]; Arrays.fill(startOffsets, -1L); int firstCellIdxInBG = (int) (rangeStartInBlockGroup / cellSize); - StripingCell firstCell = new StripingCell(ecSchema, firstCellIdxInBG); + StripingCell firstCell = new StripingCell(ecSchema, cellSize, firstCellIdxInBG); firstCell.offset = (int) (rangeStartInBlockGroup % cellSize); startOffsets[firstCell.idxInStripe] = firstCell.idxInInternalBlk * cellSize + firstCell.offset; @@ -404,7 +403,7 @@ public class StripedBlockUtil { if (idx * cellSize >= blockGroup.getBlockSize()) { break; } - StripingCell cell = new StripingCell(ecSchema, idx); + StripingCell cell = new StripingCell(ecSchema, cellSize, idx); startOffsets[cell.idxInStripe] = cell.idxInInternalBlk * cellSize; if (startOffsets[cell.idxInStripe] < earliestStart) { earliestStart = startOffsets[cell.idxInStripe]; @@ -422,8 +421,7 @@ public class StripedBlockUtil { */ @VisibleForTesting private static VerticalRange[] getRangesForInternalBlocks(ECSchema ecSchema, - StripingCell[] cells) { - int cellSize = ecSchema.getChunkSize(); + int cellSize, StripingCell[] cells) { int dataBlkNum = ecSchema.getNumDataUnits(); int parityBlkNum = ecSchema.getNumParityUnits(); @@ -486,7 +484,7 @@ public class StripedBlockUtil { } private static void calcualteChunkPositionsInBuf(ECSchema ecSchema, - AlignedStripe[] stripes, StripingCell[] cells, byte[] buf, + int cellSize, AlignedStripe[] stripes, StripingCell[] cells, byte[] buf, int offsetInBuf) { /** * | <--------------- AlignedStripe --------------->| @@ -505,7 +503,6 @@ public class StripedBlockUtil { * * Cell indexing convention defined in {@link StripingCell} */ - int cellSize = ecSchema.getChunkSize(); int done = 0; for (StripingCell cell : cells) { long cellStart = cell.idxInInternalBlk * cellSize + cell.offset; @@ -587,17 +584,17 @@ public class StripedBlockUtil { int offset; int size; - StripingCell(ECSchema ecSchema, int idxInBlkGroup) { + StripingCell(ECSchema ecSchema, int cellSize, int idxInBlkGroup) { this.schema = ecSchema; this.idxInBlkGroup = idxInBlkGroup; this.idxInInternalBlk = idxInBlkGroup / ecSchema.getNumDataUnits(); this.idxInStripe = idxInBlkGroup - this.idxInInternalBlk * ecSchema.getNumDataUnits(); this.offset = 0; - this.size = ecSchema.getChunkSize(); + this.size = cellSize; } - StripingCell(ECSchema ecSchema, int idxInInternalBlk, + StripingCell(ECSchema ecSchema, int cellSize, int idxInInternalBlk, int idxInStripe) { this.schema = ecSchema; this.idxInInternalBlk = idxInInternalBlk; @@ -605,7 +602,7 @@ public class StripedBlockUtil { this.idxInBlkGroup = idxInInternalBlk * ecSchema.getNumDataUnits() + idxInStripe; this.offset = 0; - this.size = ecSchema.getChunkSize(); + this.size = cellSize; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/erasurecoding.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/erasurecoding.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/erasurecoding.proto index 2302d1d..058ed96 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/erasurecoding.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/erasurecoding.proto @@ -37,11 +37,13 @@ message ErasureCodingInfoProto { message ErasureCodingZoneInfoProto { required string dir = 1; required ECSchemaProto schema = 2; + required uint32 cellSize = 3; } message CreateErasureCodingZoneRequestProto { required string src = 1; optional ECSchemaProto schema = 2; + optional uint32 cellSize = 3; } message CreateErasureCodingZoneResponseProto { @@ -81,4 +83,5 @@ message BlockECRecoveryInfoProto { required StorageTypesProto targetStorageTypes = 5; repeated uint32 liveBlockIndices = 6; required ECSchemaProto ecSchema = 7; + required uint32 cellSize = 8; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index 64030be..f64cf8f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -359,7 +359,8 @@ message HdfsFileStatusProto { // Optional field for erasure coding optional ECSchemaProto ecSchema = 17; -} + optional uint32 stripeCellSize = 18; +} /** * Checksum algorithms/types used in HDFS http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 9f106cf..558c45d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -1867,7 +1867,7 @@ public class DFSTestUtil { assert dir != null; dfs.mkdirs(dir); try { - dfs.getClient().createErasureCodingZone(dir.toString(), null); + dfs.getClient().createErasureCodingZone(dir.toString(), null, 0); } catch (IOException e) { if (!e.getMessage().contains("non-empty directory")) { throw e; http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index ec88a54..4918dee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@ -255,12 +255,12 @@ public class TestDFSClientRetries { Mockito.doReturn( new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( (short) 777), "owner", "group", new byte[0], new byte[0], - 1010, 0, null, (byte) 0, null)).when(mockNN).getFileInfo(anyString()); + 1010, 0, null, (byte) 0, null, 0)).when(mockNN).getFileInfo(anyString()); Mockito.doReturn( new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( (short) 777), "owner", "group", new byte[0], new byte[0], - 1010, 0, null, (byte) 0, null)) + 1010, 0, null, (byte) 0, null, 0)) .when(mockNN) .create(anyString(), (FsPermission) anyObject(), anyString(), (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index 9032d09..a71441f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -73,7 +73,7 @@ public class TestDFSStripedInputStream { cluster.waitActive(); fs = cluster.getFileSystem(); fs.mkdirs(dirPath); - fs.getClient().createErasureCodingZone(dirPath.toString(), null); + fs.getClient().createErasureCodingZone(dirPath.toString(), null, CELLSIZE); } @After @@ -94,7 +94,7 @@ public class TestDFSStripedInputStream { LocatedBlocks lbs = fs.getClient().namenode.getBlockLocations( filePath.toString(), 0, BLOCK_GROUP_SIZE * numBlocks); final DFSStripedInputStream in = new DFSStripedInputStream(fs.getClient(), - filePath.toString(), false, schema); + filePath.toString(), false, schema, CELLSIZE); List<LocatedBlock> lbList = lbs.getLocatedBlocks(); for (LocatedBlock aLbList : lbList) { @@ -146,7 +146,7 @@ public class TestDFSStripedInputStream { } } DFSStripedInputStream in = new DFSStripedInputStream(fs.getClient(), - filePath.toString(), false, schema); + filePath.toString(), false, schema, CELLSIZE); int[] startOffsets = {0, 1, CELLSIZE - 102, CELLSIZE, CELLSIZE + 102, CELLSIZE*DATA_BLK_NUM, CELLSIZE*DATA_BLK_NUM + 102, @@ -188,7 +188,7 @@ public class TestDFSStripedInputStream { } DFSStripedInputStream in = new DFSStripedInputStream(fs.getClient(), filePath.toString(), false, - ErasureCodingSchemaManager.getSystemDefaultSchema()); + ErasureCodingSchemaManager.getSystemDefaultSchema(), CELLSIZE); int readSize = BLOCK_GROUP_SIZE; byte[] readBuffer = new byte[readSize]; byte[] expected = new byte[readSize]; @@ -284,7 +284,7 @@ public class TestDFSStripedInputStream { DFSStripedInputStream in = new DFSStripedInputStream(fs.getClient(), filePath.toString(), - false, schema); + false, schema, CELLSIZE); byte[] expected = new byte[fileSize]; http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index ec98e68..e795ea7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -65,7 +65,7 @@ public class TestDFSStripedOutputStream { Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); - cluster.getFileSystem().getClient().createErasureCodingZone("/", null); + cluster.getFileSystem().getClient().createErasureCodingZone("/", null, 0); fs = cluster.getFileSystem(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index c2e588a..4ad3b2e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -74,7 +74,7 @@ public class TestDFSStripedOutputStreamWithFailure { cluster.waitActive(); dfs = cluster.getFileSystem(); dfs.mkdirs(dir); - dfs.createErasureCodingZone(dir, null); + dfs.createErasureCodingZone(dir, null, 0); } @After http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 8542131..c4aa366 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -737,7 +737,7 @@ public class TestEncryptionZones { version, new byte[suite.getAlgorithmBlockSize()], new byte[suite.getAlgorithmBlockSize()], "fakeKey", "fakeVersion"), - (byte) 0, null)) + (byte) 0, null, 0)) .when(mcp) .create(anyString(), (FsPermission) anyObject(), anyString(), (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java index 59818c0..a7c3cd4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java @@ -63,7 +63,7 @@ public class TestErasureCodingZones { fs.mkdir(testDir, FsPermission.getDirDefault()); /* Normal creation of an erasure coding zone */ - fs.getClient().createErasureCodingZone(testDir.toString(), null); + fs.getClient().createErasureCodingZone(testDir.toString(), null, 0); /* Verify files under the zone are striped */ final Path ECFilePath = new Path(testDir, "foo"); @@ -76,7 +76,7 @@ public class TestErasureCodingZones { fs.mkdir(notEmpty, FsPermission.getDirDefault()); fs.create(new Path(notEmpty, "foo")); try { - fs.getClient().createErasureCodingZone(notEmpty.toString(), null); + fs.getClient().createErasureCodingZone(notEmpty.toString(), null, 0); fail("Erasure coding zone on non-empty dir"); } catch (IOException e) { assertExceptionContains("erasure coding zone for a non-empty directory", e); @@ -86,10 +86,10 @@ public class TestErasureCodingZones { final Path zone1 = new Path("/zone1"); final Path zone2 = new Path(zone1, "zone2"); fs.mkdir(zone1, FsPermission.getDirDefault()); - fs.getClient().createErasureCodingZone(zone1.toString(), null); + fs.getClient().createErasureCodingZone(zone1.toString(), null, 0); fs.mkdir(zone2, FsPermission.getDirDefault()); try { - fs.getClient().createErasureCodingZone(zone2.toString(), null); + fs.getClient().createErasureCodingZone(zone2.toString(), null, 0); fail("Nested erasure coding zones"); } catch (IOException e) { assertExceptionContains("already in an erasure coding zone", e); @@ -99,7 +99,7 @@ public class TestErasureCodingZones { final Path fPath = new Path("/file"); fs.create(fPath); try { - fs.getClient().createErasureCodingZone(fPath.toString(), null); + fs.getClient().createErasureCodingZone(fPath.toString(), null, 0); fail("Erasure coding zone on file"); } catch (IOException e) { assertExceptionContains("erasure coding zone for a file", e); @@ -112,8 +112,8 @@ public class TestErasureCodingZones { final Path dstECDir = new Path("/dstEC"); fs.mkdir(srcECDir, FsPermission.getDirDefault()); fs.mkdir(dstECDir, FsPermission.getDirDefault()); - fs.getClient().createErasureCodingZone(srcECDir.toString(), null); - fs.getClient().createErasureCodingZone(dstECDir.toString(), null); + fs.getClient().createErasureCodingZone(srcECDir.toString(), null, 0); + fs.getClient().createErasureCodingZone(dstECDir.toString(), null, 0); final Path srcFile = new Path(srcECDir, "foo"); fs.create(srcFile); @@ -157,7 +157,7 @@ public class TestErasureCodingZones { // dir ECInfo before creating ec zone assertNull(fs.getClient().getErasureCodingInfo(src)); // dir ECInfo after creating ec zone - fs.getClient().createErasureCodingZone(src, null); //Default one will be used. + fs.getClient().createErasureCodingZone(src, null, 0); //Default one will be used. ECSchema sysDefaultSchema = ErasureCodingSchemaManager.getSystemDefaultSchema(); verifyErasureCodingInfo(src, sysDefaultSchema); fs.create(new Path(ecDir, "/child1")).close(); @@ -178,7 +178,7 @@ public class TestErasureCodingZones { // dir ECInfo before creating ec zone assertNull(fs.getClient().getErasureCodingInfo(src)); // dir ECInfo after creating ec zone - fs.getClient().createErasureCodingZone(src, usingSchema); + fs.getClient().createErasureCodingZone(src, usingSchema, 0); verifyErasureCodingInfo(src, usingSchema); fs.create(new Path(ecDir, "/child1")).close(); // verify for the files in ec zone http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECschema.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECschema.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECschema.java index f8c0667..3c400b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECschema.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECschema.java @@ -50,7 +50,7 @@ public class TestFileStatusWithECschema { final ECSchema schema1 = ErasureCodingSchemaManager.getSystemDefaultSchema(); // create EC zone on dir - fs.createErasureCodingZone(dir, schema1); + fs.createErasureCodingZone(dir, schema1, 0); final ECSchema schame2 = client.getFileInfo(dir.toUri().getPath()).getECSchema(); assertNotNull(schame2); assertTrue(schema1.equals(schame2)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java index b77ff3a..d0cd335 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java @@ -354,12 +354,12 @@ public class TestLease { Mockito.doReturn( new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( (short) 777), "owner", "group", new byte[0], new byte[0], - 1010, 0, null, (byte) 0, null)).when(mcp).getFileInfo(anyString()); + 1010, 0, null, (byte) 0, null, 0)).when(mcp).getFileInfo(anyString()); Mockito .doReturn( new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission( (short) 777), "owner", "group", new byte[0], new byte[0], - 1010, 0, null, (byte) 0, null)) + 1010, 0, null, (byte) 0, null, 0)) .when(mcp) .create(anyString(), (FsPermission) anyObject(), anyString(), (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java index dfdcee2..e5e324c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java @@ -77,7 +77,7 @@ public class TestRecoverStripedFile { cluster.waitActive(); fs = cluster.getFileSystem(); - fs.getClient().createErasureCodingZone("/", null); + fs.getClient().createErasureCodingZone("/", null, 0); List<DataNode> datanodes = cluster.getDataNodes(); for (int i = 0; i < dnNum; i++) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java index f78fb7a..1976dca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java @@ -59,7 +59,7 @@ public class TestWriteReadStripedFile { conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); - cluster.getFileSystem().getClient().createErasureCodingZone("/", null); + cluster.getFileSystem().getClient().createErasureCodingZone("/", null, cellSize); fs = cluster.getFileSystem(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index 1be00b6..a0b2038 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -663,7 +663,8 @@ public class TestPBHelper { short[] liveBlkIndices0 = new short[2]; BlockECRecoveryInfo blkECRecoveryInfo0 = new BlockECRecoveryInfo( new ExtendedBlock("bp1", 1234), dnInfos0, targetDnInfos0, - liveBlkIndices0, ErasureCodingSchemaManager.getSystemDefaultSchema()); + liveBlkIndices0, ErasureCodingSchemaManager.getSystemDefaultSchema(), + 64 * 1024); DatanodeInfo[] dnInfos1 = new DatanodeInfo[] { DFSTestUtil.getLocalDatanodeInfo(), DFSTestUtil.getLocalDatanodeInfo() }; DatanodeStorageInfo targetDnInfos_2 = BlockManagerTestUtil @@ -677,7 +678,8 @@ public class TestPBHelper { short[] liveBlkIndices1 = new short[2]; BlockECRecoveryInfo blkECRecoveryInfo1 = new BlockECRecoveryInfo( new ExtendedBlock("bp2", 3256), dnInfos1, targetDnInfos1, - liveBlkIndices1, ErasureCodingSchemaManager.getSystemDefaultSchema()); + liveBlkIndices1, ErasureCodingSchemaManager.getSystemDefaultSchema(), + 64 * 1024); List<BlockECRecoveryInfo> blkRecoveryInfosList = new ArrayList<BlockECRecoveryInfo>(); blkRecoveryInfosList.add(blkECRecoveryInfo0); blkRecoveryInfosList.add(blkECRecoveryInfo1); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java index ba76360..a35cbf4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java @@ -74,7 +74,7 @@ public class TestAddStripedBlocks { .numDataNodes(GROUP_SIZE).build(); cluster.waitActive(); dfs = cluster.getFileSystem(); - dfs.getClient().createErasureCodingZone("/", null); + dfs.getClient().createErasureCodingZone("/", null, 0); } @After http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java index 2909423..abb9bf5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java @@ -451,7 +451,7 @@ public class TestFSEditLogLoader { //set the storage policy of the directory fs.mkdir(new Path(testDir), new FsPermission("755")); - fs.getClient().getNamenode().createErasureCodingZone(testDir, null); + fs.getClient().getNamenode().createErasureCodingZone(testDir, null, 0); // Create a file with striped block Path p = new Path(testFilePath); @@ -523,7 +523,7 @@ public class TestFSEditLogLoader { //set the storage policy of the directory fs.mkdir(new Path(testDir), new FsPermission("755")); - fs.getClient().getNamenode().createErasureCodingZone(testDir, null); + fs.getClient().getNamenode().createErasureCodingZone(testDir, null, 0); //create a file with striped blocks Path p = new Path(testFilePath); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index 5e015bf..8fd0753 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -140,7 +140,7 @@ public class TestFSImage { private void testSaveAndLoadStripedINodeFile(FSNamesystem fsn, Configuration conf, boolean isUC) throws IOException{ // contruct a INode with StripedBlock for saving and loading - fsn.createErasureCodingZone("/", null, false); + fsn.createErasureCodingZone("/", null, 0, false); long id = 123456789; byte[] name = "testSaveAndLoadInodeFile_testfile".getBytes(); PermissionStatus permissionStatus = new PermissionStatus("testuser_a", @@ -402,7 +402,7 @@ public class TestFSImage { .build(); cluster.waitActive(); DistributedFileSystem fs = cluster.getFileSystem(); - fs.getClient().getNamenode().createErasureCodingZone("/", null); + fs.getClient().getNamenode().createErasureCodingZone("/", null, 0); Path file = new Path("/striped"); FSDataOutputStream out = fs.create(file); byte[] bytes = DFSTestUtil.generateSequentialBytes(0, BLOCK_SIZE); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c36df15/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 7870827..b36beb2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -1198,7 +1198,7 @@ public class TestFsck { HdfsFileStatus file = new HdfsFileStatus(length, isDir, blockReplication, blockSize, modTime, accessTime, perms, owner, group, symlink, - path, fileId, numChildren, null, storagePolicy, null); + path, fileId, numChildren, null, storagePolicy, null, 0); Result res = new Result(conf); try { @@ -1629,4 +1629,4 @@ public class TestFsck { } } } -} \ No newline at end of file +}