http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a7ebab/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java index 16f534f..e348231 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java @@ -21,6 +21,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.namenode.AclFeature; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; @@ -154,18 +155,19 @@ public class FileWithSnapshotFeature implements INode.Feature { AclStorage.removeAclFeature(aclFeature); } } - - collectBlocksAndClear(file, collectedBlocks, removedINodes); - + + getDiffs().combineAndCollectSnapshotBlocks( + file, removed, collectedBlocks, removedINodes); + long dsDelta = oldDiskspace - file.diskspaceConsumed(); return Quota.Counts.newInstance(0, dsDelta); } - + /** * If some blocks at the end of the block list no longer belongs to * any inode, collect them and update the block list. */ - private void collectBlocksAndClear(final INodeFile file, + public void collectBlocksAndClear(final INodeFile file, final BlocksMapUpdateInfo info, final List<INode> removedINodes) { // check if everything is deleted. if (isCurrentFileDeleted() && getDiffs().asList().isEmpty()) { @@ -174,13 +176,19 @@ public class FileWithSnapshotFeature implements INode.Feature { } // find max file size. final long max; + FileDiff diff = getDiffs().getLast(); if (isCurrentFileDeleted()) { - final FileDiff last = getDiffs().getLast(); - max = last == null? 0: last.getFileSize(); + max = diff == null? 0: diff.getFileSize(); } else { max = file.computeFileSize(); } - file.collectBlocksBeyondMax(max, info); + // Collect blocks that should be deleted + FileDiff last = diffs.getLast(); + BlockInfo[] snapshotBlocks = last == null ? null : last.getBlocks(); + if(snapshotBlocks == null) + file.collectBlocksBeyondMax(max, info); + else + file.collectBlocksBeyondSnapshot(snapshotBlocks, info); } }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a7ebab/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java index c512038..ced3296 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; @@ -53,8 +54,8 @@ public class BlockRecoveryCommand extends DatanodeCommand { @InterfaceAudience.Private @InterfaceStability.Evolving public static class RecoveringBlock extends LocatedBlock { - private boolean truncate; private final long newGenerationStamp; + private final Block recoveryBlock; /** * Create RecoveringBlock. @@ -62,15 +63,17 @@ public class BlockRecoveryCommand extends DatanodeCommand { public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs, long newGS) { super(b, locs, -1, false); // startOffset is unknown this.newGenerationStamp = newGS; + this.recoveryBlock = null; } /** - * RecoveryingBlock with truncate option. + * Create RecoveringBlock with copy-on-truncate option. */ - public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs, long newGS, - boolean truncate) { - this(b, locs, newGS); - this.truncate = truncate; + public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs, + Block recoveryBlock) { + super(b, locs, -1, false); // startOffset is unknown + this.newGenerationStamp = recoveryBlock.getGenerationStamp(); + this.recoveryBlock = recoveryBlock; } /** @@ -82,10 +85,10 @@ public class BlockRecoveryCommand extends DatanodeCommand { } /** - * Return whether to truncate the block to the ExtendedBlock's length. + * Return the new block. */ - public boolean getTruncateFlag() { - return truncate; + public Block getNewBlock() { + return recoveryBlock; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a7ebab/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java index 62915b4..72cb0c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java @@ -67,5 +67,6 @@ public interface InterDatanodeProtocol { * Update replica with the new generation stamp and length. */ String updateReplicaUnderRecovery(ExtendedBlock oldBlock, long recoveryId, - long newLength) throws IOException; + long newBlockId, long newLength) + throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a7ebab/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto index 47f79be..1a21777 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto @@ -59,6 +59,8 @@ message UpdateReplicaUnderRecoveryRequestProto { required ExtendedBlockProto block = 1; // Block identifier required uint64 recoveryId = 2; // New genstamp of the replica required uint64 newLength = 3; // New length of the replica + // New blockId for copy (truncate), default is 0. + optional uint64 newBlockId = 4 [default = 0]; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a7ebab/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto index 588f6c8..643a034 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto @@ -270,6 +270,7 @@ message SnapshotDiffSection { optional uint64 fileSize = 2; optional bytes name = 3; optional INodeSection.INodeFile snapshotCopy = 4; + repeated BlockProto blocks = 5; } message DiffEntry { http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a7ebab/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index d989c0a..97906b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -554,9 +554,9 @@ enum ReplicaStateProto { * Block that needs to be recovered with at a given location */ message RecoveringBlockProto { - required uint64 newGenStamp = 1; // New genstamp post recovery - required LocatedBlockProto block = 2; // Block to be recovered - optional bool truncateFlag = 3; // Block needs to be truncated + required uint64 newGenStamp = 1; // New genstamp post recovery + required LocatedBlockProto block = 2; // Block to be recovered + optional BlockProto truncateBlock = 3; // New block for recovery (truncate) } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a7ebab/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index ba43f1d..2699801 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -1113,6 +1113,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { @Override // FsDatasetSpi public String updateReplicaUnderRecovery(ExtendedBlock oldBlock, long recoveryId, + long newBlockId, long newlength) { // Caller does not care about the exact Storage UUID returned. return datanodeUuid; http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a7ebab/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java index 22d05a7..855cee4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java @@ -56,7 +56,6 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -216,10 +215,10 @@ public class TestBlockRecovery { syncList.add(record1); syncList.add(record2); - when(dn1.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(), - anyLong())).thenReturn("storage1"); - when(dn2.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(), - anyLong())).thenReturn("storage2"); + when(dn1.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(), + anyLong(), anyLong())).thenReturn("storage1"); + when(dn2.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(), + anyLong(), anyLong())).thenReturn("storage2"); dn.syncBlock(rBlock, syncList); } @@ -242,8 +241,10 @@ public class TestBlockRecovery { InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class); testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1); - verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1); - verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1); + verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, + REPLICA_LEN1); + verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, + REPLICA_LEN1); // two finalized replicas have different length replica1 = new ReplicaRecoveryInfo(BLOCK_ID, @@ -281,8 +282,10 @@ public class TestBlockRecovery { InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class); testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1); - verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1); - verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1); + verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, + REPLICA_LEN1); + verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, + REPLICA_LEN1); // rbw replica has a different length from the finalized one replica1 = new ReplicaRecoveryInfo(BLOCK_ID, @@ -294,9 +297,9 @@ public class TestBlockRecovery { dn2 = mock(InterDatanodeProtocol.class); testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1); - verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1); + verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1); verify(dn2, never()).updateReplicaUnderRecovery( - block, RECOVERY_ID, REPLICA_LEN1); + block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1); } /** @@ -320,9 +323,10 @@ public class TestBlockRecovery { InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class); testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1); - verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1); + verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, + REPLICA_LEN1); verify(dn2, never()).updateReplicaUnderRecovery( - block, RECOVERY_ID, REPLICA_LEN1); + block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1); // rbw replica has a different length from the finalized one replica1 = new ReplicaRecoveryInfo(BLOCK_ID, @@ -334,9 +338,10 @@ public class TestBlockRecovery { dn2 = mock(InterDatanodeProtocol.class); testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1); - verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1); + verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, + REPLICA_LEN1); verify(dn2, never()).updateReplicaUnderRecovery( - block, RECOVERY_ID, REPLICA_LEN1); + block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1); } /** @@ -359,8 +364,8 @@ public class TestBlockRecovery { long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2); testSyncReplicas(replica1, replica2, dn1, dn2, minLen); - verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, minLen); - verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, minLen); + verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen); + verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen); } /** @@ -382,9 +387,10 @@ public class TestBlockRecovery { InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class); testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1); - verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1); + verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, + REPLICA_LEN1); verify(dn2, never()).updateReplicaUnderRecovery( - block, RECOVERY_ID, REPLICA_LEN1); + block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1); } /** @@ -408,8 +414,8 @@ public class TestBlockRecovery { long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2); testSyncReplicas(replica1, replica2, dn1, dn2, minLen); - verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, minLen); - verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, minLen); + verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen); + verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen); } private Collection<RecoveringBlock> initRecoveringBlocks() throws IOException { @@ -510,7 +516,7 @@ public class TestBlockRecovery { } DataNode spyDN = spy(dn); doThrow(new IOException()).when(spyDN).updateReplicaUnderRecovery( - block, RECOVERY_ID, block.getNumBytes()); + block, RECOVERY_ID, BLOCK_ID, block.getNumBytes()); try { spyDN.syncBlock(rBlock, initBlockRecords(spyDN)); fail("Sync should fail"); @@ -631,7 +637,8 @@ public class TestBlockRecovery { recoveryInitResult.get()); dataNode.updateReplicaUnderRecovery(block.getBlock(), block.getBlock() - .getGenerationStamp() + 1, block.getBlockSize()); + .getGenerationStamp() + 1, block.getBlock().getBlockId(), + block.getBlockSize()); } finally { if (null != cluster) { cluster.shutdown(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a7ebab/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java index 65a5176..3609684 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java @@ -198,7 +198,8 @@ public class TestInterDatanodeProtocol { //verify updateBlock ExtendedBlock newblock = new ExtendedBlock(b.getBlockPoolId(), b.getBlockId(), b.getNumBytes()/2, b.getGenerationStamp()+1); - idp.updateReplicaUnderRecovery(b, recoveryId, newblock.getNumBytes()); + idp.updateReplicaUnderRecovery(b, recoveryId, b.getBlockId(), + newblock.getNumBytes()); checkMetaInfo(newblock, datanode); // Verify correct null response trying to init recovery for a missing block @@ -368,7 +369,8 @@ public class TestInterDatanodeProtocol { .getBlockId(), rri.getNumBytes() - 1, rri.getGenerationStamp()); try { //update should fail - fsdataset.updateReplicaUnderRecovery(tmp, recoveryid, newlength); + fsdataset.updateReplicaUnderRecovery(tmp, recoveryid, + tmp.getBlockId(), newlength); Assert.fail(); } catch(IOException ioe) { System.out.println("GOOD: getting " + ioe); @@ -377,7 +379,8 @@ public class TestInterDatanodeProtocol { //update final String storageID = fsdataset.updateReplicaUnderRecovery( - new ExtendedBlock(b.getBlockPoolId(), rri), recoveryid, newlength); + new ExtendedBlock(b.getBlockPoolId(), rri), recoveryid, + rri.getBlockId(), newlength); assertTrue(storageID != null); } finally { http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a7ebab/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java index d0502b3..eae65cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java @@ -71,6 +71,7 @@ public class TestCommitBlockSynchronization { doReturn(true).when(file).isUnderConstruction(); doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class)); + doReturn(blockInfo).when(file).getLastBlock(); doReturn("").when(namesystemSpy).closeFileCommitBlocks( any(INodeFile.class), any(BlockInfo.class)); doReturn(mock(FSEditLog.class)).when(namesystemSpy).getEditLog(); @@ -105,6 +106,7 @@ public class TestCommitBlockSynchronization { completedBlockInfo.setGenerationStamp(genStamp); doReturn(completedBlockInfo).when(namesystemSpy) .getStoredBlock(any(Block.class)); + doReturn(completedBlockInfo).when(file).getLastBlock(); // Repeat the call to make sure it does not throw namesystemSpy.commitBlockSynchronization( @@ -176,6 +178,7 @@ public class TestCommitBlockSynchronization { completedBlockInfo.setGenerationStamp(genStamp); doReturn(completedBlockInfo).when(namesystemSpy) .getStoredBlock(any(Block.class)); + doReturn(completedBlockInfo).when(file).getLastBlock(); namesystemSpy.commitBlockSynchronization( lastBlock, genStamp, length, true, false, newTargets, null); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a7ebab/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java index ba9d04e..1f854d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java @@ -18,14 +18,22 @@ package org.apache.hadoop.hdfs.server.namenode; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; -import java.net.InetAddress; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataOutputStream; @@ -39,14 +47,14 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; -import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.Time; import org.apache.log4j.Level; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -57,6 +65,7 @@ public class TestFileTruncate { GenericTestUtils.setLogLevel(NameNode.stateChangeLog, Level.ALL); GenericTestUtils.setLogLevel(FSEditLogLoader.LOG, Level.ALL); } + static final Log LOG = LogFactory.getLog(TestFileTruncate.class); static final int BLOCK_SIZE = 4; static final short REPLICATION = 3; static final int DATANODE_NUM = 3; @@ -129,6 +138,287 @@ public class TestFileTruncate { fs.delete(parent, true); } + @Test + public void testSnapshotWithAppendTruncate() throws IOException { + testSnapshotWithAppendTruncate(0, 1, 2); + testSnapshotWithAppendTruncate(0, 2, 1); + testSnapshotWithAppendTruncate(1, 0, 2); + testSnapshotWithAppendTruncate(1, 2, 0); + testSnapshotWithAppendTruncate(2, 0, 1); + testSnapshotWithAppendTruncate(2, 1, 0); + } + + /** + * Create three snapshots with appended and truncated file. + * Delete snapshots in the specified order and verify that + * remaining snapshots are still readable. + */ + void testSnapshotWithAppendTruncate(int ... deleteOrder) throws IOException { + FSDirectory fsDir = cluster.getNamesystem().getFSDirectory(); + Path parent = new Path("/test"); + fs.mkdirs(parent); + fs.setQuota(parent, 100, 1000); + fs.allowSnapshot(parent); + String truncateFile = "testSnapshotWithAppendTruncate"; + final Path src = new Path(parent, truncateFile); + int[] length = new int[4]; + length[0] = 2 * BLOCK_SIZE + BLOCK_SIZE / 2; + DFSTestUtil.createFile(fs, src, 64, length[0], BLOCK_SIZE, REPLICATION, 0L); + Block firstBlk = getLocatedBlocks(src).get(0).getBlock().getLocalBlock(); + Path[] snapshotFiles = new Path[4]; + + // Diskspace consumed should be 10 bytes * 3. [blk 1,2,3] + ContentSummary contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(30L)); + + // Add file to snapshot and append + String[] ss = new String[] {"ss0", "ss1", "ss2", "ss3"}; + Path snapshotDir = fs.createSnapshot(parent, ss[0]); + snapshotFiles[0] = new Path(snapshotDir, truncateFile); + length[1] = length[2] = length[0] + BLOCK_SIZE + 1; + DFSTestUtil.appendFile(fs, src, BLOCK_SIZE + 1); + Block lastBlk = getLocatedBlocks(src).getLastLocatedBlock() + .getBlock().getLocalBlock(); + + // Diskspace consumed should be 15 bytes * 3. [blk 1,2,3,4] + contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(45L)); + + // Create another snapshot without changes + snapshotDir = fs.createSnapshot(parent, ss[1]); + snapshotFiles[1] = new Path(snapshotDir, truncateFile); + + // Create another snapshot and append + snapshotDir = fs.createSnapshot(parent, ss[2]); + snapshotFiles[2] = new Path(snapshotDir, truncateFile); + DFSTestUtil.appendFile(fs, src, BLOCK_SIZE -1 + BLOCK_SIZE / 2); + Block appendedBlk = getLocatedBlocks(src).getLastLocatedBlock() + .getBlock().getLocalBlock(); + + // Diskspace consumed should be 20 bytes * 3. [blk 1,2,3,4,5] + contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(60L)); + + // Truncate to block boundary + int newLength = length[0] + BLOCK_SIZE / 2; + boolean isReady = fs.truncate(src, newLength); + assertTrue("Recovery is not expected.", isReady); + assertFileLength(snapshotFiles[2], length[2]); + assertFileLength(snapshotFiles[1], length[1]); + assertFileLength(snapshotFiles[0], length[0]); + assertBlockNotPresent(appendedBlk); + + // Diskspace consumed should be 16 bytes * 3. [blk 1,2,3 SS:4] + contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(48L)); + + // Truncate full block again + newLength = length[0] - BLOCK_SIZE / 2; + isReady = fs.truncate(src, newLength); + assertTrue("Recovery is not expected.", isReady); + assertFileLength(snapshotFiles[2], length[2]); + assertFileLength(snapshotFiles[1], length[1]); + assertFileLength(snapshotFiles[0], length[0]); + + // Diskspace consumed should be 16 bytes * 3. [blk 1,2 SS:3,4] + contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(48L)); + + // Truncate half of the last block + newLength -= BLOCK_SIZE / 2; + isReady = fs.truncate(src, newLength); + assertFalse("Recovery is expected.", isReady); + checkBlockRecovery(src); + assertFileLength(snapshotFiles[2], length[2]); + assertFileLength(snapshotFiles[1], length[1]); + assertFileLength(snapshotFiles[0], length[0]); + Block replacedBlk = getLocatedBlocks(src).getLastLocatedBlock() + .getBlock().getLocalBlock(); + + // Diskspace consumed should be 16 bytes * 3. [blk 1,6 SS:2,3,4] + contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(54L)); + + snapshotDir = fs.createSnapshot(parent, ss[3]); + snapshotFiles[3] = new Path(snapshotDir, truncateFile); + length[3] = newLength; + + // Delete file. Should still be able to read snapshots + int numINodes = fsDir.getInodeMapSize(); + isReady = fs.delete(src, false); + assertTrue("Delete failed.", isReady); + assertFileLength(snapshotFiles[3], length[3]); + assertFileLength(snapshotFiles[2], length[2]); + assertFileLength(snapshotFiles[1], length[1]); + assertFileLength(snapshotFiles[0], length[0]); + assertEquals("Number of INodes should not change", + numINodes, fsDir.getInodeMapSize()); + + fs.deleteSnapshot(parent, ss[3]); + + assertBlockExists(firstBlk); + assertBlockExists(lastBlk); + assertBlockNotPresent(replacedBlk); + + // Diskspace consumed should be 16 bytes * 3. [SS:1,2,3,4] + contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(48L)); + + // delete snapshots in the specified order + fs.deleteSnapshot(parent, ss[deleteOrder[0]]); + assertFileLength(snapshotFiles[deleteOrder[1]], length[deleteOrder[1]]); + assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]); + assertBlockExists(firstBlk); + assertBlockExists(lastBlk); + assertEquals("Number of INodes should not change", + numINodes, fsDir.getInodeMapSize()); + + // Diskspace consumed should be 16 bytes * 3. [SS:1,2,3,4] + contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(48L)); + + fs.deleteSnapshot(parent, ss[deleteOrder[1]]); + assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]); + assertBlockExists(firstBlk); + contentSummary = fs.getContentSummary(parent); + if(fs.exists(snapshotFiles[0])) { + // Diskspace consumed should be 0 bytes * 3. [SS:1,2,3] + assertBlockNotPresent(lastBlk); + assertThat(contentSummary.getSpaceConsumed(), is(36L)); + } else { + // Diskspace consumed should be 48 bytes * 3. [SS:1,2,3,4] + assertThat(contentSummary.getSpaceConsumed(), is(48L)); + } + assertEquals("Number of INodes should not change", + numINodes, fsDir .getInodeMapSize()); + + fs.deleteSnapshot(parent, ss[deleteOrder[2]]); + assertBlockNotPresent(firstBlk); + assertBlockNotPresent(lastBlk); + + // Diskspace consumed should be 0 bytes * 3. [] + contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(0L)); + assertNotEquals("Number of INodes should change", + numINodes, fsDir.getInodeMapSize()); + } + + /** + * Create three snapshots with file truncated 3 times. + * Delete snapshots in the specified order and verify that + * remaining snapshots are still readable. + */ + @Test + public void testSnapshotWithTruncates() throws IOException { + testSnapshotWithTruncates(0, 1, 2); + testSnapshotWithTruncates(0, 2, 1); + testSnapshotWithTruncates(1, 0, 2); + testSnapshotWithTruncates(1, 2, 0); + testSnapshotWithTruncates(2, 0, 1); + testSnapshotWithTruncates(2, 1, 0); + } + + void testSnapshotWithTruncates(int ... deleteOrder) throws IOException { + Path parent = new Path("/test"); + fs.mkdirs(parent); + fs.setQuota(parent, 100, 1000); + fs.allowSnapshot(parent); + String truncateFile = "testSnapshotWithTruncates"; + final Path src = new Path(parent, truncateFile); + int[] length = new int[3]; + length[0] = 3 * BLOCK_SIZE; + DFSTestUtil.createFile(fs, src, 64, length[0], BLOCK_SIZE, REPLICATION, 0L); + Block firstBlk = getLocatedBlocks(src).get(0).getBlock().getLocalBlock(); + Block lastBlk = getLocatedBlocks(src).getLastLocatedBlock() + .getBlock().getLocalBlock(); + Path[] snapshotFiles = new Path[3]; + + // Diskspace consumed should be 12 bytes * 3. [blk 1,2,3] + ContentSummary contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(36L)); + + // Add file to snapshot and append + String[] ss = new String[] {"ss0", "ss1", "ss2"}; + Path snapshotDir = fs.createSnapshot(parent, ss[0]); + snapshotFiles[0] = new Path(snapshotDir, truncateFile); + length[1] = 2 * BLOCK_SIZE; + boolean isReady = fs.truncate(src, 2 * BLOCK_SIZE); + assertTrue("Recovery is not expected.", isReady); + + // Diskspace consumed should be 12 bytes * 3. [blk 1,2 SS:3] + contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(36L)); + snapshotDir = fs.createSnapshot(parent, ss[1]); + snapshotFiles[1] = new Path(snapshotDir, truncateFile); + + // Create another snapshot with truncate + length[2] = BLOCK_SIZE + BLOCK_SIZE / 2; + isReady = fs.truncate(src, BLOCK_SIZE + BLOCK_SIZE / 2); + assertFalse("Recovery is expected.", isReady); + checkBlockRecovery(src); + snapshotDir = fs.createSnapshot(parent, ss[2]); + snapshotFiles[2] = new Path(snapshotDir, truncateFile); + assertFileLength(snapshotFiles[0], length[0]); + assertBlockExists(lastBlk); + + // Diskspace consumed should be 14 bytes * 3. [blk 1,4 SS:2,3] + contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(42L)); + + fs.deleteSnapshot(parent, ss[deleteOrder[0]]); + assertFileLength(snapshotFiles[deleteOrder[1]], length[deleteOrder[1]]); + assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]); + assertFileLength(src, length[2]); + assertBlockExists(firstBlk); + + contentSummary = fs.getContentSummary(parent); + if(fs.exists(snapshotFiles[0])) { + // Diskspace consumed should be 14 bytes * 3. [blk 1,4 SS:2,3] + assertThat(contentSummary.getSpaceConsumed(), is(42L)); + assertBlockExists(lastBlk); + } else { + // Diskspace consumed should be 10 bytes * 3. [blk 1,4 SS:2] + assertThat(contentSummary.getSpaceConsumed(), is(30L)); + assertBlockNotPresent(lastBlk); + } + + fs.deleteSnapshot(parent, ss[deleteOrder[1]]); + assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]); + assertFileLength(src, length[2]); + assertBlockExists(firstBlk); + + contentSummary = fs.getContentSummary(parent); + if(fs.exists(snapshotFiles[0])) { + // Diskspace consumed should be 14 bytes * 3. [blk 1,4 SS:2,3] + assertThat(contentSummary.getSpaceConsumed(), is(42L)); + assertBlockExists(lastBlk); + } else if(fs.exists(snapshotFiles[1])) { + // Diskspace consumed should be 10 bytes * 3. [blk 1,4 SS:2] + assertThat(contentSummary.getSpaceConsumed(), is(30L)); + assertBlockNotPresent(lastBlk); + } else { + // Diskspace consumed should be 6 bytes * 3. [blk 1,4 SS:] + assertThat(contentSummary.getSpaceConsumed(), is(18L)); + assertBlockNotPresent(lastBlk); + } + + fs.deleteSnapshot(parent, ss[deleteOrder[2]]); + assertFileLength(src, length[2]); + assertBlockExists(firstBlk); + + // Diskspace consumed should be 6 bytes * 3. [blk 1,4 SS:] + contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(18L)); + assertThat(contentSummary.getLength(), is(6L)); + + fs.delete(src, false); + assertBlockNotPresent(firstBlk); + + // Diskspace consumed should be 0 bytes * 3. [] + contentSummary = fs.getContentSummary(parent); + assertThat(contentSummary.getSpaceConsumed(), is(0L)); + } + /** * Failure / recovery test for truncate. * In this failure the DNs fail to recover the blocks and the NN triggers @@ -159,8 +449,6 @@ public class TestFileTruncate { boolean isReady = fs.truncate(p, newLength); assertThat("truncate should have triggered block recovery.", isReady, is(false)); - FileStatus fileStatus = fs.getFileStatus(p); - assertThat(fileStatus.getLen(), is((long) newLength)); boolean recoveryTriggered = false; for(int i = 0; i < RECOVERY_ATTEMPTS; i++) { @@ -168,8 +456,6 @@ public class TestFileTruncate { NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(), p.toUri().getPath()); if(leaseHolder.equals(HdfsServerConstants.NAMENODE_LEASE_HOLDER)) { - cluster.startDataNodes(conf, DATANODE_NUM, true, - HdfsServerConstants.StartupOption.REGULAR, null); recoveryTriggered = true; break; } @@ -177,6 +463,9 @@ public class TestFileTruncate { } assertThat("lease recovery should have occurred in ~" + SLEEP * RECOVERY_ATTEMPTS + " ms.", recoveryTriggered, is(true)); + cluster.startDataNodes(conf, DATANODE_NUM, true, + StartupOption.REGULAR, null); + cluster.waitActive(); checkBlockRecovery(p); @@ -184,10 +473,10 @@ public class TestFileTruncate { .setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD, HdfsConstants.LEASE_HARDLIMIT_PERIOD); - fileStatus = fs.getFileStatus(p); + FileStatus fileStatus = fs.getFileStatus(p); assertThat(fileStatus.getLen(), is((long) newLength)); - AppendTestUtil.checkFullFile(fs, p, newLength, contents, p.toString()); + checkFullFile(p, newLength, contents); fs.delete(p, false); } @@ -198,10 +487,9 @@ public class TestFileTruncate { public void testTruncateEditLogLoad() throws IOException { int startingFileSize = 2 * BLOCK_SIZE + BLOCK_SIZE / 2; int toTruncate = 1; - + final String s = "/testTruncateEditLogLoad"; + final Path p = new Path(s); byte[] contents = AppendTestUtil.initBuffer(startingFileSize); - - final Path p = new Path("/testTruncateEditLogLoad"); writeContents(contents, startingFileSize, p); int newLength = startingFileSize - toTruncate; @@ -209,54 +497,183 @@ public class TestFileTruncate { assertThat("truncate should have triggered block recovery.", isReady, is(false)); - checkBlockRecovery(p); - cluster.restartNameNode(); + String holder = UserGroupInformation.getCurrentUser().getUserName(); + cluster.getNamesystem().recoverLease(s, holder, ""); + + checkBlockRecovery(p); + FileStatus fileStatus = fs.getFileStatus(p); assertThat(fileStatus.getLen(), is((long) newLength)); - AppendTestUtil.checkFullFile(fs, p, newLength, contents, p.toString()); + checkFullFile(p, newLength, contents); fs.delete(p, false); } /** + * Upgrade, RollBack, and restart test for Truncate. + */ + @Test + public void testUpgradeAndRestart() throws IOException { + Path parent = new Path("/test"); + fs.mkdirs(parent); + fs.setQuota(parent, 100, 1000); + fs.allowSnapshot(parent); + String truncateFile = "testUpgrade"; + final Path p = new Path(parent, truncateFile); + int startingFileSize = 2 * BLOCK_SIZE; + int toTruncate = 1; + byte[] contents = AppendTestUtil.initBuffer(startingFileSize); + writeContents(contents, startingFileSize, p); + + Path snapshotDir = fs.createSnapshot(parent, "ss0"); + Path snapshotFile = new Path(snapshotDir, truncateFile); + + int newLengthBeforeUpgrade = startingFileSize - toTruncate; + boolean isReady = fs.truncate(p, newLengthBeforeUpgrade); + assertThat("truncate should have triggered block recovery.", + isReady, is(false)); + + checkBlockRecovery(p); + + checkFullFile(p, newLengthBeforeUpgrade, contents); + assertFileLength(snapshotFile, startingFileSize); + long totalBlockBefore = cluster.getNamesystem().getBlocksTotal(); + + restartCluster(StartupOption.UPGRADE); + + assertThat("SafeMode should be OFF", + cluster.getNamesystem().isInSafeMode(), is(false)); + assertThat("NameNode should be performing upgrade.", + cluster.getNamesystem().isUpgradeFinalized(), is(false)); + FileStatus fileStatus = fs.getFileStatus(p); + assertThat(fileStatus.getLen(), is((long) newLengthBeforeUpgrade)); + + int newLengthAfterUpgrade = newLengthBeforeUpgrade - toTruncate; + Block oldBlk = getLocatedBlocks(p).getLastLocatedBlock() + .getBlock().getLocalBlock(); + isReady = fs.truncate(p, newLengthAfterUpgrade); + assertThat("truncate should have triggered block recovery.", + isReady, is(false)); + fileStatus = fs.getFileStatus(p); + assertThat(fileStatus.getLen(), is((long) newLengthAfterUpgrade)); + assertThat("Should copy on truncate during upgrade", + getLocatedBlocks(p).getLastLocatedBlock().getBlock() + .getLocalBlock().getBlockId(), is(not(equalTo(oldBlk.getBlockId())))); + + checkBlockRecovery(p); + + checkFullFile(p, newLengthAfterUpgrade, contents); + assertThat("Total block count should be unchanged from copy-on-truncate", + cluster.getNamesystem().getBlocksTotal(), is(totalBlockBefore)); + + restartCluster(StartupOption.ROLLBACK); + + assertThat("File does not exist " + p, fs.exists(p), is(true)); + fileStatus = fs.getFileStatus(p); + assertThat(fileStatus.getLen(), is((long) newLengthBeforeUpgrade)); + checkFullFile(p, newLengthBeforeUpgrade, contents); + assertThat("Total block count should be unchanged from rolling back", + cluster.getNamesystem().getBlocksTotal(), is(totalBlockBefore)); + + restartCluster(StartupOption.REGULAR); + assertThat("Total block count should be unchanged from start-up", + cluster.getNamesystem().getBlocksTotal(), is(totalBlockBefore)); + checkFullFile(p, newLengthBeforeUpgrade, contents); + assertFileLength(snapshotFile, startingFileSize); + + // empty edits and restart + fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + fs.saveNamespace(); + cluster.restartNameNode(true); + assertThat("Total block count should be unchanged from start-up", + cluster.getNamesystem().getBlocksTotal(), is(totalBlockBefore)); + checkFullFile(p, newLengthBeforeUpgrade, contents); + assertFileLength(snapshotFile, startingFileSize); + + fs.deleteSnapshot(parent, "ss0"); + fs.delete(parent, true); + assertThat("File " + p + " shouldn't exist", fs.exists(p), is(false)); + } + + /** * Check truncate recovery. */ @Test - public void testTruncateLastBlock() throws IOException { + public void testTruncateRecovery() throws IOException { FSNamesystem fsn = cluster.getNamesystem(); - - String src = "/file"; + String client = "client"; + String clientMachine = "clientMachine"; + Path parent = new Path("/test"); + String src = "/test/testTruncateRecovery"; Path srcPath = new Path(src); byte[] contents = AppendTestUtil.initBuffer(BLOCK_SIZE); writeContents(contents, BLOCK_SIZE, srcPath); - INodeFile inode = fsn.getFSDirectory().getINode(src).asFile(); - long oldGenstamp = GenerationStamp.LAST_RESERVED_STAMP; - DatanodeDescriptor dn = DFSTestUtil.getLocalDatanodeDescriptor(); - DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo( - dn.getDatanodeUuid(), InetAddress.getLocalHost().getHostAddress()); - dn.isAlive = true; - - BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction( - new Block(0, 1, oldGenstamp), (short) 1, - HdfsServerConstants.BlockUCState.BEING_TRUNCATED, - new DatanodeStorageInfo[] {storage}); + INodesInPath iip = fsn.getFSDirectory().getINodesInPath4Write(src, true); + INodeFile file = iip.getLastINode().asFile(); + long initialGenStamp = file.getLastBlock().getGenerationStamp(); + // Test that prepareFileForTruncate sets up in-place truncate. + fsn.writeLock(); + try { + Block oldBlock = file.getLastBlock(); + Block truncateBlock = + fsn.prepareFileForTruncate(iip, client, clientMachine, 1, null); + // In-place truncate uses old block id with new genStamp. + assertThat(truncateBlock.getBlockId(), + is(equalTo(oldBlock.getBlockId()))); + assertThat(truncateBlock.getNumBytes(), + is(oldBlock.getNumBytes())); + assertThat(truncateBlock.getGenerationStamp(), + is(fsn.getBlockIdManager().getGenerationStampV2())); + assertThat(file.getLastBlock().getBlockUCState(), + is(HdfsServerConstants.BlockUCState.UNDER_RECOVERY)); + long blockRecoveryId = ((BlockInfoUnderConstruction) file.getLastBlock()) + .getBlockRecoveryId(); + assertThat(blockRecoveryId, is(initialGenStamp + 1)); + fsn.getEditLog().logTruncate( + src, client, clientMachine, BLOCK_SIZE-1, Time.now(), truncateBlock); + } finally { + fsn.writeUnlock(); + } - inode.setBlocks(new BlockInfo[] {blockInfo}); + // Re-create file and ensure we are ready to copy on truncate + writeContents(contents, BLOCK_SIZE, srcPath); + fs.allowSnapshot(parent); + fs.createSnapshot(parent, "ss0"); + iip = fsn.getFSDirectory().getINodesInPath(src, true); + file = iip.getLastINode().asFile(); + file.recordModification(iip.getLatestSnapshotId(), true); + assertThat(file.isBlockInLatestSnapshot(file.getLastBlock()), is(true)); + initialGenStamp = file.getLastBlock().getGenerationStamp(); + // Test that prepareFileForTruncate sets up copy-on-write truncate fsn.writeLock(); try { - fsn.initializeBlockRecovery(inode); - assertThat(inode.getLastBlock().getBlockUCState(), - is(HdfsServerConstants.BlockUCState.BEING_TRUNCATED)); - long blockRecoveryId = ((BlockInfoUnderConstruction) inode.getLastBlock()) + Block oldBlock = file.getLastBlock(); + Block truncateBlock = + fsn.prepareFileForTruncate(iip, client, clientMachine, 1, null); + // Copy-on-write truncate makes new block with new id and genStamp + assertThat(truncateBlock.getBlockId(), + is(not(equalTo(oldBlock.getBlockId())))); + assertThat(truncateBlock.getNumBytes() < oldBlock.getNumBytes(), + is(true)); + assertThat(truncateBlock.getGenerationStamp(), + is(fsn.getBlockIdManager().getGenerationStampV2())); + assertThat(file.getLastBlock().getBlockUCState(), + is(HdfsServerConstants.BlockUCState.UNDER_RECOVERY)); + long blockRecoveryId = ((BlockInfoUnderConstruction) file.getLastBlock()) .getBlockRecoveryId(); - assertThat(blockRecoveryId, is(oldGenstamp + 2)); + assertThat(blockRecoveryId, is(initialGenStamp + 1)); + fsn.getEditLog().logTruncate( + src, client, clientMachine, BLOCK_SIZE-1, Time.now(), truncateBlock); } finally { fsn.writeUnlock(); } + checkBlockRecovery(srcPath); + fs.deleteSnapshot(parent, "ss0"); + fs.delete(parent, true); } static void writeContents(byte[] contents, int fileLength, Path p) @@ -286,4 +703,38 @@ public class TestFileTruncate { static LocatedBlocks getLocatedBlocks(Path src) throws IOException { return fs.getClient().getLocatedBlocks(src.toString(), 0, Long.MAX_VALUE); } + + static void assertBlockExists(Block blk) { + assertNotNull("BlocksMap does not contain block: " + blk, + cluster.getNamesystem().getStoredBlock(blk)); + } + + static void assertBlockNotPresent(Block blk) { + assertNull("BlocksMap should not contain block: " + blk, + cluster.getNamesystem().getStoredBlock(blk)); + } + + static void assertFileLength(Path file, long length) throws IOException { + byte[] data = DFSTestUtil.readFileBuffer(fs, file); + assertEquals("Wrong data size in snapshot.", length, data.length); + } + + static void checkFullFile(Path p, int newLength, byte[] contents) + throws IOException { + AppendTestUtil.checkFullFile(fs, p, newLength, contents, p.toString()); + } + + static void restartCluster(StartupOption o) + throws IOException { + cluster.shutdown(); + if(StartupOption.ROLLBACK == o) + NameNode.doRollback(conf, false); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_NUM) + .format(false) + .nameNodePort(NameNode.DEFAULT_PORT) + .startupOption(o==StartupOption.ROLLBACK ? StartupOption.REGULAR : o) + .dnStartupOption(o!=StartupOption.ROLLBACK ? StartupOption.REGULAR : o) + .build(); + fs = cluster.getFileSystem(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a7ebab/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored index 002c506..dce3f47 100644 Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored differ http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a7ebab/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml index f61c075..6e8078b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <EDITS> - <EDITS_VERSION>-60</EDITS_VERSION> + <EDITS_VERSION>-61</EDITS_VERSION> <RECORD> <OPCODE>OP_START_LOG_SEGMENT</OPCODE> <DATA> @@ -13,8 +13,8 @@ <TXID>2</TXID> <DELEGATION_KEY> <KEY_ID>1</KEY_ID> - <EXPIRY_DATE>1421822547136</EXPIRY_DATE> - <KEY>24319c7d1f7c0828</KEY> + <EXPIRY_DATE>1421826999207</EXPIRY_DATE> + <KEY>ca9a0c8b240570b3</KEY> </DELEGATION_KEY> </DATA> </RECORD> @@ -24,8 +24,8 @@ <TXID>3</TXID> <DELEGATION_KEY> <KEY_ID>2</KEY_ID> - <EXPIRY_DATE>1421822547140</EXPIRY_DATE> - <KEY>254b1207021431f4</KEY> + <EXPIRY_DATE>1421826999210</EXPIRY_DATE> + <KEY>833c25a6fb2b0a6f</KEY> </DELEGATION_KEY> </DATA> </RECORD> @@ -37,19 +37,19 @@ <INODEID>16386</INODEID> <PATH>/file_create</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131348286</MTIME> - <ATIME>1421131348286</ATIME> + <MTIME>1421135800328</MTIME> + <ATIME>1421135800328</ATIME> <BLOCKSIZE>512</BLOCKSIZE> - <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME> + <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME> <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE> <OVERWRITE>true</OVERWRITE> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>6</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>9</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -60,14 +60,14 @@ <INODEID>0</INODEID> <PATH>/file_create</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131348328</MTIME> - <ATIME>1421131348286</ATIME> + <MTIME>1421135800357</MTIME> + <ATIME>1421135800328</ATIME> <BLOCKSIZE>512</BLOCKSIZE> <CLIENT_NAME></CLIENT_NAME> <CLIENT_MACHINE></CLIENT_MACHINE> <OVERWRITE>false</OVERWRITE> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> @@ -88,9 +88,9 @@ <LENGTH>0</LENGTH> <SRC>/file_create</SRC> <DST>/file_moved</DST> - <TIMESTAMP>1421131348343</TIMESTAMP> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>9</RPC_CALLID> + <TIMESTAMP>1421135800368</TIMESTAMP> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>12</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -99,9 +99,9 @@ <TXID>8</TXID> <LENGTH>0</LENGTH> <PATH>/file_moved</PATH> - <TIMESTAMP>1421131348353</TIMESTAMP> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>10</RPC_CALLID> + <TIMESTAMP>1421135800377</TIMESTAMP> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>13</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -111,9 +111,9 @@ <LENGTH>0</LENGTH> <INODEID>16387</INODEID> <PATH>/directory_mkdir</PATH> - <TIMESTAMP>1421131348366</TIMESTAMP> + <TIMESTAMP>1421135800394</TIMESTAMP> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>493</MODE> </PERMISSION_STATUS> @@ -146,8 +146,8 @@ <TXID>13</TXID> <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT> <SNAPSHOTNAME>snapshot1</SNAPSHOTNAME> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>15</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>18</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -157,8 +157,8 @@ <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT> <SNAPSHOTOLDNAME>snapshot1</SNAPSHOTOLDNAME> <SNAPSHOTNEWNAME>snapshot2</SNAPSHOTNEWNAME> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>16</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>19</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -167,8 +167,8 @@ <TXID>15</TXID> <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT> <SNAPSHOTNAME>snapshot2</SNAPSHOTNAME> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>17</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>20</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -179,19 +179,19 @@ <INODEID>16388</INODEID> <PATH>/file_create</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131348401</MTIME> - <ATIME>1421131348401</ATIME> + <MTIME>1421135800442</MTIME> + <ATIME>1421135800442</ATIME> <BLOCKSIZE>512</BLOCKSIZE> - <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME> + <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME> <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE> <OVERWRITE>true</OVERWRITE> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>18</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>21</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -202,14 +202,14 @@ <INODEID>0</INODEID> <PATH>/file_create</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131348405</MTIME> - <ATIME>1421131348401</ATIME> + <MTIME>1421135800445</MTIME> + <ATIME>1421135800442</ATIME> <BLOCKSIZE>512</BLOCKSIZE> <CLIENT_NAME></CLIENT_NAME> <CLIENT_MACHINE></CLIENT_MACHINE> <OVERWRITE>false</OVERWRITE> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> @@ -265,10 +265,10 @@ <LENGTH>0</LENGTH> <SRC>/file_create</SRC> <DST>/file_moved</DST> - <TIMESTAMP>1421131348436</TIMESTAMP> + <TIMESTAMP>1421135800485</TIMESTAMP> <OPTIONS>NONE</OPTIONS> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>25</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>28</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -279,19 +279,19 @@ <INODEID>16389</INODEID> <PATH>/file_concat_target</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131348443</MTIME> - <ATIME>1421131348443</ATIME> + <MTIME>1421135800495</MTIME> + <ATIME>1421135800495</ATIME> <BLOCKSIZE>512</BLOCKSIZE> - <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME> + <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME> <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE> <OVERWRITE>true</OVERWRITE> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>27</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>30</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -396,8 +396,8 @@ <INODEID>0</INODEID> <PATH>/file_concat_target</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131348998</MTIME> - <ATIME>1421131348443</ATIME> + <MTIME>1421135801050</MTIME> + <ATIME>1421135800495</ATIME> <BLOCKSIZE>512</BLOCKSIZE> <CLIENT_NAME></CLIENT_NAME> <CLIENT_MACHINE></CLIENT_MACHINE> @@ -418,7 +418,7 @@ <GENSTAMP>1003</GENSTAMP> </BLOCK> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> @@ -432,19 +432,19 @@ <INODEID>16390</INODEID> <PATH>/file_concat_0</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131349001</MTIME> - <ATIME>1421131349001</ATIME> + <MTIME>1421135801053</MTIME> + <ATIME>1421135801053</ATIME> <BLOCKSIZE>512</BLOCKSIZE> - <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME> + <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME> <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE> <OVERWRITE>true</OVERWRITE> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>38</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>41</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -549,8 +549,8 @@ <INODEID>0</INODEID> <PATH>/file_concat_0</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131349032</MTIME> - <ATIME>1421131349001</ATIME> + <MTIME>1421135801091</MTIME> + <ATIME>1421135801053</ATIME> <BLOCKSIZE>512</BLOCKSIZE> <CLIENT_NAME></CLIENT_NAME> <CLIENT_MACHINE></CLIENT_MACHINE> @@ -571,7 +571,7 @@ <GENSTAMP>1006</GENSTAMP> </BLOCK> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> @@ -585,19 +585,19 @@ <INODEID>16391</INODEID> <PATH>/file_concat_1</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131349036</MTIME> - <ATIME>1421131349036</ATIME> + <MTIME>1421135801095</MTIME> + <ATIME>1421135801095</ATIME> <BLOCKSIZE>512</BLOCKSIZE> - <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME> + <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME> <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE> <OVERWRITE>true</OVERWRITE> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>47</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>50</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -702,8 +702,8 @@ <INODEID>0</INODEID> <PATH>/file_concat_1</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131349060</MTIME> - <ATIME>1421131349036</ATIME> + <MTIME>1421135801126</MTIME> + <ATIME>1421135801095</ATIME> <BLOCKSIZE>512</BLOCKSIZE> <CLIENT_NAME></CLIENT_NAME> <CLIENT_MACHINE></CLIENT_MACHINE> @@ -724,7 +724,7 @@ <GENSTAMP>1009</GENSTAMP> </BLOCK> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> @@ -736,13 +736,13 @@ <TXID>57</TXID> <LENGTH>0</LENGTH> <TRG>/file_concat_target</TRG> - <TIMESTAMP>1421131349064</TIMESTAMP> + <TIMESTAMP>1421135801130</TIMESTAMP> <SOURCES> <SOURCE1>/file_concat_0</SOURCE1> <SOURCE2>/file_concat_1</SOURCE2> </SOURCES> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>55</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>58</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -753,19 +753,19 @@ <INODEID>16392</INODEID> <PATH>/file_create</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131349068</MTIME> - <ATIME>1421131349068</ATIME> + <MTIME>1421135810102</MTIME> + <ATIME>1421135810102</ATIME> <BLOCKSIZE>512</BLOCKSIZE> - <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME> + <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME> <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE> <OVERWRITE>true</OVERWRITE> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>57</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>63</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -837,8 +837,8 @@ <INODEID>0</INODEID> <PATH>/file_create</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131349085</MTIME> - <ATIME>1421131349068</ATIME> + <MTIME>1421135810122</MTIME> + <ATIME>1421135810102</ATIME> <BLOCKSIZE>512</BLOCKSIZE> <CLIENT_NAME></CLIENT_NAME> <CLIENT_MACHINE></CLIENT_MACHINE> @@ -854,7 +854,7 @@ <GENSTAMP>1011</GENSTAMP> </BLOCK> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> @@ -865,10 +865,10 @@ <DATA> <TXID>66</TXID> <SRC>/file_create</SRC> - <CLIENTNAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENTNAME> + <CLIENTNAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENTNAME> <CLIENTMACHINE>127.0.0.1</CLIENTMACHINE> <NEWLENGTH>512</NEWLENGTH> - <TIMESTAMP>1421131349088</TIMESTAMP> + <TIMESTAMP>1421135810125</TIMESTAMP> </DATA> </RECORD> <RECORD> @@ -879,15 +879,15 @@ <INODEID>16393</INODEID> <PATH>/file_symlink</PATH> <VALUE>/file_concat_target</VALUE> - <MTIME>1421131349095</MTIME> - <ATIME>1421131349095</ATIME> + <MTIME>1421135810132</MTIME> + <ATIME>1421135810132</ATIME> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>511</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>64</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>70</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -898,19 +898,19 @@ <INODEID>16394</INODEID> <PATH>/hard-lease-recovery-test</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131349098</MTIME> - <ATIME>1421131349098</ATIME> + <MTIME>1421135810135</MTIME> + <ATIME>1421135810135</ATIME> <BLOCKSIZE>512</BLOCKSIZE> - <CLIENT_NAME>DFSClient_NONMAPREDUCE_526346936_1</CLIENT_NAME> + <CLIENT_NAME>DFSClient_NONMAPREDUCE_240777107_1</CLIENT_NAME> <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE> <OVERWRITE>true</OVERWRITE> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>65</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>71</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -966,7 +966,7 @@ <OPCODE>OP_REASSIGN_LEASE</OPCODE> <DATA> <TXID>74</TXID> - <LEASEHOLDER>DFSClient_NONMAPREDUCE_526346936_1</LEASEHOLDER> + <LEASEHOLDER>DFSClient_NONMAPREDUCE_240777107_1</LEASEHOLDER> <PATH>/hard-lease-recovery-test</PATH> <NEWHOLDER>HDFS_NameNode</NEWHOLDER> </DATA> @@ -979,8 +979,8 @@ <INODEID>0</INODEID> <PATH>/hard-lease-recovery-test</PATH> <REPLICATION>1</REPLICATION> - <MTIME>1421131351230</MTIME> - <ATIME>1421131349098</ATIME> + <MTIME>1421135812235</MTIME> + <ATIME>1421135810135</ATIME> <BLOCKSIZE>512</BLOCKSIZE> <CLIENT_NAME></CLIENT_NAME> <CLIENT_MACHINE></CLIENT_MACHINE> @@ -991,7 +991,7 @@ <GENSTAMP>1013</GENSTAMP> </BLOCK> <PERMISSION_STATUS> - <USERNAME>plamenjeliazkov</USERNAME> + <USERNAME>shv</USERNAME> <GROUPNAME>supergroup</GROUPNAME> <MODE>420</MODE> </PERMISSION_STATUS> @@ -1002,13 +1002,13 @@ <DATA> <TXID>76</TXID> <POOLNAME>pool1</POOLNAME> - <OWNERNAME>plamenjeliazkov</OWNERNAME> - <GROUPNAME>staff</GROUPNAME> + <OWNERNAME>shv</OWNERNAME> + <GROUPNAME>shv</GROUPNAME> <MODE>493</MODE> <LIMIT>9223372036854775807</LIMIT> <MAXRELATIVEEXPIRY>2305843009213693951</MAXRELATIVEEXPIRY> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>72</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>78</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -1017,8 +1017,8 @@ <TXID>77</TXID> <POOLNAME>pool1</POOLNAME> <LIMIT>99</LIMIT> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>73</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>79</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -1029,9 +1029,9 @@ <PATH>/path</PATH> <REPLICATION>1</REPLICATION> <POOL>pool1</POOL> - <EXPIRATION>2305844430345046085</EXPIRATION> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>74</RPC_CALLID> + <EXPIRATION>2305844430349507141</EXPIRATION> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>80</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -1040,8 +1040,8 @@ <TXID>79</TXID> <ID>1</ID> <REPLICATION>2</REPLICATION> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>75</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>81</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -1049,8 +1049,8 @@ <DATA> <TXID>80</TXID> <ID>1</ID> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>76</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>82</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -1058,8 +1058,8 @@ <DATA> <TXID>81</TXID> <POOLNAME>pool1</POOLNAME> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>77</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>83</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -1105,8 +1105,8 @@ <NAME>a1</NAME> <VALUE>0x313233</VALUE> </XATTR> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>79</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>85</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -1119,8 +1119,8 @@ <NAME>a2</NAME> <VALUE>0x373839</VALUE> </XATTR> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>80</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>86</RPC_CALLID> </DATA> </RECORD> <RECORD> @@ -1132,22 +1132,22 @@ <NAMESPACE>USER</NAMESPACE> <NAME>a2</NAME> </XATTR> - <RPC_CLIENTID>99bcddc1-3460-4630-9904-6c7ca5811945</RPC_CLIENTID> - <RPC_CALLID>81</RPC_CALLID> + <RPC_CLIENTID>cb20a92a-2c2f-4305-a838-2a01c6e73e18</RPC_CLIENTID> + <RPC_CALLID>87</RPC_CALLID> </DATA> </RECORD> <RECORD> <OPCODE>OP_ROLLING_UPGRADE_START</OPCODE> <DATA> <TXID>86</TXID> - <STARTTIME>1421131352186</STARTTIME> + <STARTTIME>1421135813268</STARTTIME> </DATA> </RECORD> <RECORD> <OPCODE>OP_ROLLING_UPGRADE_FINALIZE</OPCODE> <DATA> <TXID>87</TXID> - <FINALIZETIME>1421131352186</FINALIZETIME> + <FINALIZETIME>1421135813268</FINALIZETIME> </DATA> </RECORD> <RECORD>