http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java index 99b617e..65a484c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java @@ -39,7 +39,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; -import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface; +import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipeline; import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.tools.DFSAdmin; @@ -530,7 +530,7 @@ public class TestClientProtocolForPipelineRecovery { DataNodeFaultInjector.set(new DataNodeFaultInjector() { @Override - public void failPipeline(ReplicaInPipelineInterface replicaInfo, + public void failPipeline(ReplicaInPipeline replicaInfo, String mirror) throws IOException { if (!lastDn.equals(mirror)) { // Only fail for second DN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java index 398bcc2..917f0db 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java @@ -34,7 +34,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica; +import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo; import org.apache.hadoop.io.IOUtils; import org.junit.Before; import org.junit.Test; @@ -172,12 +172,12 @@ public class TestCrcCorruption { final int dnIdx = 0; final DataNode dn = cluster.getDataNodes().get(dnIdx); final String bpid = cluster.getNamesystem().getBlockPoolId(); - List<FinalizedReplica> replicas = + List<ReplicaInfo> replicas = dn.getFSDataset().getFinalizedBlocks(bpid); assertTrue("Replicas do not exist", !replicas.isEmpty()); for (int idx = 0; idx < replicas.size(); idx++) { - FinalizedReplica replica = replicas.get(idx); + ReplicaInfo replica = replicas.get(idx); ExtendedBlock eb = new ExtendedBlock(bpid, replica); if (idx % 3 == 0) { LOG.info("Deliberately removing meta for block " + eb); http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index a082fbb..6034d1e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -132,7 +132,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { } // information about a single block - private class BInfo implements ReplicaInPipelineInterface { + private class BInfo implements ReplicaInPipeline { final Block theBlock; private boolean finalized = false; // if not finalized => ongoing creation SimulatedOutputStream oStream = null; @@ -330,6 +330,28 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { public boolean isOnTransientStorage() { return false; } + + @Override + public ReplicaInfo getReplicaInfo() { + return null; + } + + @Override + public void setWriter(Thread writer) { + } + + @Override + public void interruptThread() { + } + + @Override + public boolean attemptToSetWriter(Thread prevWriter, Thread newWriter) { + return false; + } + + @Override + public void stopWriter(long xceiverStopTimeout) throws IOException { + } } /** @@ -1228,7 +1250,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { } @Override - public ReplicaInPipelineInterface convertTemporaryToRbw(ExtendedBlock temporary) + public ReplicaInPipeline convertTemporaryToRbw(ExtendedBlock temporary) throws IOException { final Map<Block, BInfo> map = blockMap.get(temporary.getBlockPoolId()); if (map == null) { @@ -1302,12 +1324,12 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { } @Override - public List<FinalizedReplica> getFinalizedBlocks(String bpid) { + public List<ReplicaInfo> getFinalizedBlocks(String bpid) { throw new UnsupportedOperationException(); } @Override - public List<FinalizedReplica> getFinalizedBlocksOnPersistentStorage(String bpid) { + public List<ReplicaInfo> getFinalizedBlocksOnPersistentStorage(String bpid) { throw new UnsupportedOperationException(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java index b5951a0..a8f8b6a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java @@ -21,6 +21,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hdfs.server.common.Storage; import org.junit.Test; +import org.mockito.Mockito; import java.io.File; import java.util.Random; @@ -105,7 +106,10 @@ public class TestBlockPoolSliceStorage { LOG.info("Got subdir " + blockFileSubdir); LOG.info("Generated file path " + testFilePath); - assertThat(storage.getTrashDirectory(new File(testFilePath)), is(expectedTrashPath)); + + ReplicaInfo info = Mockito.mock(ReplicaInfo.class); + Mockito.when(info.getBlockURI()).thenReturn(new File(testFilePath).toURI()); + assertThat(storage.getTrashDirectory(info), is(expectedTrashPath)); } /* http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java index 8183de8..c09303f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java @@ -667,7 +667,7 @@ public class TestBlockRecovery { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); } - ReplicaInPipelineInterface replicaInfo = dn.data.createRbw( + ReplicaInPipeline replicaInfo = dn.data.createRbw( StorageType.DEFAULT, block, false).getReplica(); ReplicaOutputStreams streams = null; try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java index 7e56988..f08606e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.tools.DFSAdmin; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; +import org.mockito.Mockito; /** * Ensure that the DataNode correctly handles rolling upgrade @@ -114,8 +115,11 @@ public class TestDataNodeRollingUpgrade { } private File getTrashFileForBlock(File blockFile, boolean exists) { + + ReplicaInfo info = Mockito.mock(ReplicaInfo.class); + Mockito.when(info.getBlockURI()).thenReturn(blockFile.toURI()); File trashFile = new File( - dn0.getStorage().getTrashDirectoryForBlockFile(blockPoolId, blockFile)); + dn0.getStorage().getTrashDirectoryForReplica(blockPoolId, info)); assertEquals(exists, trashFile.exists()); return trashFile; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java index 3822bad..576aae0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java @@ -116,8 +116,8 @@ public class TestDirectoryScanner { private long truncateBlockFile() throws IOException { try(AutoCloseableLock lock = fds.acquireDatasetLock()) { for (ReplicaInfo b : FsDatasetTestUtil.getReplicas(fds, bpid)) { - File f = b.getBlockFile(); - File mf = b.getMetaFile(); + File f = new File(b.getBlockURI()); + File mf = new File(b.getMetadataURI()); // Truncate a block file that has a corresponding metadata file if (f.exists() && f.length() != 0 && mf.exists()) { FileOutputStream s = null; @@ -141,8 +141,8 @@ public class TestDirectoryScanner { private long deleteBlockFile() { try(AutoCloseableLock lock = fds.acquireDatasetLock()) { for (ReplicaInfo b : FsDatasetTestUtil.getReplicas(fds, bpid)) { - File f = b.getBlockFile(); - File mf = b.getMetaFile(); + File f = new File(b.getBlockURI()); + File mf = new File(b.getMetadataURI()); // Delete a block file that has corresponding metadata file if (f.exists() && mf.exists() && f.delete()) { LOG.info("Deleting block file " + f.getAbsolutePath()); @@ -157,10 +157,9 @@ public class TestDirectoryScanner { private long deleteMetaFile() { try(AutoCloseableLock lock = fds.acquireDatasetLock()) { for (ReplicaInfo b : FsDatasetTestUtil.getReplicas(fds, bpid)) { - File file = b.getMetaFile(); // Delete a metadata file - if (file.exists() && file.delete()) { - LOG.info("Deleting metadata file " + file.getAbsolutePath()); + if (b.metadataExists() && b.deleteMetadata()) { + LOG.info("Deleting metadata " + b.getMetadataURI()); return b.getBlockId(); } } @@ -184,8 +183,8 @@ public class TestDirectoryScanner { } // Volume without a copy of the block. Make a copy now. - File sourceBlock = b.getBlockFile(); - File sourceMeta = b.getMetaFile(); + File sourceBlock = new File(b.getBlockURI()); + File sourceMeta = new File(b.getMetadataURI()); String sourceRoot = b.getVolume().getBasePath(); String destRoot = v.getBasePath(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java index dd7d239..4e724bc7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java @@ -80,7 +80,7 @@ public class TestSimulatedFSDataset { ExtendedBlock b = new ExtendedBlock(bpid, blkID, 0, 0); // we pass expected len as zero, - fsdataset should use the sizeof actual // data written - ReplicaInPipelineInterface bInfo = fsdataset.createRbw( + ReplicaInPipeline bInfo = fsdataset.createRbw( StorageType.DEFAULT, b, false).getReplica(); ReplicaOutputStreams out = bInfo.createStreams(true, DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTransferRbw.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTransferRbw.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTransferRbw.java index c3cb862..708fbaf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTransferRbw.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTransferRbw.java @@ -57,7 +57,7 @@ public class TestTransferRbw { String bpid) throws InterruptedException { return (ReplicaBeingWritten)getReplica(datanode, bpid, ReplicaState.RBW); } - private static ReplicaInPipeline getReplica(final DataNode datanode, + private static LocalReplicaInPipeline getReplica(final DataNode datanode, final String bpid, final ReplicaState expectedState) throws InterruptedException { final Collection<ReplicaInfo> replicas = FsDatasetTestUtil.getReplicas( datanode.getFSDataset(), bpid); @@ -68,7 +68,7 @@ public class TestTransferRbw { Assert.assertEquals(1, replicas.size()); final ReplicaInfo r = replicas.iterator().next(); Assert.assertEquals(expectedState, r.getState()); - return (ReplicaInPipeline)r; + return (LocalReplicaInPipeline)r; } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java index fcd960a..1268108 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java @@ -87,12 +87,12 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> { } @Override - public List<FinalizedReplica> getFinalizedBlocks(String bpid) { + public List<ReplicaInfo> getFinalizedBlocks(String bpid) { return null; } @Override - public List<FinalizedReplica> getFinalizedBlocksOnPersistentStorage(String bpid) { + public List<ReplicaInfo> getFinalizedBlocksOnPersistentStorage(String bpid) { return null; } @@ -159,7 +159,7 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> { } @Override - public ReplicaInPipelineInterface convertTemporaryToRbw( + public ReplicaInPipeline convertTemporaryToRbw( ExtendedBlock temporary) throws IOException { return new ExternalReplicaInPipeline(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java index a0039bc..90c3b8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java @@ -23,11 +23,12 @@ import java.io.OutputStream; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.datanode.ChunkChecksum; -import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface; +import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipeline; +import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; import org.apache.hadoop.util.DataChecksum; -public class ExternalReplicaInPipeline implements ReplicaInPipelineInterface { +public class ExternalReplicaInPipeline implements ReplicaInPipeline { @Override public void setNumBytes(long bytesReceived) { @@ -105,4 +106,25 @@ public class ExternalReplicaInPipeline implements ReplicaInPipelineInterface { public boolean isOnTransientStorage() { return false; } + + @Override + public ReplicaInfo getReplicaInfo() { + return null; + } + + public void setWriter(Thread writer) { + } + + public void stopWriter(long xceiverStopTimeout) + throws IOException { + } + + @Override + public boolean attemptToSetWriter(Thread prevWriter, Thread newWriter) { + return false; + } + + @Override + public void interruptThread() { + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/TestExternalDataset.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/TestExternalDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/TestExternalDataset.java index 82a6951..e439152 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/TestExternalDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/TestExternalDataset.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.datanode.extdataset; import org.apache.hadoop.hdfs.server.datanode.Replica; -import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface; +import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipeline; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.junit.Test; @@ -75,7 +75,7 @@ public class TestExternalDataset { */ @Test public void testInstantiateReplicaInPipeline() throws Throwable { - ReplicaInPipelineInterface inst = new ExternalReplicaInPipeline(); + ReplicaInPipeline inst = new ExternalReplicaInPipeline(); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java index d2f3db0..e1825f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java @@ -26,8 +26,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.DF; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataStorage; import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil; @@ -35,11 +35,11 @@ import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica; import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils; import org.apache.hadoop.hdfs.server.datanode.Replica; import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten; -import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipeline; +import org.apache.hadoop.hdfs.server.datanode.ReplicaBuilder; +import org.apache.hadoop.hdfs.server.datanode.LocalReplicaInPipeline; import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo; import org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException; import org.apache.hadoop.hdfs.server.datanode.ReplicaUnderRecovery; -import org.apache.hadoop.hdfs.server.datanode.ReplicaWaitingToBeRecovered; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi.FsVolumeReferences; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.test.GenericTestUtils; @@ -205,8 +205,8 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils { dataset = (FsDatasetImpl) datanode.getFSDataset(); } - private File getBlockFile(ExtendedBlock eb) throws IOException { - return dataset.getBlockFile(eb.getBlockPoolId(), eb.getBlockId()); + private ReplicaInfo getBlockFile(ExtendedBlock eb) throws IOException { + return dataset.getReplicaInfo(eb); } /** @@ -217,8 +217,8 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils { throws ReplicaNotFoundException { File blockFile; try { - blockFile = dataset.getBlockFile( - block.getBlockPoolId(), block.getBlockId()); + ReplicaInfo r = dataset.getReplicaInfo(block); + blockFile = new File(r.getBlockURI()); } catch (IOException e) { LOG.error("Block file for " + block + " does not existed:", e); throw new ReplicaNotFoundException(block); @@ -240,7 +240,7 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils { public Replica createFinalizedReplica(FsVolumeSpi volume, ExtendedBlock block) throws IOException { FsVolumeImpl vol = (FsVolumeImpl) volume; - ReplicaInfo info = new FinalizedReplica(block.getLocalBlock(), vol, + FinalizedReplica info = new FinalizedReplica(block.getLocalBlock(), vol, vol.getCurrentDir().getParentFile()); dataset.volumeMap.add(block.getBlockPoolId(), info); info.getBlockFile().createNewFile(); @@ -260,7 +260,7 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils { public Replica createReplicaInPipeline( FsVolumeSpi volume, ExtendedBlock block) throws IOException { FsVolumeImpl vol = (FsVolumeImpl) volume; - ReplicaInPipeline rip = new ReplicaInPipeline( + LocalReplicaInPipeline rip = new LocalReplicaInPipeline( block.getBlockId(), block.getGenerationStamp(), volume, vol.createTmpFile( block.getBlockPoolId(), block.getLocalBlock()).getParentFile(), @@ -305,9 +305,11 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils { FsVolumeImpl vol = (FsVolumeImpl) volume; final String bpid = eb.getBlockPoolId(); final Block block = eb.getLocalBlock(); - ReplicaWaitingToBeRecovered rwbr = - new ReplicaWaitingToBeRecovered(eb.getLocalBlock(), volume, - vol.createRbwFile(bpid, block).getParentFile()); + ReplicaInfo rwbr = new ReplicaBuilder(ReplicaState.RWR) + .setBlock(eb.getLocalBlock()) + .setFsVolume(volume) + .setDirectoryToUse(vol.createRbwFile(bpid, block).getParentFile()) + .build(); dataset.volumeMap.add(bpid, rwbr); return rwbr; } @@ -354,6 +356,7 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils { "Meta file " + metaFile + " already exists." ); } + dataset.volumeMap.add(block.getBlockPoolId(), finalized); } } @@ -379,25 +382,21 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils { @Override public long getStoredDataLength(ExtendedBlock block) throws IOException { - File f = getBlockFile(block); - try (RandomAccessFile raf = new RandomAccessFile(f, "r")) { - return raf.length(); - } + ReplicaInfo r = getBlockFile(block); + return r.getBlockDataLength(); } @Override public long getStoredGenerationStamp(ExtendedBlock block) throws IOException { - File f = getBlockFile(block); - File dir = f.getParentFile(); - File[] files = FileUtil.listFiles(dir); - return FsDatasetUtil.getGenerationStampFromFile(files, f); + ReplicaInfo r = getBlockFile(block); + return r.getGenerationStamp(); } @Override public void changeStoredGenerationStamp( ExtendedBlock block, long newGenStamp) throws IOException { - File blockFile = - dataset.getBlockFile(block.getBlockPoolId(), block.getBlockId()); + ReplicaInfo r = dataset.getReplicaInfo(block); + File blockFile = new File(r.getBlockURI()); File metaFile = FsDatasetUtil.findMetaFile(blockFile); File newMetaFile = new File( DatanodeUtil.getMetaName(blockFile.getAbsolutePath(), newGenStamp)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java index 665befa..b42c052 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetTestUtil.java @@ -24,16 +24,16 @@ import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; import java.util.Collection; -import java.util.Random; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.datanode.LocalReplica; import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo; +import org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; -import org.apache.hadoop.io.IOUtils; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; @@ -41,12 +41,21 @@ import static org.junit.Assert.fail; public class FsDatasetTestUtil { public static File getFile(FsDatasetSpi<?> fsd, String bpid, long bid) { - return ((FsDatasetImpl)fsd).getFile(bpid, bid, false); + ReplicaInfo r; + try { + r = ((FsDatasetImpl)fsd).getReplicaInfo(bpid, bid); + return new File(r.getBlockURI()); + } catch (ReplicaNotFoundException e) { + FsDatasetImpl.LOG.warn(String.format( + "Replica with id %d was not found in block pool %s.", bid, bpid), e); + } + return null; } public static File getBlockFile(FsDatasetSpi<?> fsd, String bpid, Block b ) throws IOException { - return ((FsDatasetImpl)fsd).getBlockFile(bpid, b.getBlockId()); + ReplicaInfo r = ((FsDatasetImpl)fsd).getReplicaInfo(bpid, b.getBlockId()); + return new File(r.getBlockURI()); } public static File getMetaFile(FsDatasetSpi<?> fsd, String bpid, Block b) @@ -57,7 +66,8 @@ public class FsDatasetTestUtil { public static boolean breakHardlinksIfNeeded(FsDatasetSpi<?> fsd, ExtendedBlock block) throws IOException { - final ReplicaInfo info = ((FsDatasetImpl)fsd).getReplicaInfo(block); + final LocalReplica info = + (LocalReplica) ((FsDatasetImpl)fsd).getReplicaInfo(block); return info.breakHardLinksIfNeeded(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c9862b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java index 4ba3d81..320af7b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils; import org.apache.hadoop.hdfs.server.datanode.ReplicaAlreadyExistsException; -import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface; +import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipeline; import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo; import org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; @@ -483,7 +483,7 @@ public class TestWriteToReplica { long newGenStamp = blocks[NON_EXISTENT].getGenerationStamp() * 10; blocks[NON_EXISTENT].setGenerationStamp(newGenStamp); try { - ReplicaInPipelineInterface replicaInfo = + ReplicaInPipeline replicaInfo = dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]).getReplica(); Assert.assertTrue(replicaInfo.getGenerationStamp() == newGenStamp); Assert.assertTrue( --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
