HDFS-13509. Bug fix for breakHardlinks() of ReplicaInfo/LocalReplica, and fix TestFileAppend failures on Windows. Contributed by Xiao Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0c788aa Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0c788aa Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0c788aa Branch: refs/heads/YARN-8200 Commit: c0c788aafc892373b23ae0c7905d913fd788c3a6 Parents: c844181 Author: Inigo Goiri <inigo...@apache.org> Authored: Sat Apr 28 09:07:56 2018 -0700 Committer: Inigo Goiri <inigo...@apache.org> Committed: Sat Apr 28 09:07:56 2018 -0700 ---------------------------------------------------------------------- .../hdfs/server/datanode/ReplicaInfo.java | 21 ++++--- .../org/apache/hadoop/hdfs/TestFileAppend.java | 61 +++++++++++++------- 2 files changed, 53 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0c788aa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java index 9817f97..f3f6db1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java @@ -240,19 +240,22 @@ abstract public class ReplicaInfo extends Block final FileIoProvider fileIoProvider = getFileIoProvider(); final File tmpFile = DatanodeUtil.createFileWithExistsCheck( getVolume(), b, DatanodeUtil.getUnlinkTmpFile(file), fileIoProvider); - try (FileInputStream in = fileIoProvider.getFileInputStream( + try { + try (FileInputStream in = fileIoProvider.getFileInputStream( getVolume(), file)) { - try (FileOutputStream out = fileIoProvider.getFileOutputStream( - getVolume(), tmpFile)) { - IOUtils.copyBytes(in, out, 16 * 1024); - } - if (file.length() != tmpFile.length()) { - throw new IOException("Copy of file " + file + " size " + file.length()+ - " into file " + tmpFile + - " resulted in a size of " + tmpFile.length()); + try (FileOutputStream out = fileIoProvider.getFileOutputStream( + getVolume(), tmpFile)) { + IOUtils.copyBytes(in, out, 16 * 1024); + } + if (file.length() != tmpFile.length()) { + throw new IOException("Copy of file " + file + " size " + + file.length() + " into file " + tmpFile + + " resulted in a size of " + tmpFile.length()); + } } fileIoProvider.replaceFile(getVolume(), tmpFile, file); } catch (IOException e) { + DataNode.LOG.error("Cannot breakHardlinks for file " + file, e); if (!fileIoProvider.delete(getVolume(), tmpFile)) { DataNode.LOG.info("detachFile failed to delete temporary file " + tmpFile); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0c788aa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java index fbf09fb..59cc31a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetUtil; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Time; import org.junit.Assert; @@ -120,7 +121,9 @@ public class TestFileAppend{ @Test public void testBreakHardlinksIfNeeded() throws IOException { Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .build(); FileSystem fs = cluster.getFileSystem(); InetSocketAddress addr = new InetSocketAddress("localhost", cluster.getNameNodePort()); @@ -186,7 +189,9 @@ public class TestFileAppend{ public void testSimpleFlush() throws IOException { Configuration conf = new HdfsConfiguration(); fileContents = AppendTestUtil.initBuffer(AppendTestUtil.FILE_SIZE); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .build(); DistributedFileSystem fs = cluster.getFileSystem(); try { @@ -239,7 +244,9 @@ public class TestFileAppend{ public void testComplexFlush() throws IOException { Configuration conf = new HdfsConfiguration(); fileContents = AppendTestUtil.initBuffer(AppendTestUtil.FILE_SIZE); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .build(); DistributedFileSystem fs = cluster.getFileSystem(); try { @@ -286,7 +293,9 @@ public class TestFileAppend{ @Test(expected = FileNotFoundException.class) public void testFileNotFound() throws IOException { Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .build(); FileSystem fs = cluster.getFileSystem(); try { Path file1 = new Path("/nonexistingfile.dat"); @@ -301,7 +310,9 @@ public class TestFileAppend{ @Test public void testAppendTwice() throws Exception { Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .build(); final FileSystem fs1 = cluster.getFileSystem(); final FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(conf); try { @@ -340,7 +351,9 @@ public class TestFileAppend{ @Test public void testAppend2Twice() throws Exception { Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .build(); final DistributedFileSystem fs1 = cluster.getFileSystem(); final FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(conf); try { @@ -386,8 +399,9 @@ public class TestFileAppend{ HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.ENABLE_KEY, false); - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(4).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, + builderBaseDir).numDataNodes(4).build(); final DistributedFileSystem fs = cluster.getFileSystem(); try { final Path p = new Path("/testMultipleAppend/foo"); @@ -439,8 +453,9 @@ public class TestFileAppend{ final long softLimit = 1L; final long hardLimit = 9999999L; - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) - .build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(1).build(); cluster.setLeasePeriod(softLimit, hardLimit); cluster.waitActive(); @@ -479,8 +494,9 @@ public class TestFileAppend{ final long softLimit = 1L; final long hardLimit = 9999999L; - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) - .build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(1).build(); cluster.setLeasePeriod(softLimit, hardLimit); cluster.waitActive(); @@ -526,8 +542,9 @@ public class TestFileAppend{ Configuration conf = new HdfsConfiguration(); conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "false"); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3) - .build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(3).build(); DistributedFileSystem fs = null; try { fs = cluster.getFileSystem(); @@ -541,7 +558,7 @@ public class TestFileAppend{ String dnAddress = dnProp.datanode.getXferAddress().toString(); if (dnAddress.startsWith("/")) { dnAddress = dnAddress.substring(1); -} + } // append again to bump genstamps for (int i = 0; i < 2; i++) { @@ -579,8 +596,9 @@ public class TestFileAppend{ Configuration conf = new HdfsConfiguration(); conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "false"); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3) - .build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(3).build(); DistributedFileSystem fs = null; final String hello = "hello\n"; try { @@ -651,8 +669,9 @@ public class TestFileAppend{ conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); conf.setInt("dfs.min.replication", 1); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) - .build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(1).build(); try { DistributedFileSystem fs = cluster.getFileSystem(); Path fileName = new Path("/appendCorruptBlock"); @@ -677,7 +696,9 @@ public class TestFileAppend{ conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); conf.setInt("dfs.min.replication", 1); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .build(); try { cluster.waitActive(); DataNode dn = cluster.getDataNodes().get(0); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org