Author: daryn Date: Thu Dec 5 15:51:28 2013 New Revision: 1548176 URL: http://svn.apache.org/r1548176 Log: svn merge -c 1548175 FIXES: HADOOP-10129. Distcp may succeed when it fails (daryn)
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java - copied unchanged from r1548175, hadoop/common/trunk/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/pom.xml hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/pom.xml?rev=1548176&r1=1548175&r2=1548176&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/pom.xml (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/pom.xml Thu Dec 5 15:51:28 2013 @@ -95,6 +95,11 @@ <scope>test</scope> <type>test-jar</type> </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java?rev=1548176&r1=1548175&r2=1548176&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java Thu Dec 5 15:51:28 2013 @@ -32,6 +32,8 @@ import org.apache.hadoop.tools.util.Dist import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.security.Credentials; +import com.google.common.annotations.VisibleForTesting; + import java.io.*; import java.util.Stack; @@ -107,12 +109,13 @@ public class SimpleCopyListing extends C /** {@inheritDoc} */ @Override public void doBuildListing(Path pathToListingFile, DistCpOptions options) throws IOException { - - SequenceFile.Writer fileListWriter = null; - + doBuildListing(getWriter(pathToListingFile), options); + } + + @VisibleForTesting + public void doBuildListing(SequenceFile.Writer fileListWriter, + DistCpOptions options) throws IOException { try { - fileListWriter = getWriter(pathToListingFile); - for (Path path: options.getSourcePaths()) { FileSystem sourceFS = path.getFileSystem(getConf()); path = makeQualified(path); @@ -140,8 +143,10 @@ public class SimpleCopyListing extends C writeToFileListing(fileListWriter, rootStatus, sourcePathRoot, localFile); } } + fileListWriter.close(); + fileListWriter = null; } finally { - IOUtils.closeStream(fileListWriter); + IOUtils.cleanup(LOG, fileListWriter); } } Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java?rev=1548176&r1=1548175&r2=1548176&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java Thu Dec 5 15:51:28 2013 @@ -30,6 +30,8 @@ import org.apache.hadoop.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import com.google.common.annotations.VisibleForTesting; + import java.io.*; import java.util.EnumSet; @@ -176,7 +178,8 @@ public class RetriableFileCopyCommand ex return new Path(root, ".distcp.tmp." + context.getTaskAttemptID().toString()); } - private long copyBytes(FileStatus sourceFileStatus, OutputStream outStream, + @VisibleForTesting + long copyBytes(FileStatus sourceFileStatus, OutputStream outStream, int bufferSize, Mapper.Context context) throws IOException { Path source = sourceFileStatus.getPath(); @@ -193,6 +196,8 @@ public class RetriableFileCopyCommand ex updateContextStatus(totalBytesRead, context, sourceFileStatus); bytesRead = inStream.read(buf); } + outStream.close(); + outStream = null; } finally { IOUtils.cleanup(LOG, outStream, inStream); } Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java?rev=1548176&r1=1548175&r2=1548176&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java Thu Dec 5 15:51:28 2013 @@ -18,6 +18,8 @@ package org.apache.hadoop.tools; +import static org.mockito.Mockito.*; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; @@ -35,6 +37,7 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.AfterClass; +import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.util.List; @@ -249,4 +252,29 @@ public class TestCopyListing extends Sim IOUtils.closeStream(reader); } } + + @Test + public void testFailOnCloseError() throws IOException { + File inFile = File.createTempFile("TestCopyListingIn", null); + inFile.deleteOnExit(); + File outFile = File.createTempFile("TestCopyListingOut", null); + outFile.deleteOnExit(); + List<Path> srcs = new ArrayList<Path>(); + srcs.add(new Path(inFile.toURI())); + + Exception expectedEx = new IOException("boom"); + SequenceFile.Writer writer = mock(SequenceFile.Writer.class); + doThrow(expectedEx).when(writer).close(); + + SimpleCopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS); + DistCpOptions options = new DistCpOptions(srcs, new Path(outFile.toURI())); + Exception actualEx = null; + try { + listing.doBuildListing(writer, options); + } catch (Exception e) { + actualEx = e; + } + Assert.assertNotNull("close writer didn't fail", actualEx); + Assert.assertEquals(expectedEx, actualEx); + } }