Author: tgraves Date: Wed Aug 1 14:03:01 2012 New Revision: 1368003 URL: http://svn.apache.org/viewvc?rev=1368003&view=rev Log: merge -r 1368001:1368002 from trunk. FIXES: HADOOP-8633
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java - copied unchanged from r1368002, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1368003&r1=1368002&r2=1368003&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Aug 1 14:03:01 2012 @@ -692,6 +692,9 @@ Release 0.23.3 - UNRELEASED org.apache.hadoop.classification.InterfaceAudience not found (Trevor Robinson via tgraves) + HADOOP-8633. Interrupted FsShell copies may leave tmp files (Daryn Sharp + via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1368003&r1=1368002&r2=1368003&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Wed Aug 1 14:03:01 2012 @@ -24,6 +24,8 @@ import java.io.InputStream; import java.util.LinkedList; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException; import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; @@ -232,31 +234,65 @@ abstract class CommandWithDestination ex if (target.exists && (target.stat.isDirectory() || !overwrite)) { throw new PathExistsException(target.toString()); } - target.fs.setWriteChecksum(writeChecksum); - PathData tempFile = null; + TargetFileSystem targetFs = new TargetFileSystem(target.fs); try { - tempFile = target.createTempFile(target+"._COPYING_"); - FSDataOutputStream out = target.fs.create(tempFile.path, true); - IOUtils.copyBytes(in, out, getConf(), true); + PathData tempTarget = target.suffix("._COPYING_"); + targetFs.setWriteChecksum(writeChecksum); + targetFs.writeStreamToFile(in, tempTarget); + targetFs.rename(tempTarget, target); + } finally { + targetFs.close(); // last ditch effort to ensure temp file is removed + } + } + + // Helper filter filesystem that registers created files as temp files to + // be deleted on exit unless successfully renamed + private static class TargetFileSystem extends FilterFileSystem { + TargetFileSystem(FileSystem fs) { + super(fs); + } + + void writeStreamToFile(InputStream in, PathData target) throws IOException { + FSDataOutputStream out = null; + try { + out = create(target); + IOUtils.copyBytes(in, out, getConf(), true); + } finally { + IOUtils.closeStream(out); // just in case copyBytes didn't + } + } + + // tag created files as temp files + FSDataOutputStream create(PathData item) throws IOException { + try { + return create(item.path, true); + } finally { // might have been created but stream was interrupted + deleteOnExit(item.path); + } + } + + void rename(PathData src, PathData target) throws IOException { // the rename method with an option to delete the target is deprecated - if (target.exists && !target.fs.delete(target.path, false)) { + if (target.exists && !delete(target.path, false)) { // too bad we don't know why it failed PathIOException e = new PathIOException(target.toString()); e.setOperation("delete"); throw e; } - if (!tempFile.fs.rename(tempFile.path, target.path)) { + if (!rename(src.path, target.path)) { // too bad we don't know why it failed - PathIOException e = new PathIOException(tempFile.toString()); + PathIOException e = new PathIOException(src.toString()); e.setOperation("rename"); e.setTargetPath(target.toString()); throw e; } - tempFile = null; - } finally { - if (tempFile != null) { - tempFile.fs.delete(tempFile.path, false); - } + // cancel delete on exit if rename is successful + cancelDeleteOnExit(src.path); + } + @Override + public void close() { + // purge any remaining temp files, but don't close underlying fs + processDeleteOnExit(); } } } \ No newline at end of file Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java?rev=1368003&r1=1368002&r2=1368003&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java Wed Aug 1 14:03:01 2012 @@ -60,7 +60,7 @@ public class PathData implements Compara * @throws IOException if anything goes wrong... */ public PathData(String pathString, Configuration conf) throws IOException { - this(FileSystem.get(URI.create(pathString), conf), pathString); + this(FileSystem.get(stringToUri(pathString), conf), pathString); } /** @@ -170,16 +170,13 @@ public class PathData implements Compara } /** - * Returns a temporary file for this PathData with the given extension. - * The file will be deleted on exit. - * @param extension for the temporary file + * Returns a new PathData with the given extension. + * @param extension for the suffix * @return PathData * @throws IOException shouldn't happen */ - public PathData createTempFile(String extension) throws IOException { - PathData tmpFile = new PathData(fs, uri+"._COPYING_"); - fs.deleteOnExit(tmpFile.path); - return tmpFile; + public PathData suffix(String extension) throws IOException { + return new PathData(fs, this+extension); } /**