Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Feb 10 03:04:05 2012 @@ -1,4 +1,4 @@ -/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java:1227776-1241553 +/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java:1227776-1242605 /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164339,1164771,1166402,1167318,1167383,1170379,1170459,1171221,1171297,1172916,1173402,1176550,1176986,1177002,1177035,1177399,1177487,1177531,1177859,1177864,1178639,1179512,1182189,1182205,1182214,1182641,1183132,1189613,1189932,1189982,1190109,1190611,1195575,1195760,1196113,1196129,1197885,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204363,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1210208,1210319-1210320,1212004,1212021,1212062,1212073,1212084,1212615,1213537,1213586,1213592-1213593,1213619,1213954,1214046,1214114,1214499,1215358,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227861,1227964,1229347,1230398,1231569,1231572,123162 7,1231640,1231732,1233605,1234555,1235135,1235137,1235956,1236444,1236456,1239752,1240897,1240928 /hadoop/core/branches/branch-0.19/core/src/java:713112 /hadoop/core/trunk/src/core:776175-785643,785929-786278
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java?rev=1242642&r1=1242641&r2=1242642&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java Fri Feb 10 03:04:05 2012 @@ -43,6 +43,7 @@ public abstract class ChecksumFileSystem private static final byte[] CHECKSUM_VERSION = new byte[] {'c', 'r', 'c', 0}; private int bytesPerChecksum = 512; private boolean verifyChecksum = true; + private boolean writeChecksum = true; public static double getApproxChkSumLength(long size) { return ChecksumFSOutputSummer.CHKSUM_AS_FRACTION * size; @@ -67,6 +68,11 @@ public abstract class ChecksumFileSystem this.verifyChecksum = verifyChecksum; } + @Override + public void setWriteChecksum(boolean writeChecksum) { + this.writeChecksum = writeChecksum; + } + /** get the raw file system */ public FileSystem getRawFileSystem() { return fs; @@ -428,9 +434,20 @@ public abstract class ChecksumFileSystem throw new IOException("Mkdirs failed to create " + parent); } } - final FSDataOutputStream out = new FSDataOutputStream( - new ChecksumFSOutputSummer(this, f, overwrite, bufferSize, replication, - blockSize, progress), null); + final FSDataOutputStream out; + if (writeChecksum) { + out = new FSDataOutputStream( + new ChecksumFSOutputSummer(this, f, overwrite, bufferSize, replication, + blockSize, progress), null); + } else { + out = fs.create(f, permission, overwrite, bufferSize, replication, + blockSize, progress); + // remove the checksum file since we aren't writing one + Path checkFile = getChecksumFile(f); + if (fs.exists(checkFile)) { + fs.delete(checkFile, true); + } + } if (permission != null) { setPermission(f, permission); } Modified: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1242642&r1=1242641&r2=1242642&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Fri Feb 10 03:04:05 2012 @@ -1937,6 +1937,15 @@ public abstract class FileSystem extends } /** + * Set the write checksum flag. This is only applicable if the + * corresponding FileSystem supports checksum. By default doesn't do anything. + * @param writeChecksum + */ + public void setWriteChecksum(boolean writeChecksum) { + //doesn't do anything + } + + /** * Return a list of file status objects that corresponds to the list of paths * excluding those non-existent paths. * Modified: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1242642&r1=1242641&r2=1242642&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java Fri Feb 10 03:04:05 2012 @@ -360,6 +360,11 @@ public class FilterFileSystem extends Fi public void setVerifyChecksum(boolean verifyChecksum) { fs.setVerifyChecksum(verifyChecksum); } + + @Override + public void setWriteChecksum(boolean writeChecksum) { + fs.setVerifyChecksum(writeChecksum); + } @Override public Configuration getConf() { Modified: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1242642&r1=1242641&r2=1242642&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Fri Feb 10 03:04:05 2012 @@ -41,7 +41,9 @@ import org.apache.hadoop.io.IOUtils; */ abstract class CommandWithDestination extends FsCommand { protected PathData dst; - protected boolean overwrite = false; + private boolean overwrite = false; + private boolean verifyChecksum = true; + private boolean writeChecksum = true; /** * @@ -53,6 +55,14 @@ abstract class CommandWithDestination ex overwrite = flag; } + protected void setVerifyChecksum(boolean flag) { + verifyChecksum = flag; + } + + protected void setWriteChecksum(boolean flag) { + writeChecksum = flag; + } + /** * The last arg is expected to be a local path, if only one argument is * given then the destination will be the current directory @@ -201,6 +211,7 @@ abstract class CommandWithDestination ex * @throws IOException if copy fails */ protected void copyFileToTarget(PathData src, PathData target) throws IOException { + src.fs.setVerifyChecksum(verifyChecksum); copyStreamToTarget(src.fs.open(src.path), target); } @@ -217,6 +228,7 @@ abstract class CommandWithDestination ex if (target.exists && (target.stat.isDirectory() || !overwrite)) { throw new PathExistsException(target.toString()); } + target.fs.setWriteChecksum(writeChecksum); PathData tempFile = null; try { tempFile = target.createTempFile(target+"._COPYING_"); Modified: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java?rev=1242642&r1=1242641&r2=1242642&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java Fri Feb 10 03:04:05 2012 @@ -25,7 +25,6 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.fs.ChecksumFileSystem; import org.apache.hadoop.fs.FileUtil; /** Various commands for copy files */ @@ -103,43 +102,17 @@ class CopyCommands { "to the local name. <src> is kept. When copying multiple,\n" + "files, the destination must be a directory."; - /** - * The prefix for the tmp file used in copyToLocal. - * It must be at least three characters long, required by - * {@link java.io.File#createTempFile(String, String, File)}. - */ - private boolean copyCrc; - private boolean verifyChecksum; - @Override protected void processOptions(LinkedList<String> args) throws IOException { CommandFormat cf = new CommandFormat( 1, Integer.MAX_VALUE, "crc", "ignoreCrc"); cf.parse(args); - copyCrc = cf.getOpt("crc"); - verifyChecksum = !cf.getOpt("ignoreCrc"); - + setWriteChecksum(cf.getOpt("crc")); + setVerifyChecksum(!cf.getOpt("ignoreCrc")); setRecursive(true); getLocalDestination(args); } - - @Override - protected void copyFileToTarget(PathData src, PathData target) - throws IOException { - src.fs.setVerifyChecksum(verifyChecksum); - - if (copyCrc && !(src.fs instanceof ChecksumFileSystem)) { - displayWarning(src.fs + ": Does not support checksums"); - copyCrc = false; - } - - super.copyFileToTarget(src, target); - if (copyCrc) { - // should we delete real file if crc copy fails? - super.copyFileToTarget(src.getChecksumFile(), target.getChecksumFile()); - } - } } /** Modified: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java?rev=1242642&r1=1242641&r2=1242642&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java Fri Feb 10 03:04:05 2012 @@ -27,7 +27,6 @@ import java.net.URISyntaxException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.ChecksumFileSystem; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; @@ -170,19 +169,6 @@ public class PathData { } /** - * Return the corresponding crc data for a file. Avoids exposing the fs - * contortions to the caller. - * @return PathData of the crc file - * @throws IOException is anything goes wrong - */ - public PathData getChecksumFile() throws IOException { - checkIfExists(FileTypeRequirement.SHOULD_NOT_BE_DIRECTORY); - ChecksumFileSystem srcFs = (ChecksumFileSystem)fs; - Path srcPath = srcFs.getChecksumFile(path); - return new PathData(srcFs.getRawFileSystem(), srcPath.toString()); - } - - /** * Returns a temporary file for this PathData with the given extension. * The file will be deleted on exit. * @param extension for the temporary file Modified: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java?rev=1242642&r1=1242641&r2=1242642&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java Fri Feb 10 03:04:05 2012 @@ -470,6 +470,15 @@ public class ViewFileSystem extends File } } + @Override + public void setWriteChecksum(final boolean writeChecksum) { + List<InodeTree.MountPoint<FileSystem>> mountPoints = + fsState.getMountPoints(); + for (InodeTree.MountPoint<FileSystem> mount : mountPoints) { + mount.target.targetFileSystem.setWriteChecksum(writeChecksum); + } + } + public MountPoint[] getMountPoints() { List<InodeTree.MountPoint<FileSystem>> mountPoints = fsState.getMountPoints(); Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/test/core/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Feb 10 03:04:05 2012 @@ -1,4 +1,4 @@ -/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/core:1227776-1241553 +/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/core:1227776-1242605 /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164339,1166009,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1183132,1189613,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456,1239752,1240897,1240928 /hadoop/core/branches/branch-0.19/core/src/test/core:713112 /hadoop/core/trunk/src/test/core:776175-785643,785929-786278 Modified: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/pom.xml?rev=1242642&r1=1242641&r2=1242642&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23-PB/hadoop-common-project/pom.xml (original) +++ hadoop/common/branches/branch-0.23-PB/hadoop-common-project/pom.xml Fri Feb 10 03:04:05 2012 @@ -17,12 +17,12 @@ <parent> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-project</artifactId> - <version>0.23.1-SNAPSHOT</version> + <version>0.23.2-SNAPSHOT</version> <relativePath>../hadoop-project</relativePath> </parent> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common-project</artifactId> - <version>0.23.1-SNAPSHOT</version> + <version>0.23.2-SNAPSHOT</version> <description>Apache Hadoop Common Project</description> <name>Apache Hadoop Common Project</name> <packaging>pom</packaging>
