Author: atm Date: Tue Feb 11 02:47:05 2014 New Revision: 1566965 URL: http://svn.apache.org/r1566965 Log: HADOOP-10326. M/R jobs can not access S3 if Kerberos is enabled. Contributed by bc Wong.
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1566965&r1=1566964&r2=1566965&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Feb 11 02:47:05 2014 @@ -331,6 +331,9 @@ Release 2.4.0 - UNRELEASED HADOOP-10330. TestFrameDecoder fails if it cannot bind port 12345. (Arpit Agarwal) + HADOOP-10326. M/R jobs can not access S3 if Kerberos is enabled. (bc Wong + via atm) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java?rev=1566965&r1=1566964&r2=1566965&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java Tue Feb 11 02:47:05 2014 @@ -443,6 +443,12 @@ public class S3FileSystem extends FileSy return getConf().getLong("fs.s3.block.size", 64 * 1024 * 1024); } + @Override + public String getCanonicalServiceName() { + // Does not support Token + return null; + } + // diagnostic methods void dump() throws IOException { Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java?rev=1566965&r1=1566964&r2=1566965&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java Tue Feb 11 02:47:05 2014 @@ -733,4 +733,10 @@ public class NativeS3FileSystem extends public Path getWorkingDirectory() { return workingDir; } + + @Override + public String getCanonicalServiceName() { + // Does not support Token + return null; + } } Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java?rev=1566965&r1=1566964&r2=1566965&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java Tue Feb 11 02:47:05 2014 @@ -54,5 +54,10 @@ public abstract class S3FileSystemContra assertEquals("Double default block size", newBlockSize, fs.getFileStatus(file).getBlockSize()); } - + + public void testCanonicalName() throws Exception { + assertNull("s3 doesn't support security token and shouldn't have canonical name", + fs.getCanonicalServiceName()); + } + } Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java?rev=1566965&r1=1566964&r2=1566965&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java Tue Feb 11 02:47:05 2014 @@ -48,7 +48,12 @@ public abstract class NativeS3FileSystem store.purge("test"); super.tearDown(); } - + + public void testCanonicalName() throws Exception { + assertNull("s3n doesn't support security token and shouldn't have canonical name", + fs.getCanonicalServiceName()); + } + public void testListStatusForRoot() throws Exception { FileStatus[] paths = fs.listStatus(path("/")); assertEquals("Root directory is not empty; ", 0, paths.length); @@ -60,7 +65,7 @@ public abstract class NativeS3FileSystem assertEquals(1, paths.length); assertEquals(path("/test"), paths[0].getPath()); } - + public void testNoTrailingBackslashOnBucket() throws Exception { assertTrue(fs.getFileStatus(new Path(fs.getUri().toString())).isDirectory()); }