svn commit: r1392368 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/bin/hadoop
Author: bobby Date: Mon Oct 1 14:53:50 2012 New Revision: 1392368 URL: http://svn.apache.org/viewvc?rev=1392368view=rev Log: svn merge -c 1391780 FIXES: HADOOP-8386. hadoop script doesn't work if 'cd' prints to stdout (default behavior in Ubuntu). Contributed by Christopher Berner and Andy Isaacson. (harsh) Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/bin/hadoop Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392368r1=1392367r2=1392368view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 14:53:50 2012 @@ -12,6 +12,10 @@ Release 0.23.5 - UNRELEASED BUG FIXES +HADOOP-8386. hadoop script doesn't work if 'cd' prints to stdout +(default behavior in some bash setups (esp. Ubuntu)) +(Chiristopher Berner and Andy Isaacson via harsh) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/bin/hadoop URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/bin/hadoop?rev=1392368r1=1392367r2=1392368view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/bin/hadoop (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/bin/hadoop Mon Oct 1 14:53:50 2012 @@ -19,7 +19,7 @@ bin=`which $0` bin=`dirname ${bin}` -bin=`cd $bin; pwd` +bin=`cd $bin /dev/null; pwd` DEFAULT_LIBEXEC_DIR=$bin/../libexec HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
svn commit: r1392437 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/docs/src/documentation/content/xdocs/file_system_shell.xml
Author: bobby Date: Mon Oct 1 16:47:09 2012 New Revision: 1392437 URL: http://svn.apache.org/viewvc?rev=1392437view=rev Log: svn merge -c 1390109 FIXES: HADOOP-8791. Fix rm command documentation to indicte it deletes files and not directories. Contributed by Jing Zhao. Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/file_system_shell.xml Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392437r1=1392436r2=1392437view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 16:47:09 2012 @@ -16,6 +16,9 @@ Release 0.23.5 - UNRELEASED (default behavior in some bash setups (esp. Ubuntu)) (Chiristopher Berner and Andy Isaacson via harsh) +HADOOP-8791. Fix rm command documentation to indicte it deletes +files and not directories. (Jing Zhao via suresh) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/file_system_shell.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/file_system_shell.xml?rev=1392437r1=1392436r2=1392437view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/file_system_shell.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/file_system_shell.xml Mon Oct 1 16:47:09 2012 @@ -418,15 +418,15 @@ codeUsage: hdfs dfs -rm [-skipTrash] URI [URI #x2026;] /code /p p - Delete files specified as args. Only deletes non empty directory and files. If the code-skipTrash/code option + Delete files specified as args. Only deletes files. If the code-skipTrash/code option is specified, the trash, if enabled, will be bypassed and the specified file(s) deleted immediately. This can be useful when it is necessary to delete files from an over-quota directory. - Refer to rmr for recursive deletes.br/ + Use -rm -r or rmr for recursive deletes.br/ Example: /p ul li - code hdfs dfs -rm hdfs://nn.example.com/file /user/hadoop/emptydir /code + code hdfs dfs -rm hdfs://nn.example.com/file /code /li /ul pExit Code:/p @@ -442,7 +442,7 @@ p codeUsage: hdfs dfs -rmr [-skipTrash] URI [URI #x2026;]/code /p - pRecursive version of delete. If the code-skipTrash/code option + pRecursive version of delete. The rmr command recursively deletes the directory and any content under it. If the code-skipTrash/code option is specified, the trash, if enabled, will be bypassed and the specified file(s) deleted immediately. This can be useful when it is necessary to delete files from an over-quota directory.br/ Example:
svn commit: r1392458 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java src/test/java/org/apa
Author: bobby Date: Mon Oct 1 17:34:49 2012 New Revision: 1392458 URL: http://svn.apache.org/viewvc?rev=1392458view=rev Log: svn merge -c 1384627 FIXES: HADOOP-8755. Print thread dump when tests fail due to timeout. Contributed by Andrey Klochkov. Added: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java - copied unchanged from r1384627, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TimedOutTestsListener.java - copied unchanged from r1384627, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TimedOutTestsListener.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392458r1=1392457r2=1392458view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 17:34:49 2012 @@ -8,6 +8,9 @@ Release 0.23.5 - UNRELEASED IMPROVEMENTS +HADOOP-8755. Print thread dump when tests fail due to timeout. (Andrey +Klochkov via atm) + OPTIMIZATIONS BUG FIXES
svn commit: r1392465 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/ftp/ src/main/java/org/apache/hadoop/fs/s3/ src/main/java/org/a
Author: bobby Date: Mon Oct 1 17:44:15 2012 New Revision: 1392465 URL: http://svn.apache.org/viewvc?rev=1392465view=rev Log: svn merge -c 1386451 FIXES: HADOOP-8819. Incorrectly is used instead of in some file system implementations. Contributed by Brandon Li. Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392465r1=1392464r2=1392465view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 17:44:15 2012 @@ -13,6 +13,9 @@ Release 0.23.5 - UNRELEASED OPTIMIZATIONS +HADOOP-8819. Incorrectly is used instead of in some file system +implementations. (Brandon Li via suresh) + BUG FIXES HADOOP-8386. hadoop script doesn't work if 'cd' prints to stdout Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java?rev=1392465r1=1392464r2=1392465view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java Mon Oct 1 17:44:15 2012 @@ -476,7 +476,7 @@ public class FTPFileSystem extends FileS if (created) { String parentDir = parent.toUri().getPath(); client.changeWorkingDirectory(parentDir); -created = created client.makeDirectory(pathName); +created = created client.makeDirectory(pathName); } } else if (isFile(client, absolute)) { throw new IOException(String.format( Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java?rev=1392465r1=1392464r2=1392465view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java Mon Oct 1 17:44:15 2012 @@ -73,7 +73,7 @@ public class FTPInputStream extends FSIn if (byteRead = 0) { pos++; } -if (stats != null byteRead = 0) { +if (stats != null byteRead = 0) { stats.incrementBytesRead(1); } return byteRead; @@ -88,7 +88,7 @@ public class FTPInputStream extends FSIn if (result 0) { pos += result; } -if (stats != null result 0) { +if (stats != null result 0) { stats.incrementBytesRead(result); } Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java?rev=1392465r1=1392464r2=1392465view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java Mon Oct 1 17:44:15 2012 @@ -113,7 +113,7 @@ class S3InputStream extends FSInputStrea
svn commit: r1392466 - /hadoop/common/trunk/hadoop-project/pom.xml
Author: atm Date: Mon Oct 1 17:44:23 2012 New Revision: 1392466 URL: http://svn.apache.org/viewvc?rev=1392466view=rev Log: HADOOP-8851. Use -XX:+HeapDumpOnOutOfMemoryError JVM option in the forked tests. Contributed by Ivan A. Veselovsky. Modified: hadoop/common/trunk/hadoop-project/pom.xml Modified: hadoop/common/trunk/hadoop-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-project/pom.xml?rev=1392466r1=1392465r2=1392466view=diff == --- hadoop/common/trunk/hadoop-project/pom.xml (original) +++ hadoop/common/trunk/hadoop-project/pom.xml Mon Oct 1 17:44:23 2012 @@ -834,7 +834,7 @@ configuration forkModealways/forkMode forkedProcessTimeoutInSeconds900/forkedProcessTimeoutInSeconds - argLine-Xmx1024m/argLine + argLine-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError/argLine environmentVariables LD_LIBRARY_PATH${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib/LD_LIBRARY_PATH MALLOC_ARENA_MAX4/MALLOC_ARENA_MAX
svn commit: r1392466 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: atm Date: Mon Oct 1 17:44:23 2012 New Revision: 1392466 URL: http://svn.apache.org/viewvc?rev=1392466view=rev Log: HADOOP-8851. Use -XX:+HeapDumpOnOutOfMemoryError JVM option in the forked tests. Contributed by Ivan A. Veselovsky. Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392466r1=1392465r2=1392466view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 17:44:23 2012 @@ -285,6 +285,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-8736. Add Builder for building RPC server. (Brandon Li via Suresh) +HADOOP-8851. Use -XX:+HeapDumpOnOutOfMemoryError JVM option in the forked +tests. (Ivan A. Veselovsky via atm) + OPTIMIZATIONS HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
svn commit: r1392469 - /hadoop/common/branches/branch-2/hadoop-project/pom.xml
Author: atm Date: Mon Oct 1 17:45:29 2012 New Revision: 1392469 URL: http://svn.apache.org/viewvc?rev=1392469view=rev Log: HADOOP-8851. Use -XX:+HeapDumpOnOutOfMemoryError JVM option in the forked tests. Contributed by Ivan A. Veselovsky. Modified: hadoop/common/branches/branch-2/hadoop-project/pom.xml Modified: hadoop/common/branches/branch-2/hadoop-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-project/pom.xml?rev=1392469r1=1392468r2=1392469view=diff == --- hadoop/common/branches/branch-2/hadoop-project/pom.xml (original) +++ hadoop/common/branches/branch-2/hadoop-project/pom.xml Mon Oct 1 17:45:29 2012 @@ -874,7 +874,7 @@ configuration forkModealways/forkMode forkedProcessTimeoutInSeconds900/forkedProcessTimeoutInSeconds - argLine-Xmx1024m/argLine + argLine-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError/argLine environmentVariables LD_LIBRARY_PATH${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib/LD_LIBRARY_PATH MALLOC_ARENA_MAX4/MALLOC_ARENA_MAX
svn commit: r1392469 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Author: atm Date: Mon Oct 1 17:45:29 2012 New Revision: 1392469 URL: http://svn.apache.org/viewvc?rev=1392469view=rev Log: HADOOP-8851. Use -XX:+HeapDumpOnOutOfMemoryError JVM option in the forked tests. Contributed by Ivan A. Veselovsky. Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392469r1=1392468r2=1392469view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 17:45:29 2012 @@ -25,6 +25,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-3957. Change MutableQuantiles to use a shared thread for rolling over metrics. (Andrew Wang via todd) +HADOOP-8851. Use -XX:+HeapDumpOnOutOfMemoryError JVM option in the forked +tests. (Ivan A. Veselovsky via atm) + OPTIMIZATIONS HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
svn commit: r1392492 - in /hadoop/common/branches/branch-0.23/hadoop-tools: hadoop-archives/src/test/java/org/apache/hadoop/tools/ hadoop-extras/src/test/java/org/apache/hadoop/tools/
Author: bobby Date: Mon Oct 1 18:15:44 2012 New Revision: 1392492 URL: http://svn.apache.org/viewvc?rev=1392492view=rev Log: svn merge -c 1383494 FIXES: HADOOP-8789. Tests setLevel(Level.OFF) should be Level.ERROR. Contributed by Andy Isaacson Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java?rev=1392492r1=1392491r2=1392492view=diff == --- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java Mon Oct 1 18:15:44 2012 @@ -52,11 +52,11 @@ public class TestHadoopArchives extends { ((Log4JLogger)LogFactory.getLog(org.apache.hadoop.security.Groups.class) -).getLogger().setLevel(Level.OFF); +).getLogger().setLevel(Level.ERROR); ((Log4JLogger)org.apache.hadoop.ipc.Server.LOG -).getLogger().setLevel(Level.OFF); +).getLogger().setLevel(Level.ERROR); ((Log4JLogger)org.apache.hadoop.util.AsyncDiskService.LOG -).getLogger().setLevel(Level.OFF); +).getLogger().setLevel(Level.ERROR); } private static final String inputDir = input; Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java?rev=1392492r1=1392491r2=1392492view=diff == --- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java Mon Oct 1 18:15:44 2012 @@ -61,9 +61,9 @@ import org.junit.Ignore; public class TestCopyFiles extends TestCase { { ((Log4JLogger)LogFactory.getLog(org.apache.hadoop.hdfs.StateChange) -).getLogger().setLevel(Level.OFF); -((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.OFF); - ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.OFF); +).getLogger().setLevel(Level.ERROR); +((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ERROR); + ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ERROR); ((Log4JLogger)DistCpV1.LOG).getLogger().setLevel(Level.ALL); } Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java?rev=1392492r1=1392491r2=1392492view=diff == --- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java Mon Oct 1 18:15:44 2012 @@ -46,9 +46,9 @@ import org.junit.Ignore; public class TestDistCh extends junit.framework.TestCase { { ((Log4JLogger)LogFactory.getLog(org.apache.hadoop.hdfs.StateChange) -).getLogger().setLevel(Level.OFF); -((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.OFF); - ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.OFF); +).getLogger().setLevel(Level.ERROR); +((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ERROR); + ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ERROR); } static final Long RANDOM_NUMBER_GENERATOR_SEED = null;
svn commit: r1392519 - in /hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src: main/java/org/apache/hadoop/tools/OptionsParser.java test/java/org/apache/hadoop/tools/TestOptionsParser.j
Author: bobby Date: Mon Oct 1 18:59:55 2012 New Revision: 1392519 URL: http://svn.apache.org/viewvc?rev=1392519view=rev Log: svn merge -c 1382119 FIXES: HADOOP-8775. MR2 distcp permits non-positive value to -bandwidth option which causes job never to complete. Contributed by Sandy Ryza. Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java?rev=1392519r1=1392518r2=1392519view=diff == --- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java Mon Oct 1 18:59:55 2012 @@ -156,6 +156,10 @@ public class OptionsParser { try { Integer mapBandwidth = Integer.parseInt( getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()).trim()); +if (mapBandwidth.intValue() = 0) { + throw new IllegalArgumentException(Bandwidth specified is not positive: + + mapBandwidth); +} option.setMapBandwidth(mapBandwidth); } catch (NumberFormatException e) { throw new IllegalArgumentException(Bandwidth specified is invalid: + Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java?rev=1392519r1=1392518r2=1392519view=diff == --- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java Mon Oct 1 18:59:55 2012 @@ -110,6 +110,24 @@ public class TestOptionsParser { hdfs://localhost:8020/target/}); Assert.assertEquals(options.getMapBandwidth(), 11); } + + @Test(expected=IllegalArgumentException.class) + public void testParseNonPositiveBandwidth() { +OptionsParser.parse(new String[] { +-bandwidth, +-11, +hdfs://localhost:8020/source/first, +hdfs://localhost:8020/target/}); + } + + @Test(expected=IllegalArgumentException.class) + public void testParseZeroBandwidth() { +OptionsParser.parse(new String[] { +-bandwidth, +0, +hdfs://localhost:8020/source/first, +hdfs://localhost:8020/target/}); + } @Test public void testParseSkipCRC() {
svn commit: r1392521 - /hadoop/common/branches/branch-0.23/hadoop-project/pom.xml
Author: bobby Date: Mon Oct 1 19:03:27 2012 New Revision: 1392521 URL: http://svn.apache.org/viewvc?rev=1392521view=rev Log: svn merge -c 1392466 FIXES: HADOOP-8851. Use -XX:+HeapDumpOnOutOfMemoryError JVM option in the forked tests. Contributed by Ivan A. Veselovsky. Modified: hadoop/common/branches/branch-0.23/hadoop-project/pom.xml Modified: hadoop/common/branches/branch-0.23/hadoop-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-project/pom.xml?rev=1392521r1=1392520r2=1392521view=diff == --- hadoop/common/branches/branch-0.23/hadoop-project/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-project/pom.xml Mon Oct 1 19:03:27 2012 @@ -855,7 +855,7 @@ configuration forkModealways/forkMode forkedProcessTimeoutInSeconds900/forkedProcessTimeoutInSeconds - argLine-Xmx1024m/argLine + argLine-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError/argLine environmentVariables LD_LIBRARY_PATH${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib/LD_LIBRARY_PATH /environmentVariables
svn commit: r1392521 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Mon Oct 1 19:03:27 2012 New Revision: 1392521 URL: http://svn.apache.org/viewvc?rev=1392521view=rev Log: svn merge -c 1392466 FIXES: HADOOP-8851. Use -XX:+HeapDumpOnOutOfMemoryError JVM option in the forked tests. Contributed by Ivan A. Veselovsky. Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392521r1=1392520r2=1392521view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 19:03:27 2012 @@ -14,6 +14,9 @@ Release 0.23.5 - UNRELEASED HADOOP-8789. Tests setLevel(Level.OFF) should be Level.ERROR. (Andy Isaacson via eli) +HADOOP-8851. Use -XX:+HeapDumpOnOutOfMemoryError JVM option in the forked +tests. (Ivan A. Veselovsky via atm) + OPTIMIZATIONS HADOOP-8819. Incorrectly is used instead of in some file system
svn commit: r1392557 - in /hadoop/common/site/main: author/src/documentation/content/xdocs/ publish/
Author: jlowe Date: Mon Oct 1 19:56:35 2012 New Revision: 1392557 URL: http://svn.apache.org/viewvc?rev=1392557view=rev Log: Added jlowe to committer list Modified: hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml hadoop/common/site/main/publish/bylaws.html hadoop/common/site/main/publish/bylaws.pdf hadoop/common/site/main/publish/index.html hadoop/common/site/main/publish/index.pdf hadoop/common/site/main/publish/issue_tracking.html hadoop/common/site/main/publish/issue_tracking.pdf hadoop/common/site/main/publish/linkmap.html hadoop/common/site/main/publish/linkmap.pdf hadoop/common/site/main/publish/mailing_lists.html hadoop/common/site/main/publish/mailing_lists.pdf hadoop/common/site/main/publish/privacy_policy.html hadoop/common/site/main/publish/privacy_policy.pdf hadoop/common/site/main/publish/releases.html hadoop/common/site/main/publish/releases.pdf hadoop/common/site/main/publish/version_control.html hadoop/common/site/main/publish/version_control.pdf hadoop/common/site/main/publish/who.html hadoop/common/site/main/publish/who.pdf Modified: hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml URL: http://svn.apache.org/viewvc/hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml?rev=1392557r1=1392556r2=1392557view=diff == --- hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml (original) +++ hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml Mon Oct 1 19:56:35 2012 @@ -533,6 +533,14 @@ /tr tr + tdjlowe/td + tdJason Lowe/td + tdYahoo!/td + td/td + td-6/td + /tr + + tr tdjohan/td tda href=http://people.apache.org/~johan;Johan Oskarsson/a/td tdTwitter/td Modified: hadoop/common/site/main/publish/bylaws.html URL: http://svn.apache.org/viewvc/hadoop/common/site/main/publish/bylaws.html?rev=1392557r1=1392556r2=1392557view=diff == --- hadoop/common/site/main/publish/bylaws.html (original) +++ hadoop/common/site/main/publish/bylaws.html Mon Oct 1 19:56:35 2012 @@ -3,7 +3,7 @@ head META http-equiv=Content-Type content=text/html; charset=UTF-8 meta content=Apache Forrest name=Generator -meta name=Forrest-version content=0.8 +meta name=Forrest-version content=0.9 meta name=Forrest-skin-name content=hadoop-pelt titleApache Hadoop Project Bylaws/title link type=text/css href=skin/basic.css rel=stylesheet @@ -238,7 +238,7 @@ document.write(Last Published: + docu /div -a name=N1000C/aa name=Introduction/a +a name=N1000D/aa name=Introduction/a h2 class=h3Introduction/h2 div class=section pThis document defines the bylaws under which the Apache Hadoop project @@ -256,7 +256,7 @@ document.write(Last Published: + docu /div -a name=N10027/aa name=Roles+and+Responsibilities/a +a name=N10028/aa name=Roles+and+Responsibilities/a h2 class=h3Roles and Responsibilities/h2 div class=section pApache projects define a set of roles with associated rights and @@ -393,7 +393,7 @@ document.write(Last Published: + docu /div -a name=N10081/aa name=Decision+Making/a +a name=N10082/aa name=Decision+Making/a h2 class=h3Decision Making/h2 div class=section pWithin the Hadoop project, different types of decisions require Modified: hadoop/common/site/main/publish/bylaws.pdf URL: http://svn.apache.org/viewvc/hadoop/common/site/main/publish/bylaws.pdf?rev=1392557r1=1392556r2=1392557view=diff == Binary files - no diff available. Modified: hadoop/common/site/main/publish/index.html URL: http://svn.apache.org/viewvc/hadoop/common/site/main/publish/index.html?rev=1392557r1=1392556r2=1392557view=diff == --- hadoop/common/site/main/publish/index.html (original) +++ hadoop/common/site/main/publish/index.html Mon Oct 1 19:56:35 2012 @@ -3,7 +3,7 @@ head META http-equiv=Content-Type content=text/html; charset=UTF-8 meta content=Apache Forrest name=Generator -meta name=Forrest-version content=0.8 +meta name=Forrest-version content=0.9 meta name=Forrest-skin-name content=hadoop-pelt titleWelcome to Apachetrade; Hadoopreg;!/title link type=text/css href=skin/basic.css rel=stylesheet @@ -276,7 +276,7 @@ document.write(Last Published: + docu /div -a name=N1000C/aa name=What+Is+Apache+Hadoop%3F/a +a name=N1000D/aa name=What+Is+Apache+Hadoop%3F/a h2 class=h3What Is Apache Hadoop?/h2 div class=section p @@ -371,7 +371,7 @@ document.write(Last Published: + docu /div -a name=N10071/aa name=Getting+Started/a +a name=N10072/aa name=Getting+Started/a h2 class=h3Getting Started /h2 div class=section
svn commit: r1392567 - in /hadoop/common/branches/branch-1-win: ./ src/core/org/apache/hadoop/fs/ src/core/org/apache/hadoop/util/ src/test/org/apache/hadoop/filecache/ src/test/org/apache/hadoop/fs/
Author: suresh Date: Mon Oct 1 20:27:33 2012 New Revision: 1392567 URL: http://svn.apache.org/viewvc?rev=1392567view=rev Log: HADOOP-8694. Add support for windows native symbolic links. Contributed by Chuan Liu. Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/filecache/TestMRWithDistributedCache.java hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/fs/TestFileUtil.java hadoop/common/branches/branch-1-win/src/winutils/chmod.c hadoop/common/branches/branch-1-win/src/winutils/common.c hadoop/common/branches/branch-1-win/src/winutils/common.h hadoop/common/branches/branch-1-win/src/winutils/hardlink.c hadoop/common/branches/branch-1-win/src/winutils/ls.c hadoop/common/branches/branch-1-win/src/winutils/main.c hadoop/common/branches/branch-1-win/src/winutils/winutils.vcxproj Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1392567r1=1392566r2=1392567view=diff == --- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original) +++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Mon Oct 1 20:27:33 2012 @@ -137,3 +137,6 @@ BUG FIXES HADOOP-8733. Fix the tests TestStreamingTaskLog, TestJvmManager and TestLinuxTaskControllerLaunchArgs to not fail on Windows. (Ivan Mitic via vinodkv) + +HADOOP-8694. Add support for windows native symbolic links. +(Chuan Liu via suresh) Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java?rev=1392567r1=1392566r2=1392567view=diff == --- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java (original) +++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java Mon Oct 1 20:27:33 2012 @@ -39,6 +39,13 @@ import org.apache.hadoop.util.Shell.Shel public class FileUtil { private static final Log LOG = LogFactory.getLog(FileUtil.class); + /* The error code is defined in winutils to indicate insufficient + * privilege to create symbolic links. This value need to keep in + * sync with the constant of the same name in: + * src\winutils\common.h + * */ + public static final int SYMLINK_NO_PRIVILEGE = 2; + /** * convert an array of FileStatus to an array of Path * @@ -554,115 +561,46 @@ public class FileUtil { } } - //review minwei: temp hack to copy file - private static void copyDirectory(String fromFileName, String toFileName) - throws IOException { - -File fromFolder = new File(fromFileName); -File toFolder = new File(toFileName); -if (fromFolder.isFile()) { - copyFile(fromFileName, toFileName); - return; -} - -File[] filelist = fromFolder.listFiles(); -if (filelist == null) { - return; -} - -String fromPath = fromFileName; -String toPath = toFileName; -if (!toFolder.exists()) - toFolder.mkdirs(); -for (int i = 0; i filelist.length; i++) { - String subPath = filelist[i].getName(); - if (filelist[i].isDirectory()) { -copyDirectory(fromPath + / + subPath, toPath + / + subPath); - } else { -copyFile(fromPath + / + subPath, toPath + / + subPath); - } -} - } - - private static void copyFile(String fromFileName, String toFileName) - throws IOException { -File fromFile = new File(fromFileName); -File toFile = new File(toFileName); - -if (!fromFile.exists()) - throw new IOException(FileCopy: + no such source file: -+ fromFileName); - -if (fromFile.isDirectory()) { - copyDirectory(fromFileName, toFileName); - return; -} - -if (!fromFile.canRead()) - throw new IOException(FileCopy: + source file is unreadable: -+ fromFileName); - -// Make sure the parent directory exist for the toFileName -if (toFile.getParent() != null) { - File toFileParentDir = new File(toFile.getParent()); - if (!toFileParentDir.exists() !toFileParentDir.mkdirs()) { -throw new IOException(FileCopy: failed to create target directory: - + toFileParentDir.getPath()); - } -} - -InputStream from = null; -OutputStream to = null; -try { - from = new BufferedInputStream(new FileInputStream(fromFile)); - to = new BufferedOutputStream(new FileOutputStream(toFile));
svn commit: r1392570 - in /hadoop/common/branches/branch-1-win: ./ src/core/org/apache/hadoop/fs/ src/core/org/apache/hadoop/util/ src/test/org/apache/hadoop/filecache/ src/test/org/apache/hadoop/fs/
Author: suresh Date: Mon Oct 1 20:31:25 2012 New Revision: 1392570 URL: http://svn.apache.org/viewvc?rev=1392570view=rev Log: Reverting r1392567 because this commit missed adding a newly added file Added: hadoop/common/branches/branch-1-win/src/winutils/symlink.c Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/filecache/TestMRWithDistributedCache.java hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/fs/TestFileUtil.java hadoop/common/branches/branch-1-win/src/winutils/chmod.c hadoop/common/branches/branch-1-win/src/winutils/common.c hadoop/common/branches/branch-1-win/src/winutils/common.h hadoop/common/branches/branch-1-win/src/winutils/hardlink.c hadoop/common/branches/branch-1-win/src/winutils/ls.c hadoop/common/branches/branch-1-win/src/winutils/main.c hadoop/common/branches/branch-1-win/src/winutils/winutils.vcxproj Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1392570r1=1392569r2=1392570view=diff == --- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original) +++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Mon Oct 1 20:31:25 2012 @@ -137,6 +137,3 @@ BUG FIXES HADOOP-8733. Fix the tests TestStreamingTaskLog, TestJvmManager and TestLinuxTaskControllerLaunchArgs to not fail on Windows. (Ivan Mitic via vinodkv) - -HADOOP-8694. Add support for windows native symbolic links. -(Chuan Liu via suresh) Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java?rev=1392570r1=1392569r2=1392570view=diff == --- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java (original) +++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java Mon Oct 1 20:31:25 2012 @@ -39,13 +39,6 @@ import org.apache.hadoop.util.Shell.Shel public class FileUtil { private static final Log LOG = LogFactory.getLog(FileUtil.class); - /* The error code is defined in winutils to indicate insufficient - * privilege to create symbolic links. This value need to keep in - * sync with the constant of the same name in: - * src\winutils\common.h - * */ - public static final int SYMLINK_NO_PRIVILEGE = 2; - /** * convert an array of FileStatus to an array of Path * @@ -561,46 +554,115 @@ public class FileUtil { } } + //review minwei: temp hack to copy file + private static void copyDirectory(String fromFileName, String toFileName) + throws IOException { + +File fromFolder = new File(fromFileName); +File toFolder = new File(toFileName); +if (fromFolder.isFile()) { + copyFile(fromFileName, toFileName); + return; +} + +File[] filelist = fromFolder.listFiles(); +if (filelist == null) { + return; +} + +String fromPath = fromFileName; +String toPath = toFileName; +if (!toFolder.exists()) + toFolder.mkdirs(); +for (int i = 0; i filelist.length; i++) { + String subPath = filelist[i].getName(); + if (filelist[i].isDirectory()) { +copyDirectory(fromPath + / + subPath, toPath + / + subPath); + } else { +copyFile(fromPath + / + subPath, toPath + / + subPath); + } +} + } + + private static void copyFile(String fromFileName, String toFileName) + throws IOException { +File fromFile = new File(fromFileName); +File toFile = new File(toFileName); + +if (!fromFile.exists()) + throw new IOException(FileCopy: + no such source file: ++ fromFileName); + +if (fromFile.isDirectory()) { + copyDirectory(fromFileName, toFileName); + return; +} + +if (!fromFile.canRead()) + throw new IOException(FileCopy: + source file is unreadable: ++ fromFileName); + +// Make sure the parent directory exist for the toFileName +if (toFile.getParent() != null) { + File toFileParentDir = new File(toFile.getParent()); + if (!toFileParentDir.exists() !toFileParentDir.mkdirs()) { +throw new IOException(FileCopy: failed to create target directory: + + toFileParentDir.getPath()); + } +} + +InputStream from = null; +OutputStream to = null; +try { + from = new BufferedInputStream(new FileInputStream(fromFile)); + to =
svn commit: r1392575 - /hadoop/common/branches/branch-1-win/src/winutils/symlink.c
Author: suresh Date: Mon Oct 1 20:35:52 2012 New Revision: 1392575 URL: http://svn.apache.org/viewvc?rev=1392575view=rev Log: Reverting r1392570 Removed: hadoop/common/branches/branch-1-win/src/winutils/symlink.c
svn commit: r1392576 - in /hadoop/common/branches/branch-1-win/src: core/org/apache/hadoop/fs/ core/org/apache/hadoop/util/ test/org/apache/hadoop/filecache/ test/org/apache/hadoop/fs/ winutils/
Author: suresh Date: Mon Oct 1 20:36:29 2012 New Revision: 1392576 URL: http://svn.apache.org/viewvc?rev=1392576view=rev Log: HADOOP-8694. Add support for windows native symbolic links. Contributed by Chuan Liu. Added: hadoop/common/branches/branch-1-win/src/winutils/symlink.c Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/Shell.java hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/filecache/TestMRWithDistributedCache.java hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/fs/TestFileUtil.java hadoop/common/branches/branch-1-win/src/winutils/chmod.c hadoop/common/branches/branch-1-win/src/winutils/common.c hadoop/common/branches/branch-1-win/src/winutils/common.h hadoop/common/branches/branch-1-win/src/winutils/hardlink.c hadoop/common/branches/branch-1-win/src/winutils/ls.c hadoop/common/branches/branch-1-win/src/winutils/main.c hadoop/common/branches/branch-1-win/src/winutils/winutils.vcxproj Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java?rev=1392576r1=1392575r2=1392576view=diff == --- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java (original) +++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/fs/FileUtil.java Mon Oct 1 20:36:29 2012 @@ -39,6 +39,13 @@ import org.apache.hadoop.util.Shell.Shel public class FileUtil { private static final Log LOG = LogFactory.getLog(FileUtil.class); + /* The error code is defined in winutils to indicate insufficient + * privilege to create symbolic links. This value need to keep in + * sync with the constant of the same name in: + * src\winutils\common.h + * */ + public static final int SYMLINK_NO_PRIVILEGE = 2; + /** * convert an array of FileStatus to an array of Path * @@ -554,115 +561,46 @@ public class FileUtil { } } - //review minwei: temp hack to copy file - private static void copyDirectory(String fromFileName, String toFileName) - throws IOException { - -File fromFolder = new File(fromFileName); -File toFolder = new File(toFileName); -if (fromFolder.isFile()) { - copyFile(fromFileName, toFileName); - return; -} - -File[] filelist = fromFolder.listFiles(); -if (filelist == null) { - return; -} - -String fromPath = fromFileName; -String toPath = toFileName; -if (!toFolder.exists()) - toFolder.mkdirs(); -for (int i = 0; i filelist.length; i++) { - String subPath = filelist[i].getName(); - if (filelist[i].isDirectory()) { -copyDirectory(fromPath + / + subPath, toPath + / + subPath); - } else { -copyFile(fromPath + / + subPath, toPath + / + subPath); - } -} - } - - private static void copyFile(String fromFileName, String toFileName) - throws IOException { -File fromFile = new File(fromFileName); -File toFile = new File(toFileName); - -if (!fromFile.exists()) - throw new IOException(FileCopy: + no such source file: -+ fromFileName); - -if (fromFile.isDirectory()) { - copyDirectory(fromFileName, toFileName); - return; -} - -if (!fromFile.canRead()) - throw new IOException(FileCopy: + source file is unreadable: -+ fromFileName); - -// Make sure the parent directory exist for the toFileName -if (toFile.getParent() != null) { - File toFileParentDir = new File(toFile.getParent()); - if (!toFileParentDir.exists() !toFileParentDir.mkdirs()) { -throw new IOException(FileCopy: failed to create target directory: - + toFileParentDir.getPath()); - } -} - -InputStream from = null; -OutputStream to = null; -try { - from = new BufferedInputStream(new FileInputStream(fromFile)); - to = new BufferedOutputStream(new FileOutputStream(toFile)); - byte[] buffer = new byte[4*1024*1024]; - int bytesRead; - - while ((bytesRead = from.read(buffer)) != -1) -to.write(buffer, 0, bytesRead); // write -} finally { - if (from != null) -try { - from.close(); -} catch (IOException e) { - ; -} - if (to != null) -try { - to.close(); -} catch (IOException e) { - ; -} -} - } - /** * Create a soft link between a src and destination - * only on a local disk. HDFS does not support this + * only on a local disk. HDFS does not support this. + * On Windows, when symlink creation fails due to security + * setting, we will log a warning. The return code in
svn commit: r1392578 - in /hadoop/common/branches/branch-1-win: CHANGES.branch-1-win.txt src/core/org/apache/hadoop/security/UserGroupInformation.java
Author: suresh Date: Mon Oct 1 20:39:06 2012 New Revision: 1392578 URL: http://svn.apache.org/viewvc?rev=1392578view=rev Log: HADOOP-8836. UGI should throw exception in case winutils.exe cannot be loaded. Contributed by Bikas Saha. Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1392578r1=1392577r2=1392578view=diff == --- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original) +++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Mon Oct 1 20:39:06 2012 @@ -137,3 +137,9 @@ BUG FIXES HADOOP-8733. Fix the tests TestStreamingTaskLog, TestJvmManager and TestLinuxTaskControllerLaunchArgs to not fail on Windows. (Ivan Mitic via vinodkv) + +HADOOP-8694. Add support for windows native symbolic links. +(Chuan Liu via suresh) + +HADOOP-8836. UGI should throw exception in case winutils.exe cannot be +loaded. (Bikas Saha via suresh) Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java?rev=1392578r1=1392577r2=1392578view=diff == --- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java (original) +++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/security/UserGroupInformation.java Mon Oct 1 20:39:06 2012 @@ -1040,7 +1040,7 @@ public class UserGroupInformation { ListString result = groups.getGroups(getShortUserName()); return result.toArray(new String[result.size()]); } catch (IOException ie) { - LOG.warn(No groups available for user + getShortUserName()); + LOG.warn(No groups available for user + getShortUserName(), ie); return new String[0]; } }
svn commit: r1392622 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java src/test/java/org/apache/hadoo
Author: jlowe Date: Mon Oct 1 21:24:24 2012 New Revision: 1392622 URL: http://svn.apache.org/viewvc?rev=1392622view=rev Log: svn merge -c 1331007 FIXES: HADOOP-8310. FileContext#checkPath should handle URIs with no port. Contributed by Aaron T. Myers. Added: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java - copied, changed from r1331007, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392622r1=1392621r2=1392622view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 21:24:24 2012 @@ -34,6 +34,8 @@ Release 0.23.5 - UNRELEASED HADOOP-8775. MR2 distcp permits non-positive value to -bandwidth option which causes job never to complete. (Sandy Ryza via atm) +HADOOP-8310. FileContext#checkPath should handle URIs with no port. (atm) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1392622r1=1392621r2=1392622view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java Mon Oct 1 21:24:24 2012 @@ -352,20 +352,23 @@ public abstract class AbstractFileSystem } } String thisScheme = this.getUri().getScheme(); -String thisAuthority = this.getUri().getAuthority(); +String thisHost = this.getUri().getHost(); +String thatHost = uri.getHost(); -// Schemes and authorities must match. +// Schemes and hosts must match. // Allow for null Authority for file:/// if (!thisScheme.equalsIgnoreCase(thatScheme) || - (thisAuthority != null -!thisAuthority.equalsIgnoreCase(thatAuthority)) || - (thisAuthority == null thatAuthority != null)) { + (thisHost != null +!thisHost.equalsIgnoreCase(thatHost)) || + (thisHost == null thatHost != null)) { throw new InvalidPathException(Wrong FS: + path + , expected: + this.getUri()); } +// Ports must match, unless this FS instance is using the default port, in +// which case the port may be omitted from the given URI int thisPort = this.getUri().getPort(); -int thatPort = path.toUri().getPort(); +int thatPort = uri.getPort(); if (thatPort == -1) { // -1 = defaultPort of Uri scheme thatPort = this.getUriDefaultPort(); } Copied: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java (from r1331007, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java) URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java?p2=hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.javap1=hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.javar1=1331007r2=1392622rev=1392622view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java Mon Oct 1 21:24:24 2012 @@ -24,6 +24,7 @@ import java.net.URISyntaxException; import java.util.EnumSet; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.security.AccessControlException; import
svn commit: r1392627 - in /hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java src/test/java/org/apache/had
Author: bobby Date: Mon Oct 1 21:29:34 2012 New Revision: 1392627 URL: http://svn.apache.org/viewvc?rev=1392627view=rev Log: svn merge -c 1392622 FIXES: HADOOP-8310. FileContext#checkPath should handle URIs with no port. Contributed by Aaron T. Myers. Added: hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java - copied unchanged from r1392622, hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java Modified: hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java Modified: hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392627r1=1392626r2=1392627view=diff == --- hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 21:29:34 2012 @@ -1,17 +1,5 @@ Hadoop Change Log -Release 0.23.5 - UNRELEASED - - INCOMPATIBLE CHANGES - - NEW FEATURES - - IMPROVEMENTS - - OPTIMIZATIONS - - BUG FIXES - Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES @@ -37,6 +25,8 @@ Release 0.23.4 - UNRELEASED HADOOP-8623. hadoop jar command should respect HADOOP_OPTS. (Steven Willis via suresh) +HADOOP-8310. FileContext#checkPath should handle URIs with no port. (atm) + Release 0.23.3 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1392627r1=1392626r2=1392627view=diff == --- hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java (original) +++ hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java Mon Oct 1 21:29:34 2012 @@ -352,20 +352,23 @@ public abstract class AbstractFileSystem } } String thisScheme = this.getUri().getScheme(); -String thisAuthority = this.getUri().getAuthority(); +String thisHost = this.getUri().getHost(); +String thatHost = uri.getHost(); -// Schemes and authorities must match. +// Schemes and hosts must match. // Allow for null Authority for file:/// if (!thisScheme.equalsIgnoreCase(thatScheme) || - (thisAuthority != null -!thisAuthority.equalsIgnoreCase(thatAuthority)) || - (thisAuthority == null thatAuthority != null)) { + (thisHost != null +!thisHost.equalsIgnoreCase(thatHost)) || + (thisHost == null thatHost != null)) { throw new InvalidPathException(Wrong FS: + path + , expected: + this.getUri()); } +// Ports must match, unless this FS instance is using the default port, in +// which case the port may be omitted from the given URI int thisPort = this.getUri().getPort(); -int thatPort = path.toUri().getPort(); +int thatPort = uri.getPort(); if (thatPort == -1) { // -1 = defaultPort of Uri scheme thatPort = this.getUriDefaultPort(); }
svn commit: r1392631 - /hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
Author: bobby Date: Mon Oct 1 21:31:24 2012 New Revision: 1392631 URL: http://svn.apache.org/viewvc?rev=1392631view=rev Log: Updated release notes for 0.23.4-rc1 Modified: hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html Modified: hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html?rev=1392631r1=1392630r2=1392631view=diff == --- hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html (original) +++ hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html Mon Oct 1 21:31:24 2012 @@ -12,6 +12,16 @@ These release notes include new develope a name=changes/ h2Changes since Hadoop 0.23.3/h2 ul +li a href=https://issues.apache.org/jira/browse/YARN-137;YARN-137/a. + Major improvement reported by Siddharth Seth and fixed by Siddharth Seth (scheduler)br + bChange the default scheduler to the CapacityScheduler/bbr + blockquoteThere's some bugs in the FifoScheduler atm - doesn't distribute tasks across nodes and some headroom (available resource) issues. +That's not the best experience for users trying out the 2.0 branch. The CS with the default configuration of a single queue behaves the same as the FifoScheduler and doesn't have these issues. +/blockquote/li +li a href=https://issues.apache.org/jira/browse/YARN-108;YARN-108/a. + Critical bug reported by Jason Lowe and fixed by Jason Lowe (nodemanager)br + bFSDownload can create cache directories with the wrong permissions/bbr + blockquoteWhen the cluster is configured with a restrictive umask, e.g.: {{fs.permissions.umask-mode=0077}}, the nodemanager can end up creating directory entries in the public cache with the wrong permissions. The permissions can end up where only the nodemanager user can access files in the public cache, preventing jobs from running properly./blockquote/li li a href=https://issues.apache.org/jira/browse/YARN-106;YARN-106/a. Major bug reported by Jason Lowe and fixed by Jason Lowe (nodemanager)br bNodemanager needs to set permissions of local directories/bbr @@ -144,6 +154,10 @@ java.lang.NullPointerException Minor improvement reported by Steven Willis and fixed by Steven Willis (scripts)br bhadoop jar command should respect HADOOP_OPTS/bbr blockquote/blockquote/li +li a href=https://issues.apache.org/jira/browse/HADOOP-8310;HADOOP-8310/a. + Major bug reported by Aaron T. Myers and fixed by Aaron T. Myers (fs)br + bFileContext#checkPath should handle URIs with no port/bbr + blockquote/blockquote/li li a href=https://issues.apache.org/jira/browse/HADOOP-8183;HADOOP-8183/a. Minor improvement reported by Harsh J and fixed by Harsh J (util)br bStop using mapred.used.genericoptionsparser to avoid unnecessary warnings/bbr
svn commit: r1392637 - in /hadoop/common/branches/branch-1: ./ src/hdfs/org/apache/hadoop/hdfs/ src/hdfs/org/apache/hadoop/hdfs/tools/ src/test/org/apache/hadoop/hdfs/
Author: omalley Date: Mon Oct 1 21:36:38 2012 New Revision: 1392637 URL: http://svn.apache.org/viewvc?rev=1392637view=rev Log: HDFS-3461. Make HFTP and HSFTP use http and https respectively for getting, renewing, and cancelling the underlying token. Systems with weak crypto (kssl) configured will continue to use https. (omalley) Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/build.xml hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/HsftpFileSystem.java hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestHftpFileSystem.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1392637r1=1392636r2=1392637view=diff == --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Mon Oct 1 21:36:38 2012 @@ -610,6 +610,10 @@ Release 1.1.0 - unreleased MAPREDUCE-4558. Disable TestJobTrackerSafeMode (sseth) +HDFS-3461. Make HFTP and HSFTP use http and https respectively for +getting, renewing, and cancelling the underlying token. Systems with +weak crypto (kssl) configured will continue to use https. (omalley) + Release 1.0.4 - Unreleased NEW FEATURES Modified: hadoop/common/branches/branch-1/build.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/build.xml?rev=1392637r1=1392636r2=1392637view=diff == --- hadoop/common/branches/branch-1/build.xml (original) +++ hadoop/common/branches/branch-1/build.xml Mon Oct 1 21:36:38 2012 @@ -765,6 +765,7 @@ provider classname=org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier$Renewer/ provider classname=org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier$Renewer/ provider classname=org.apache.hadoop.hdfs.HftpFileSystem$TokenManager/ +provider classname=org.apache.hadoop.hdfs.HsftpFileSystem$TokenManager/ provider classname=org.apache.hadoop.hdfs.web.WebHdfsFileSystem$DtRenewer/ /service fileset dir=${conf.dir} includes=${jar.properties.list} / Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1392637r1=1392636r2=1392637view=diff == --- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java (original) +++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java Mon Oct 1 21:36:38 2012 @@ -48,7 +48,6 @@ import org.apache.hadoop.hdfs.security.t import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector; import org.apache.hadoop.hdfs.server.namenode.JspHelper; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.StreamFile; import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher; import org.apache.hadoop.io.Text; @@ -91,12 +90,12 @@ public class HftpFileSystem extends File private URI hftpURI; protected InetSocketAddress nnAddr; - protected InetSocketAddress nnSecureAddr; public static final String HFTP_TIMEZONE = UTC; public static final String HFTP_DATE_FORMAT = -MM-dd'T'HH:mm:ssZ; private Token? delegationToken; + private boolean createdToken = false; private Token? renewToken; private static final HftpDelegationTokenSelector hftpTokenSelector = new HftpDelegationTokenSelector(); @@ -120,27 +119,14 @@ public class HftpFileSystem extends File DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT); } - protected int getDefaultSecurePort() { -return !SecurityUtil.useKsslAuth() ? getDefaultPort() : -getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, -DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT); - } - protected InetSocketAddress getNamenodeAddr(URI uri) { // use authority so user supplied uri can override port return NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort()); } - protected InetSocketAddress getNamenodeSecureAddr(URI uri) { -// must only use the host and the configured https port -return NetUtils.makeSocketAddr(uri.getHost(), getDefaultSecurePort()); - } - @Override public String getCanonicalServiceName() { -// unlike other filesystems, hftp's service is the secure port, not the -//
svn commit: r1392640 - in /hadoop/common/branches/branch-1.1: ./ src/hdfs/org/apache/hadoop/hdfs/ src/hdfs/org/apache/hadoop/hdfs/tools/ src/test/org/apache/hadoop/hdfs/
Author: omalley Date: Mon Oct 1 21:38:58 2012 New Revision: 1392640 URL: http://svn.apache.org/viewvc?rev=1392640view=rev Log: HDFS-3461. Make HFTP and HSFTP use http and https respectively for getting, renewing, and cancelling the underlying token. Systems with weak crypto (kssl) configured will continue to use https. (omalley) Modified: hadoop/common/branches/branch-1.1/ (props changed) hadoop/common/branches/branch-1.1/CHANGES.txt (contents, props changed) hadoop/common/branches/branch-1.1/build.xml hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/HsftpFileSystem.java hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java hadoop/common/branches/branch-1.1/src/test/org/apache/hadoop/hdfs/TestHftpFileSystem.java Propchange: hadoop/common/branches/branch-1.1/ -- Merged /hadoop/common/branches/branch-1:r1392637 Modified: hadoop/common/branches/branch-1.1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/CHANGES.txt?rev=1392640r1=1392639r2=1392640view=diff == --- hadoop/common/branches/branch-1.1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1.1/CHANGES.txt Mon Oct 1 21:38:58 2012 @@ -396,12 +396,16 @@ Release 1.1.0 - 2012.09.28 MAPREDUCE-4558. Disable TestJobTrackerSafeMode (sseth) -HDFS-3701. HDFS may miss the final block when reading a file opened for writing -if one of the datanode is dead. (umamahesh and nkeywal via umamahesh) +HDFS-3701. HDFS may miss the final block when reading a file opened for +writing if one of the datanode is dead. (umamahesh and nkeywal via +umamahesh) HDFS-2751. Backport: Datanode may incorrectly drop OS cache behind reads even for short reads. (Brandon Li via szetszwo) +HDFS-3461. Make HFTP and HSFTP use http and https respectively for +getting, renewing, and cancelling the underlying token. Systems with +weak crypto (kssl) configured will continue to use https. (omalley) Release 1.0.4 - Unreleased Propchange: hadoop/common/branches/branch-1.1/CHANGES.txt -- Merged /hadoop/common/branches/branch-1/CHANGES.txt:r1392637 Modified: hadoop/common/branches/branch-1.1/build.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/build.xml?rev=1392640r1=1392639r2=1392640view=diff == --- hadoop/common/branches/branch-1.1/build.xml (original) +++ hadoop/common/branches/branch-1.1/build.xml Mon Oct 1 21:38:58 2012 @@ -758,6 +758,7 @@ provider classname=org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier$Renewer/ provider classname=org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier$Renewer/ provider classname=org.apache.hadoop.hdfs.HftpFileSystem$TokenManager/ +provider classname=org.apache.hadoop.hdfs.HsftpFileSystem$TokenManager/ provider classname=org.apache.hadoop.hdfs.web.WebHdfsFileSystem$DtRenewer/ /service fileset dir=${conf.dir} includes=${jar.properties.list} / Modified: hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1392640r1=1392639r2=1392640view=diff == --- hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java (original) +++ hadoop/common/branches/branch-1.1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java Mon Oct 1 21:38:58 2012 @@ -48,7 +48,6 @@ import org.apache.hadoop.hdfs.security.t import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector; import org.apache.hadoop.hdfs.server.namenode.JspHelper; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.StreamFile; import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher; import org.apache.hadoop.io.Text; @@ -91,12 +90,12 @@ public class HftpFileSystem extends File private URI hftpURI; protected InetSocketAddress nnAddr; - protected InetSocketAddress nnSecureAddr; public static final String HFTP_TIMEZONE = UTC; public static final String HFTP_DATE_FORMAT = -MM-dd'T'HH:mm:ssZ; private Token? delegationToken; + private boolean createdToken = false; private Token? renewToken; private static final HftpDelegationTokenSelector hftpTokenSelector = new
svn commit: r1392671 - /hadoop/common/tags/release-0.23.4-rc1/
Author: bobby Date: Mon Oct 1 22:38:52 2012 New Revision: 1392671 URL: http://svn.apache.org/viewvc?rev=1392671view=rev Log: Hadoop 0.23.4-rc1 release. Added: hadoop/common/tags/release-0.23.4-rc1/ (props changed) - copied from r1392658, hadoop/common/branches/branch-0.23.4/ Propchange: hadoop/common/tags/release-0.23.4-rc1/ -- --- svn:ignore (added) +++ svn:ignore Mon Oct 1 22:38:52 2012 @@ -0,0 +1,5 @@ +.classpath +.git +.project +.settings +target Propchange: hadoop/common/tags/release-0.23.4-rc1/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Mon Oct 1 22:38:52 2012 @@ -0,0 +1 @@ +/hadoop/common/trunk:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166009,1166402,1167001,1167383,1167662,1170085,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1189613,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456,1239752,1240897,1240928,1243065,1243104,1244766,1245751,1245762,1293419,1304099,1351818,1373683
svn commit: r1392675 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: acmurthy Date: Mon Oct 1 22:42:46 2012 New Revision: 1392675 URL: http://svn.apache.org/viewvc?rev=1392675view=rev Log: HADOOP-8738. Reverted since it broke MR based system tests. Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392675r1=1392674r2=1392675view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 22:42:46 2012 @@ -564,8 +564,6 @@ Release 2.0.2-alpha - 2012-09-07 HADOOP-8031. Configuration class fails to find embedded .jar resources; should use URL.openStream() (genman via tucu) -HADOOP-8738. junit JAR is showing up in the distro (tucu) - HADOOP-8737. cmake: always use JAVA_HOME to find libjvm.so, jni.h, jni_md.h. (Colin Patrick McCabe via eli)
svn commit: r1392676 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Author: acmurthy Date: Mon Oct 1 22:44:08 2012 New Revision: 1392676 URL: http://svn.apache.org/viewvc?rev=1392676view=rev Log: HADOOP-8738. Reverted since it broke MR based system tests. Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392676r1=1392675r2=1392676view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 22:44:08 2012 @@ -309,8 +309,6 @@ Release 2.0.2-alpha - 2012-09-07 HADOOP-8031. Configuration class fails to find embedded .jar resources; should use URL.openStream() (genman via tucu) -HADOOP-8738. junit JAR is showing up in the distro (tucu) - HADOOP-8737. cmake: always use JAVA_HOME to find libjvm.so, jni.h, jni_md.h. (Colin Patrick McCabe via eli)
svn commit: r1392677 - /hadoop/common/branches/branch-2.0.2-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
Author: acmurthy Date: Mon Oct 1 22:44:33 2012 New Revision: 1392677 URL: http://svn.apache.org/viewvc?rev=1392677view=rev Log: HADOOP-8738. Reverted since it broke MR based system tests. Modified: hadoop/common/branches/branch-2.0.2-alpha/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2.0.2-alpha/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.2-alpha/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392677r1=1392676r2=1392677view=diff == --- hadoop/common/branches/branch-2.0.2-alpha/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.0.2-alpha/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 22:44:33 2012 @@ -250,8 +250,6 @@ Release 2.0.2-alpha - 2012-09-07 HADOOP-8031. Configuration class fails to find embedded .jar resources; should use URL.openStream() (genman via tucu) -HADOOP-8738. junit JAR is showing up in the distro (tucu) - HADOOP-8737. cmake: always use JAVA_HOME to find libjvm.so, jni.h, jni_md.h. (Colin Patrick McCabe via eli)
svn commit: r1392680 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
Author: acmurthy Date: Mon Oct 1 22:52:38 2012 New Revision: 1392680 URL: http://svn.apache.org/viewvc?rev=1392680view=rev Log: Updated release notes for 2.0.2-alpha. Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html?rev=1392680r1=1392679r2=1392680view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html Mon Oct 1 22:52:38 2012 @@ -2098,10 +2098,6 @@ Please see hdfs-default.xml for detailed Major bug reported by Colin Patrick McCabe and fixed by Colin Patrick McCabe (native)br bSyntax error on cmake version 2.6 patch 2 in JNIFlags.cmake/bbr blockquote/blockquote/li -li a href=https://issues.apache.org/jira/browse/HADOOP-8738;HADOOP-8738/a. - Critical bug reported by Alejandro Abdelnur and fixed by Alejandro Abdelnur (build)br - bjunit JAR is showing up in the distro/bbr - blockquote/blockquote/li li a href=https://issues.apache.org/jira/browse/HADOOP-8737;HADOOP-8737/a. Minor bug reported by Colin Patrick McCabe and fixed by Colin Patrick McCabe (native)br bcmake: always use JAVA_HOME to find libjvm.so, jni.h, jni_md.h/bbr
svn commit: r1392681 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
Author: acmurthy Date: Mon Oct 1 22:53:09 2012 New Revision: 1392681 URL: http://svn.apache.org/viewvc?rev=1392681view=rev Log: Updated release notes for 2.0.2-alpha. Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html?rev=1392681r1=1392680r2=1392681view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html Mon Oct 1 22:53:09 2012 @@ -2098,10 +2098,6 @@ Please see hdfs-default.xml for detailed Major bug reported by Colin Patrick McCabe and fixed by Colin Patrick McCabe (native)br bSyntax error on cmake version 2.6 patch 2 in JNIFlags.cmake/bbr blockquote/blockquote/li -li a href=https://issues.apache.org/jira/browse/HADOOP-8738;HADOOP-8738/a. - Critical bug reported by Alejandro Abdelnur and fixed by Alejandro Abdelnur (build)br - bjunit JAR is showing up in the distro/bbr - blockquote/blockquote/li li a href=https://issues.apache.org/jira/browse/HADOOP-8737;HADOOP-8737/a. Minor bug reported by Colin Patrick McCabe and fixed by Colin Patrick McCabe (native)br bcmake: always use JAVA_HOME to find libjvm.so, jni.h, jni_md.h/bbr
svn commit: r1392682 - /hadoop/common/branches/branch-2.0.2-alpha/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
Author: acmurthy Date: Mon Oct 1 22:53:15 2012 New Revision: 1392682 URL: http://svn.apache.org/viewvc?rev=1392682view=rev Log: Updated release notes for 2.0.2-alpha. Modified: hadoop/common/branches/branch-2.0.2-alpha/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html Modified: hadoop/common/branches/branch-2.0.2-alpha/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.2-alpha/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html?rev=1392682r1=1392681r2=1392682view=diff == --- hadoop/common/branches/branch-2.0.2-alpha/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html (original) +++ hadoop/common/branches/branch-2.0.2-alpha/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html Mon Oct 1 22:53:15 2012 @@ -2098,10 +2098,6 @@ Please see hdfs-default.xml for detailed Major bug reported by Colin Patrick McCabe and fixed by Colin Patrick McCabe (native)br bSyntax error on cmake version 2.6 patch 2 in JNIFlags.cmake/bbr blockquote/blockquote/li -li a href=https://issues.apache.org/jira/browse/HADOOP-8738;HADOOP-8738/a. - Critical bug reported by Alejandro Abdelnur and fixed by Alejandro Abdelnur (build)br - bjunit JAR is showing up in the distro/bbr - blockquote/blockquote/li li a href=https://issues.apache.org/jira/browse/HADOOP-8737;HADOOP-8737/a. Minor bug reported by Colin Patrick McCabe and fixed by Colin Patrick McCabe (native)br bcmake: always use JAVA_HOME to find libjvm.so, jni.h, jni_md.h/bbr
svn commit: r1392683 - /hadoop/common/tags/release-2.0.2-alpha-rc2/
Author: acmurthy Date: Mon Oct 1 22:53:43 2012 New Revision: 1392683 URL: http://svn.apache.org/viewvc?rev=1392683view=rev Log: hadoop-2.0.2-alpha RC2. Added: hadoop/common/tags/release-2.0.2-alpha-rc2/ (props changed) - copied from r1392682, hadoop/common/branches/branch-2.0.2-alpha/ Propchange: hadoop/common/tags/release-2.0.2-alpha-rc2/ -- --- svn:ignore (added) +++ svn:ignore Mon Oct 1 22:53:43 2012 @@ -0,0 +1,5 @@ +.classpath +.git +.project +.settings +target Propchange: hadoop/common/tags/release-2.0.2-alpha-rc2/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Mon Oct 1 22:53:43 2012 @@ -0,0 +1,3 @@ +/hadoop/common/branches/HDFS-3042:1306184-1342109 +/hadoop/common/branches/branch-0.23-PB:1227776-1294021 +/hadoop/common/trunk:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166009,1166402,1167001,1167383,1167662,1170085,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1189613,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456,1239752,1240897,1240928,1243065,1243104,1244766,1245751,1245762,1293419,1295061,1295227,1296556,1298044,1298696,1298700,1299045,1299434,1299963,1301308,1301312,1301820,1301871,1302624, 1302704-1302705,1303474,1304063,1304099,1304112,1304118,1305230,1309625,1310185,1311556,1312029,1329319,1333557,1334216,1342112,1348207,1349124,1349616,1351818,1361813,1373683,1374696,1375450,1375829,1379646
svn commit: r1392703 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/viewfs/ src/test/java/org/apache/hadoop/fs/viewfs/
Author: atm Date: Tue Oct 2 01:17:00 2012 New Revision: 1392703 URL: http://svn.apache.org/viewvc?rev=1392703view=rev Log: HADOOP-8616. ViewFS configuration requires a trailing slash. Contributed by Sandy Ryza. Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392703r1=1392702r2=1392703view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Oct 2 01:17:00 2012 @@ -307,6 +307,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-8791. Fix rm command documentation to indicte it deletes files and not directories. (Jing Zhao via suresh) +HADOOP-8616. ViewFS configuration requires a trailing slash. (Sandy Ryza +via atm) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java?rev=1392703r1=1392702r2=1392703view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java Tue Oct 2 01:17:00 2012 @@ -89,7 +89,11 @@ class ChRootedFileSystem extends FilterF public ChRootedFileSystem(final URI uri, Configuration conf) throws IOException { super(FileSystem.get(uri, conf)); -chRootPathPart = new Path(uri.getPath()); +String pathString = uri.getPath(); +if (pathString.isEmpty()) { + pathString = /; +} +chRootPathPart = new Path(pathString); chRootPathPartString = chRootPathPart.toUri().getPath(); myUri = uri; workingDir = getHomeDirectory(); Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java?rev=1392703r1=1392702r2=1392703view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java Tue Oct 2 01:17:00 2012 @@ -205,9 +205,13 @@ public class ViewFs extends AbstractFile protected AbstractFileSystem getTargetFileSystem(final URI uri) throws URISyntaxException, UnsupportedFileSystemException { + String pathString = uri.getPath(); + if (pathString.isEmpty()) { +pathString = /; + } return new ChRootedFs( AbstractFileSystem.createFileSystem(uri, config), - new Path(uri.getPath())); + new Path(pathString)); } @Override Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java?rev=1392703r1=1392702r2=1392703view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java Tue Oct 2 01:17:00 2012 @@ -342,6 +342,15 @@ public class TestChRootedFileSystem { chrootFs.close(); verify(mockFs).delete(eq(rawPath), eq(true)); } + + @Test + public void testURIEmptyPath() throws IOException { +Configuration conf = new Configuration(); +conf.setClass(fs.mockfs.impl, MockFileSystem.class, FileSystem.class); + +URI chrootUri = URI.create(mockfs://foo); +new
svn commit: r1392705 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/viewfs/ src/test/java/org/apache/hadoop/fs/viewfs/
Author: atm Date: Tue Oct 2 01:26:27 2012 New Revision: 1392705 URL: http://svn.apache.org/viewvc?rev=1392705view=rev Log: Revert an errant commit of HADOOP-8616. Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392705r1=1392704r2=1392705view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Oct 2 01:26:27 2012 @@ -307,9 +307,6 @@ Release 2.0.3-alpha - Unreleased HADOOP-8791. Fix rm command documentation to indicte it deletes files and not directories. (Jing Zhao via suresh) -HADOOP-8616. ViewFS configuration requires a trailing slash. (Sandy Ryza -via atm) - Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java?rev=1392705r1=1392704r2=1392705view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java Tue Oct 2 01:26:27 2012 @@ -89,11 +89,7 @@ class ChRootedFileSystem extends FilterF public ChRootedFileSystem(final URI uri, Configuration conf) throws IOException { super(FileSystem.get(uri, conf)); -String pathString = uri.getPath(); -if (pathString.isEmpty()) { - pathString = /; -} -chRootPathPart = new Path(pathString); +chRootPathPart = new Path(uri.getPath()); chRootPathPartString = chRootPathPart.toUri().getPath(); myUri = uri; workingDir = getHomeDirectory(); Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java?rev=1392705r1=1392704r2=1392705view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java Tue Oct 2 01:26:27 2012 @@ -205,13 +205,9 @@ public class ViewFs extends AbstractFile protected AbstractFileSystem getTargetFileSystem(final URI uri) throws URISyntaxException, UnsupportedFileSystemException { - String pathString = uri.getPath(); - if (pathString.isEmpty()) { -pathString = /; - } return new ChRootedFs( AbstractFileSystem.createFileSystem(uri, config), - new Path(pathString)); + new Path(uri.getPath())); } @Override Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java?rev=1392705r1=1392704r2=1392705view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java Tue Oct 2 01:26:27 2012 @@ -342,15 +342,6 @@ public class TestChRootedFileSystem { chrootFs.close(); verify(mockFs).delete(eq(rawPath), eq(true)); } - - @Test - public void testURIEmptyPath() throws IOException { -Configuration conf = new Configuration(); -conf.setClass(fs.mockfs.impl, MockFileSystem.class, FileSystem.class); - -URI chrootUri = URI.create(mockfs://foo); -new ChRootedFileSystem(chrootUri, conf); - }
svn commit: r1392707 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/viewfs/ src/test/java/org/apache/hadoop/fs/viewfs/
Author: atm Date: Tue Oct 2 01:28:17 2012 New Revision: 1392707 URL: http://svn.apache.org/viewvc?rev=1392707view=rev Log: HADOOP-8616. ViewFS configuration requires a trailing slash. Contributed by Sandy Ryza. Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsURIs.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392707r1=1392706r2=1392707view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Oct 2 01:28:17 2012 @@ -307,6 +307,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-8791. Fix rm command documentation to indicte it deletes files and not directories. (Jing Zhao via suresh) +HADOOP-8616. ViewFS configuration requires a trailing slash. (Sandy Ryza +via atm) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java?rev=1392707r1=1392706r2=1392707view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java Tue Oct 2 01:28:17 2012 @@ -89,7 +89,11 @@ class ChRootedFileSystem extends FilterF public ChRootedFileSystem(final URI uri, Configuration conf) throws IOException { super(FileSystem.get(uri, conf)); -chRootPathPart = new Path(uri.getPath()); +String pathString = uri.getPath(); +if (pathString.isEmpty()) { + pathString = /; +} +chRootPathPart = new Path(pathString); chRootPathPartString = chRootPathPart.toUri().getPath(); myUri = uri; workingDir = getHomeDirectory(); Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java?rev=1392707r1=1392706r2=1392707view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java Tue Oct 2 01:28:17 2012 @@ -205,9 +205,13 @@ public class ViewFs extends AbstractFile protected AbstractFileSystem getTargetFileSystem(final URI uri) throws URISyntaxException, UnsupportedFileSystemException { + String pathString = uri.getPath(); + if (pathString.isEmpty()) { +pathString = /; + } return new ChRootedFs( AbstractFileSystem.createFileSystem(uri, config), - new Path(uri.getPath())); + new Path(pathString)); } @Override Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java?rev=1392707r1=1392706r2=1392707view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java Tue Oct 2 01:28:17 2012 @@ -342,6 +342,15 @@ public class TestChRootedFileSystem { chrootFs.close(); verify(mockFs).delete(eq(rawPath), eq(true)); } + + @Test + public void testURIEmptyPath() throws IOException { +Configuration conf = new Configuration(); +
svn commit: r1392709 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/viewfs/ src/test/java/org/apache/hadoop/fs/viewfs/
Author: atm Date: Tue Oct 2 01:30:08 2012 New Revision: 1392709 URL: http://svn.apache.org/viewvc?rev=1392709view=rev Log: HADOOP-8616. ViewFS configuration requires a trailing slash. Contributed by Sandy Ryza. Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsURIs.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392709r1=1392708r2=1392709view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Tue Oct 2 01:30:08 2012 @@ -50,6 +50,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-8855. SSL-based image transfer does not work when Kerberos is disabled. (todd via eli) +HADOOP-8616. ViewFS configuration requires a trailing slash. (Sandy Ryza +via atm) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java?rev=1392709r1=1392708r2=1392709view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java Tue Oct 2 01:30:08 2012 @@ -89,7 +89,11 @@ class ChRootedFileSystem extends FilterF public ChRootedFileSystem(final URI uri, Configuration conf) throws IOException { super(FileSystem.get(uri, conf)); -chRootPathPart = new Path(uri.getPath()); +String pathString = uri.getPath(); +if (pathString.isEmpty()) { + pathString = /; +} +chRootPathPart = new Path(pathString); chRootPathPartString = chRootPathPart.toUri().getPath(); myUri = uri; workingDir = getHomeDirectory(); Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java?rev=1392709r1=1392708r2=1392709view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java Tue Oct 2 01:30:08 2012 @@ -205,9 +205,13 @@ public class ViewFs extends AbstractFile protected AbstractFileSystem getTargetFileSystem(final URI uri) throws URISyntaxException, UnsupportedFileSystemException { + String pathString = uri.getPath(); + if (pathString.isEmpty()) { +pathString = /; + } return new ChRootedFs( AbstractFileSystem.createFileSystem(uri, config), - new Path(uri.getPath())); + new Path(pathString)); } @Override Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java?rev=1392709r1=1392708r2=1392709view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java Tue Oct 2 01:30:08 2012 @@ -342,6 +342,15 @@