svn commit: r1412294 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/util/StringUtils.java
Author: tgraves Date: Wed Nov 21 21:03:01 2012 New Revision: 1412294 URL: http://svn.apache.org/viewvc?rev=1412294view=rev Log: HADOOP-8931. Add Java version to startup message. (eli) Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1412294r1=1412293r2=1412294view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Wed Nov 21 21:03:01 2012 @@ -8,6 +8,8 @@ Release 0.23.6 - UNRELEASED IMPROVEMENTS +HADOOP-8931. Add Java version to startup message. (eli) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java?rev=1412294r1=1412293r2=1412294view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java Wed Nov 21 21:03:01 2012 @@ -604,7 +604,8 @@ public class StringUtils { build = + VersionInfo.getUrl() + -r + VersionInfo.getRevision() + ; compiled by ' + VersionInfo.getUser() - + ' on + VersionInfo.getDate()} + + ' on + VersionInfo.getDate(), + java = + System.getProperty(java.version) } ) );
svn commit: r1411884 - /hadoop/common/trunk/pom.xml
Author: tgraves Date: Tue Nov 20 21:22:20 2012 New Revision: 1411884 URL: http://svn.apache.org/viewvc?rev=1411884view=rev Log: HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins (Robert Parker via tgraves) Modified: hadoop/common/trunk/pom.xml Modified: hadoop/common/trunk/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/pom.xml?rev=1411884r1=1411883r2=1411884view=diff == --- hadoop/common/trunk/pom.xml (original) +++ hadoop/common/trunk/pom.xml Tue Nov 20 21:22:20 2012 @@ -517,7 +517,7 @@ xsi:schemaLocation=http://maven.apache. groupIdcom.atlassian.maven.plugins/groupId artifactIdmaven-clover2-plugin/artifactId configuration - includesAllSourceRootstrue/includesAllSourceRoots + includesAllSourceRootsfalse/includesAllSourceRoots includesTestSourceRootstrue/includesTestSourceRoots licenseLocation${cloverLicenseLocation}/licenseLocation cloverDatabase${cloverDatabase}/cloverDatabase
svn commit: r1411884 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: tgraves Date: Tue Nov 20 21:22:20 2012 New Revision: 1411884 URL: http://svn.apache.org/viewvc?rev=1411884view=rev Log: HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins (Robert Parker via tgraves) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1411884r1=1411883r2=1411884view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Nov 20 21:22:20 2012 @@ -1142,6 +1142,9 @@ Release 0.23.6 - UNRELEASED BUG FIXES +HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins +(Robert Parker via tgraves) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES
svn commit: r1411891 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Author: tgraves Date: Tue Nov 20 21:28:46 2012 New Revision: 1411891 URL: http://svn.apache.org/viewvc?rev=1411891view=rev Log: merge -r 1411883:1411884 from trunk. FIXES: HADOOP-9072 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1411891r1=1411890r2=1411891view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Tue Nov 20 21:28:46 2012 @@ -854,6 +854,9 @@ Release 0.23.6 - UNRELEASED BUG FIXES +HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins +(Robert Parker via tgraves) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES
svn commit: r1411891 - /hadoop/common/branches/branch-2/pom.xml
Author: tgraves Date: Tue Nov 20 21:28:46 2012 New Revision: 1411891 URL: http://svn.apache.org/viewvc?rev=1411891view=rev Log: merge -r 1411883:1411884 from trunk. FIXES: HADOOP-9072 Modified: hadoop/common/branches/branch-2/pom.xml Modified: hadoop/common/branches/branch-2/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/pom.xml?rev=1411891r1=1411890r2=1411891view=diff == --- hadoop/common/branches/branch-2/pom.xml (original) +++ hadoop/common/branches/branch-2/pom.xml Tue Nov 20 21:28:46 2012 @@ -410,7 +410,7 @@ xsi:schemaLocation=http://maven.apache. groupIdcom.atlassian.maven.plugins/groupId artifactIdmaven-clover2-plugin/artifactId configuration - includesAllSourceRootstrue/includesAllSourceRoots + includesAllSourceRootsfalse/includesAllSourceRoots includesTestSourceRootstrue/includesTestSourceRoots licenseLocation${cloverLicenseLocation}/licenseLocation cloverDatabase${cloverDatabase}/cloverDatabase
svn commit: r1411892 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Author: tgraves Date: Tue Nov 20 21:28:54 2012 New Revision: 1411892 URL: http://svn.apache.org/viewvc?rev=1411892view=rev Log: merge -r 1411883:1411884 from trunk. FIXES: HADOOP-9072 Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1411892r1=1411891r2=1411892view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Tue Nov 20 21:28:54 2012 @@ -12,6 +12,9 @@ Release 0.23.6 - UNRELEASED BUG FIXES +HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins +(Robert Parker via tgraves) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES
svn commit: r1411892 - /hadoop/common/branches/branch-0.23/pom.xml
Author: tgraves Date: Tue Nov 20 21:28:54 2012 New Revision: 1411892 URL: http://svn.apache.org/viewvc?rev=1411892view=rev Log: merge -r 1411883:1411884 from trunk. FIXES: HADOOP-9072 Modified: hadoop/common/branches/branch-0.23/pom.xml Modified: hadoop/common/branches/branch-0.23/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/pom.xml?rev=1411892r1=1411891r2=1411892view=diff == --- hadoop/common/branches/branch-0.23/pom.xml (original) +++ hadoop/common/branches/branch-0.23/pom.xml Tue Nov 20 21:28:54 2012 @@ -398,7 +398,7 @@ groupIdcom.atlassian.maven.plugins/groupId artifactIdmaven-clover2-plugin/artifactId configuration - includesAllSourceRootstrue/includesAllSourceRoots + includesAllSourceRootsfalse/includesAllSourceRoots includesTestSourceRootstrue/includesTestSourceRoots licenseLocation${cloverLicenseLocation}/licenseLocation cloverDatabase${cloverDatabase}/cloverDatabase
svn commit: r1411441 - in /hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project: hadoop-hdfs-httpfs/pom.xml hadoop-hdfs/CHANGES.txt hadoop-hdfs/pom.xml pom.xml
Author: tgraves Date: Mon Nov 19 21:53:12 2012 New Revision: 1411441 URL: http://svn.apache.org/viewvc?rev=1411441view=rev Log: Preparing 0.23.5 release Modified: hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/pom.xml Modified: hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml Mon Nov 19 21:53:12 2012 @@ -22,12 +22,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-hdfs-httpfs/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version packagingwar/packaging nameApache Hadoop HttpFS/name Modified: hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Nov 19 21:53:12 2012 @@ -1,17 +1,5 @@ Hadoop HDFS Change Log -Release 0.23.6 - UNRELEASED - - INCOMPATIBLE CHANGES - - NEW FEATURES - - IMPROVEMENTS - - OPTIMIZATIONS - - BUG FIXES - Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/pom.xml Mon Nov 19 21:53:12 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project-dist/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../../hadoop-project-dist/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-hdfs/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version descriptionApache Hadoop HDFS/description nameApache Hadoop HDFS/name packagingjar/packaging Modified: hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/pom.xml Mon Nov 19 21:53:12 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-hdfs-project/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version descriptionApache Hadoop HDFS Project/description nameApache Hadoop HDFS Project/name packagingpom/packaging
svn commit: r1411470 - /hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/build/aop.xml
Author: tgraves Date: Mon Nov 19 22:41:55 2012 New Revision: 1411470 URL: http://svn.apache.org/viewvc?rev=1411470view=rev Log: Preparing 0.23.5 release Modified: hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/build/aop.xml Modified: hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/build/aop.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/build/aop.xml?rev=1411470r1=1411469r2=1411470view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/build/aop.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/build/aop.xml Mon Nov 19 22:41:55 2012 @@ -21,7 +21,7 @@ property name=aspectversion value=1.6.5/ !-- TODO this has to be changed synchronously with build.xml version prop.-- !-- this workarounds of test-patch setting its own 'version' -- - property name=project.version value=0.23.6-SNAPSHOT/ + property name=project.version value=0.23.5/ !-- Properties common for all fault injections -- property name=build-fi.dir value=${basedir}/build-fi/
svn commit: r1411441 - in /hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/ hadoop-mapreduce-client/hadoo
Author: tgraves Date: Mon Nov 19 21:53:12 2012 New Revision: 1411441 URL: http://svn.apache.org/viewvc?rev=1411441view=rev Log: Preparing 0.23.5 release Modified: hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/pom.xml Modified: hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/CHANGES.txt?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/CHANGES.txt Mon Nov 19 21:53:12 2012 @@ -1,20 +1,5 @@ Hadoop MapReduce Change Log -Release 0.23.6 - UNRELEASED - - INCOMPATIBLE CHANGES - - NEW FEATURES - - IMPROVEMENTS - - OPTIMIZATIONS - - BUG FIXES - -MAPREDUCE-4802. Takes a long time to load the task list on the AM for -large jobs (Ravi Prakash via bobby) - Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES @@ -123,6 +108,9 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4801. ShuffleHandler can generate large logs due to prematurely closed channels (jlowe via bobby) + +MAPREDUCE-4802. Takes a long time to load the task list on the AM for +large jobs (Ravi Prakash via bobby) Release 0.23.4 Modified: hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml Mon Nov 19 21:53:12 2012 @@ -19,12 +19,12 @@ parent artifactIdhadoop-mapreduce-client/artifactId groupIdorg.apache.hadoop/groupId -version0.23.6-SNAPSHOT/version +version0.23.5/version /parent modelVersion4.0.0/modelVersion groupIdorg.apache.hadoop/groupId artifactIdhadoop-mapreduce-client-app/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version namehadoop-mapreduce-client-app/name properties Modified: hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml Mon Nov 19 21:53:12 2012 @@ -19,12 +19,12 @@ parent artifactIdhadoop-mapreduce-client/artifactId groupIdorg.apache.hadoop/groupId -version0.23.6-SNAPSHOT/version +version0.23.5/version /parent modelVersion4.0.0/modelVersion groupIdorg.apache.hadoop/groupId artifactIdhadoop-mapreduce-client-common/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version namehadoop-mapreduce-client-common/name properties Modified: hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client
svn commit: r1411442 - /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
Author: tgraves Date: Mon Nov 19 21:54:42 2012 New Revision: 1411442 URL: http://svn.apache.org/viewvc?rev=1411442view=rev Log: Fixing CHANGES.txt to show jiras in re-branched 0.23.5 release Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1411442r1=1411441r2=1411442view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon Nov 19 21:54:42 2012 @@ -12,9 +12,6 @@ Release 0.23.6 - UNRELEASED BUG FIXES -MAPREDUCE-4802. Takes a long time to load the task list on the AM for -large jobs (Ravi Prakash via bobby) - Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES @@ -123,6 +120,9 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4801. ShuffleHandler can generate large logs due to prematurely closed channels (jlowe via bobby) + +MAPREDUCE-4802. Takes a long time to load the task list on the AM for +large jobs (Ravi Prakash via bobby) Release 0.23.4
svn commit: r1411470 - /hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
Author: tgraves Date: Mon Nov 19 22:41:55 2012 New Revision: 1411470 URL: http://svn.apache.org/viewvc?rev=1411470view=rev Log: Preparing 0.23.5 release Modified: hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml Modified: hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml?rev=1411470r1=1411469r2=1411470view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml Mon Nov 19 22:41:55 2012 @@ -105,7 +105,7 @@ target symlink link=${applink.base}.jar resource=mr-app.jar failonerror=false/ -symlink link=${applink.base}-0.23.6-SNAPSHOT.jar +symlink link=${applink.base}-0.23.5.jar resource=mr-app.jar failonerror=false/ /target /configuration
svn commit: r1411441 - in /hadoop/common/branches/branch-0.23.5/hadoop-yarn-project: ./ hadoop-yarn/ hadoop-yarn/hadoop-yarn-api/ hadoop-yarn/hadoop-yarn-applications/ hadoop-yarn/hadoop-yarn-applicat
Author: tgraves Date: Mon Nov 19 21:53:12 2012 New Revision: 1411441 URL: http://svn.apache.org/viewvc?rev=1411441view=rev Log: Preparing 0.23.5 release Modified: hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/pom.xml Modified: hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/CHANGES.txt?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/CHANGES.txt Mon Nov 19 21:53:12 2012 @@ -1,26 +1,5 @@ Hadoop YARN Change Log -Release 0.23.6 - UNRELEASED - - INCOMPATIBLE CHANGES - - NEW FEATURES - - IMPROVEMENTS - - OPTIMIZATIONS - - BUG FIXES - -YARN-188. Coverage fixing for CapacityScheduler (Aleksey Gorshkov via -bobby) - -YARN-214. RMContainerImpl does not handle event EXPIRE at state RUNNING -(jeagles via bobby) - -YARN-151. Browser thinks RM main page JS is taking too long -(Ravi Prakash via bobby) - Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES @@ -111,6 +90,15 @@ Release 0.23.5 - UNRELEASED YARN-219. NM should aggregate logs when application finishes. (bobby) +YARN-188. Coverage fixing for CapacityScheduler (Aleksey Gorshkov via +bobby) + +YARN-214. RMContainerImpl does not handle event EXPIRE at state RUNNING +(jeagles via bobby) + +YARN-151. Browser thinks RM main page JS is taking too long +(Ravi Prakash via bobby) + Release 0.23.4 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml Mon Nov 19 21:53:12 2012 @@ -19,12 +19,12 @@ parent artifactIdhadoop-yarn/artifactId groupIdorg.apache.hadoop/groupId -version0.23.6-SNAPSHOT/version +version0.23.5/version /parent modelVersion4.0.0/modelVersion groupIdorg.apache.hadoop/groupId artifactIdhadoop-yarn-api/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version namehadoop-yarn-api/name properties Modified: hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml Mon Nov 19 21:53:12 2012 @@ -19,12 +19,12 @@ parent artifactIdhadoop-yarn-applications
svn commit: r1411442 - /hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
Author: tgraves Date: Mon Nov 19 21:54:42 2012 New Revision: 1411442 URL: http://svn.apache.org/viewvc?rev=1411442view=rev Log: Fixing CHANGES.txt to show jiras in re-branched 0.23.5 release Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt?rev=1411442r1=1411441r2=1411442view=diff == --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Mon Nov 19 21:54:42 2012 @@ -12,15 +12,6 @@ Release 0.23.6 - UNRELEASED BUG FIXES -YARN-188. Coverage fixing for CapacityScheduler (Aleksey Gorshkov via -bobby) - -YARN-214. RMContainerImpl does not handle event EXPIRE at state RUNNING -(jeagles via bobby) - -YARN-151. Browser thinks RM main page JS is taking too long -(Ravi Prakash via bobby) - Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES @@ -111,6 +102,16 @@ Release 0.23.5 - UNRELEASED YARN-219. NM should aggregate logs when application finishes. (bobby) +YARN-188. Coverage fixing for CapacityScheduler (Aleksey Gorshkov via +bobby) + +YARN-214. RMContainerImpl does not handle event EXPIRE at state RUNNING +(jeagles via bobby) + +YARN-151. Browser thinks RM main page JS is taking too long +(Ravi Prakash via bobby) + + Release 0.23.4 INCOMPATIBLE CHANGES
svn commit: r1411429 - /hadoop/common/branches/branch-0.23.5/
Author: tgraves Date: Mon Nov 19 21:33:35 2012 New Revision: 1411429 URL: http://svn.apache.org/viewvc?rev=1411429view=rev Log: Remove branch-0.23.5. It is going to be re-branched to pick up a couple more jira. Removed: hadoop/common/branches/branch-0.23.5/
svn commit: r1411435 - /hadoop/common/branches/branch-0.23.5/
Author: tgraves Date: Mon Nov 19 21:38:34 2012 New Revision: 1411435 URL: http://svn.apache.org/viewvc?rev=1411435view=rev Log: Branching 0.23.5 release Added: hadoop/common/branches/branch-0.23.5/ (props changed) - copied from r1411434, hadoop/common/branches/branch-0.23/ Propchange: hadoop/common/branches/branch-0.23.5/ -- --- svn:ignore (added) +++ svn:ignore Mon Nov 19 21:38:34 2012 @@ -0,0 +1,5 @@ +.classpath +.git +.project +.settings +target Propchange: hadoop/common/branches/branch-0.23.5/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Mon Nov 19 21:38:34 2012 @@ -0,0 +1 @@ +/hadoop/common/trunk:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166009,1166402,1167001,1167383,1167662,1170085,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1189613,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456,1239752,1240897,1240928,1243065,1243104,1244766,1245751,1245762,1293419,1304099,1351818,1373683
svn commit: r1411441 - in /hadoop/common/branches/branch-0.23.5: ./ hadoop-assemblies/ hadoop-client/ hadoop-dist/ hadoop-minicluster/ hadoop-project-dist/ hadoop-project/ hadoop-tools/ hadoop-tools/h
Author: tgraves Date: Mon Nov 19 21:53:12 2012 New Revision: 1411441 URL: http://svn.apache.org/viewvc?rev=1411441view=rev Log: Preparing 0.23.5 release Modified: hadoop/common/branches/branch-0.23.5/hadoop-assemblies/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-client/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-dist/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-minicluster/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-project-dist/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-project/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-archives/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-datajoin/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-distcp/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-extras/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-gridmix/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-pipes/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-rumen/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-streaming/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-tools-dist/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/pom.xml hadoop/common/branches/branch-0.23.5/pom.xml Modified: hadoop/common/branches/branch-0.23.5/hadoop-assemblies/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-assemblies/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-assemblies/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-assemblies/pom.xml Mon Nov 19 21:53:12 2012 @@ -23,12 +23,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-assemblies/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version nameApache Hadoop Assemblies/name descriptionApache Hadoop Assemblies/description Modified: hadoop/common/branches/branch-0.23.5/hadoop-client/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-client/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-client/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-client/pom.xml Mon Nov 19 21:53:12 2012 @@ -18,12 +18,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-client/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version packagingjar/packaging descriptionApache Hadoop Client/description Modified: hadoop/common/branches/branch-0.23.5/hadoop-dist/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-dist/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-dist/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-dist/pom.xml Mon Nov 19 21:53:12 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-dist/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version descriptionApache Hadoop Distribution/description nameApache Hadoop Distribution/name packagingjar/packaging Modified: hadoop/common/branches/branch-0.23.5/hadoop-minicluster/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-minicluster/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-minicluster/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-minicluster/pom.xml Mon Nov 19 21:53:12 2012 @@ -18,12 +18,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-minicluster/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version packagingjar/packaging descriptionApache Hadoop Mini-Cluster
svn commit: r1411470 [1/2] - /hadoop/common/branches/branch-0.23.5/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
Author: tgraves Date: Mon Nov 19 22:41:55 2012 New Revision: 1411470 URL: http://svn.apache.org/viewvc?rev=1411470view=rev Log: Preparing 0.23.5 release Modified: hadoop/common/branches/branch-0.23.5/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
svn commit: r1409032 - in /hadoop/common/trunk/hadoop-mapreduce-project: CHANGES.txt hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainer
Author: tgraves Date: Tue Nov 13 23:47:57 2012 New Revision: 1409032 URL: http://svn.apache.org/viewvc?rev=1409032view=rev Log: MAPREDUCE-4517. Too many INFO messages written out during AM to RM heartbeat (Jason Lowe via tgraves) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1409032r1=1409031r2=1409032view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Nov 13 23:47:57 2012 @@ -660,6 +660,9 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4786. Job End Notification retry interval is 5 milliseconds by default (Ravi Prakash via bobby) + +MAPREDUCE-4517. Too many INFO messages written out during AM to RM heartbeat +(Jason Lowe via tgraves) Release 0.23.4 - UNRELEASED Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1409032r1=1409031r2=1409032view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Tue Nov 13 23:47:57 2012 @@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.RackResolver; @@ -145,6 +146,8 @@ public class RMContainerAllocator extend BlockingQueueContainerAllocatorEvent eventQueue = new LinkedBlockingQueueContainerAllocatorEvent(); + private ScheduleStats scheduleStats = new ScheduleStats(); + public RMContainerAllocator(ClientService clientService, AppContext context) { super(clientService, context); this.stopped = new AtomicBoolean(false); @@ -208,13 +211,10 @@ public class RMContainerAllocator extend @Override protected synchronized void heartbeat() throws Exception { -LOG.info(Before Scheduling: + getStat()); +scheduleStats.updateAndLogIfChanged(Before Scheduling: ); ListContainer allocatedContainers = getResources(); -LOG.info(After Scheduling: + getStat()); if (allocatedContainers.size() 0) { - LOG.info(Before Assign: + getStat()); scheduledRequests.assign(allocatedContainers); - LOG.info(After Assign: + getStat()); } int completedMaps = getJob().getCompletedMaps(); @@ -235,6 +235,8 @@ public class RMContainerAllocator extend maxReduceRampupLimit, reduceSlowStart); recalculateReduceSchedule = false; } + +scheduleStats.updateAndLogIfChanged(After Scheduling: ); } @Override @@ -245,7 +247,7 @@ public class RMContainerAllocator extend } eventHandlingThread.interrupt(); super.stop(); -LOG.info(Final Stats: + getStat()); +scheduleStats.log(Final Stats: ); } public boolean getIsReduceStarted() { @@ -427,7 +429,9 @@ public class RMContainerAllocator extend return; } -LOG.info(Recalculating schedule...); +int headRoom = getAvailableResources() != null ? +getAvailableResources().getMemory() : 0; +LOG.info(Recalculating schedule, headroom= + headRoom); //check for slow start if (!getIsReduceStarted()) {//not set yet @@ -536,24 +540,6 @@ public class RMContainerAllocator extend } } - /** - * Synchronized to avoid findbugs warnings - */ - private synchronized String getStat() { -return PendingReduces: + pendingReduces.size() + - ScheduledMaps: + scheduledRequests.maps.size() + - ScheduledReduces: + scheduledRequests.reduces.size() + - AssignedMaps: + assignedRequests.maps.size() + - AssignedReduces: + assignedRequests.reduces.size() + - completedMaps: + getJob().getCompletedMaps() + - completedReduces
svn commit: r1409034 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: CHANGES.txt hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm
Author: tgraves Date: Tue Nov 13 23:49:04 2012 New Revision: 1409034 URL: http://svn.apache.org/viewvc?rev=1409034view=rev Log: merge -r 1409031:1409032 from trunk. FIXES: MAPREDUCE-4517 Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1409034r1=1409033r2=1409034view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue Nov 13 23:49:04 2012 @@ -513,6 +513,9 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4786. Job End Notification retry interval is 5 milliseconds by default (Ravi Prakash via bobby) + +MAPREDUCE-4517. Too many INFO messages written out during AM to RM heartbeat +(Jason Lowe via tgraves) Release 0.23.4 - UNRELEASED Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1409034r1=1409033r2=1409034view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Tue Nov 13 23:49:04 2012 @@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.RackResolver; @@ -145,6 +146,8 @@ public class RMContainerAllocator extend BlockingQueueContainerAllocatorEvent eventQueue = new LinkedBlockingQueueContainerAllocatorEvent(); + private ScheduleStats scheduleStats = new ScheduleStats(); + public RMContainerAllocator(ClientService clientService, AppContext context) { super(clientService, context); this.stopped = new AtomicBoolean(false); @@ -208,13 +211,10 @@ public class RMContainerAllocator extend @Override protected synchronized void heartbeat() throws Exception { -LOG.info(Before Scheduling: + getStat()); +scheduleStats.updateAndLogIfChanged(Before Scheduling: ); ListContainer allocatedContainers = getResources(); -LOG.info(After Scheduling: + getStat()); if (allocatedContainers.size() 0) { - LOG.info(Before Assign: + getStat()); scheduledRequests.assign(allocatedContainers); - LOG.info(After Assign: + getStat()); } int completedMaps = getJob().getCompletedMaps(); @@ -235,6 +235,8 @@ public class RMContainerAllocator extend maxReduceRampupLimit, reduceSlowStart); recalculateReduceSchedule = false; } + +scheduleStats.updateAndLogIfChanged(After Scheduling: ); } @Override @@ -245,7 +247,7 @@ public class RMContainerAllocator extend } eventHandlingThread.interrupt(); super.stop(); -LOG.info(Final Stats: + getStat()); +scheduleStats.log(Final Stats: ); } public boolean getIsReduceStarted() { @@ -427,7 +429,9 @@ public class RMContainerAllocator extend return; } -LOG.info(Recalculating schedule...); +int headRoom = getAvailableResources() != null ? +getAvailableResources().getMemory() : 0; +LOG.info(Recalculating schedule, headroom= + headRoom); //check for slow start if (!getIsReduceStarted()) {//not set yet @@ -536,24 +540,6 @@ public class RMContainerAllocator extend } } - /** - * Synchronized to avoid findbugs warnings - */ - private synchronized String getStat() { -return PendingReduces: + pendingReduces.size() + - ScheduledMaps: + scheduledRequests.maps.size() + - ScheduledReduces: + scheduledRequests.reduces.size() + - AssignedMaps: + assignedRequests.maps.size() + - AssignedReduces: + assignedRequests.reduces.size
svn commit: r1409035 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: CHANGES.txt hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app
Author: tgraves Date: Tue Nov 13 23:50:23 2012 New Revision: 1409035 URL: http://svn.apache.org/viewvc?rev=1409035view=rev Log: merge -r 1409031:1409032 from trunk. FIXES: MAPREDUCE-4517 Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1409035r1=1409034r2=1409035view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue Nov 13 23:50:23 2012 @@ -96,6 +96,9 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4786. Job End Notification retry interval is 5 milliseconds by default (Ravi Prakash via bobby) + +MAPREDUCE-4517. Too many INFO messages written out during AM to RM heartbeat +(Jason Lowe via tgraves) Release 0.23.4 - UNRELEASED Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1409035r1=1409034r2=1409035view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Tue Nov 13 23:50:23 2012 @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.RackResolver; @@ -140,6 +141,8 @@ public class RMContainerAllocator extend BlockingQueueContainerAllocatorEvent eventQueue = new LinkedBlockingQueueContainerAllocatorEvent(); + private ScheduleStats scheduleStats = new ScheduleStats(); + public RMContainerAllocator(ClientService clientService, AppContext context) { super(clientService, context); this.stopped = new AtomicBoolean(false); @@ -203,13 +206,10 @@ public class RMContainerAllocator extend @Override protected synchronized void heartbeat() throws Exception { -LOG.info(Before Scheduling: + getStat()); +scheduleStats.updateAndLogIfChanged(Before Scheduling: ); ListContainer allocatedContainers = getResources(); -LOG.info(After Scheduling: + getStat()); if (allocatedContainers.size() 0) { - LOG.info(Before Assign: + getStat()); scheduledRequests.assign(allocatedContainers); - LOG.info(After Assign: + getStat()); } int completedMaps = getJob().getCompletedMaps(); @@ -230,6 +230,8 @@ public class RMContainerAllocator extend maxReduceRampupLimit, reduceSlowStart); recalculateReduceSchedule = false; } + +scheduleStats.updateAndLogIfChanged(After Scheduling: ); } @Override @@ -240,7 +242,7 @@ public class RMContainerAllocator extend } eventHandlingThread.interrupt(); super.stop(); -LOG.info(Final Stats: + getStat()); +scheduleStats.log(Final Stats: ); } public boolean getIsReduceStarted() { @@ -422,7 +424,9 @@ public class RMContainerAllocator extend return; } -LOG.info(Recalculating schedule...); +int headRoom = getAvailableResources() != null ? +getAvailableResources().getMemory() : 0; +LOG.info(Recalculating schedule, headroom= + headRoom); //check for slow start if (!getIsReduceStarted()) {//not set yet @@ -531,24 +535,6 @@ public class RMContainerAllocator extend } } - /** - * Synchronized to avoid findbugs warnings - */ - private synchronized String getStat() { -return PendingReduces: + pendingReduces.size() + - ScheduledMaps: + scheduledRequests.maps.size() + - ScheduledReduces: + scheduledRequests.reduces.size() + - AssignedMaps: + assignedRequests.maps.size() + - AssignedReduces
svn commit: r1407590 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/common/ src/main/java/org/apache/hadoop/hdfs/server/datano
Author: tgraves Date: Fri Nov 9 18:51:43 2012 New Revision: 1407590 URL: http://svn.apache.org/viewvc?rev=1407590view=rev Log: HDFS-4162. Some malformed and unquoted HTML strings are returned from datanode web ui. (Darek Dagit via tgraves) Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1407590r1=1407589r2=1407590view=diff == --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Nov 9 18:51:43 2012 @@ -45,6 +45,9 @@ Release 0.23.5 - UNRELEASED HDFS-4090. getFileChecksum() result incompatible when called against zero-byte files. (Kihwal Lee via daryn) +HDFS-4162. Some malformed and unquoted HTML strings are returned from +datanode web ui. (Darek Dagit via tgraves) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java?rev=1407590r1=1407589r2=1407590view=diff == --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java Fri Nov 9 18:51:43 2012 @@ -387,15 +387,15 @@ public class JspHelper { if (!parts[i].equals()) { tempPath.append(parts[i]); out.print(a href=\browseDirectory.jsp + ?dir= - + tempPath.toString() + namenodeInfoPort= + namenodeInfoPort + + HtmlQuoting.quoteHtmlChars(tempPath.toString()) + namenodeInfoPort= + namenodeInfoPort + getDelegationTokenUrlParam(tokenString) + getUrlParam(NAMENODE_ADDRESS, nnAddress)); - out.print(\ + parts[i] + /a + Path.SEPARATOR); + out.print(\ + HtmlQuoting.quoteHtmlChars(parts[i]) + /a + Path.SEPARATOR); tempPath.append(Path.SEPARATOR); } } if(parts.length 0) { -out.print(parts[parts.length-1]); +out.print(HtmlQuoting.quoteHtmlChars(parts[parts.length-1])); } } catch (UnsupportedEncodingException ex) { @@ -410,16 +410,16 @@ public class JspHelper { String nnAddress) throws IOException { out.print(form action=\browseDirectory.jsp\ method=\get\ name=\goto\); out.print(Goto : ); -out.print(input name=\dir\ type=\text\ width=\50\ id\dir\ value=\+ file+\); -out.print(input name=\go\ type=\submit\ value=\go\); +out.print(input name=\dir\ type=\text\ width=\50\ id=\dir\ value=\+ HtmlQuoting.quoteHtmlChars(file)+\/); +out.print(input name=\go\ type=\submit\ value=\go\/); out.print(input name=\namenodeInfoPort\ type=\hidden\ -+ value=\ + namenodeInfoPort + \); ++ value=\ + namenodeInfoPort + \/); if (UserGroupInformation.isSecurityEnabled()) { out.print(input name=\ + DELEGATION_PARAMETER_NAME - + \ type=\hidden\ value=\ + tokenString + \); + + \ type=\hidden\ value=\ + tokenString + \/); } out.print(input name=\+ NAMENODE_ADDRESS +\ type=\hidden\ -+ value=\ + nnAddress + \); ++ value=\ + nnAddress + \/); out.print(/form); } Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java?rev=1407590r1=1407589r2=1407590view=diff == --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java (original) +++ hadoop
svn commit: r1404177 - /hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
Author: tgraves Date: Wed Oct 31 14:57:12 2012 New Revision: 1404177 URL: http://svn.apache.org/viewvc?rev=1404177view=rev Log: MAPREDUCE-4752. Reduce MR AM memory usage through String Interning (Robert Evans via tgraves) Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java?rev=1404177r1=1404176r2=1404177view=diff == --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java Wed Oct 31 14:57:12 2012 @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.util; import java.util.Iterator; import java.util.Map; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -109,6 +110,7 @@ public class Apps { } else { val = val + SYSTEM_PATH_SEPARATOR + value; } -environment.put(variable, val); +environment.put(StringInterner.weakIntern(variable), +StringInterner.weakIntern(val)); } }
svn commit: r1404179 - /hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
Author: tgraves Date: Wed Oct 31 14:59:47 2012 New Revision: 1404179 URL: http://svn.apache.org/viewvc?rev=1404179view=rev Log: MAPREDUCE-4752. Reduce MR AM memory usage through String Interning (Robert Evans via tgraves) Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java?rev=1404179r1=1404178r2=1404179view=diff == --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java Wed Oct 31 14:59:47 2012 @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.util; import java.util.Iterator; import java.util.Map; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -109,6 +110,7 @@ public class Apps { } else { val = val + SYSTEM_PATH_SEPARATOR + value; } -environment.put(variable, val); +environment.put(StringInterner.weakIntern(variable), +StringInterner.weakIntern(val)); } }
svn commit: r1404177 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop: conf/Configuration.java util/StringInterner.java
Author: tgraves Date: Wed Oct 31 14:57:12 2012 New Revision: 1404177 URL: http://svn.apache.org/viewvc?rev=1404177view=rev Log: MAPREDUCE-4752. Reduce MR AM memory usage through String Interning (Robert Evans via tgraves) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1404177r1=1404176r2=1404177view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java Wed Oct 31 14:57:12 2012 @@ -75,6 +75,7 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.util.StringUtils; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; @@ -2002,13 +2003,16 @@ public class Configuration implements It continue; Element field = (Element)fieldNode; if (name.equals(field.getTagName()) field.hasChildNodes()) -attr = ((Text)field.getFirstChild()).getData().trim(); +attr = StringInterner.weakIntern( +((Text)field.getFirstChild()).getData().trim()); if (value.equals(field.getTagName()) field.hasChildNodes()) -value = ((Text)field.getFirstChild()).getData(); +value = StringInterner.weakIntern( +((Text)field.getFirstChild()).getData()); if (final.equals(field.getTagName()) field.hasChildNodes()) finalParameter = true.equals(((Text)field.getFirstChild()).getData()); if (source.equals(field.getTagName()) field.hasChildNodes()) -source.add(((Text)field.getFirstChild()).getData()); +source.add(StringInterner.weakIntern( +((Text)field.getFirstChild()).getData())); } source.add(name); Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java?rev=1404177r1=1404176r2=1404177view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java Wed Oct 31 14:57:12 2012 @@ -59,6 +59,9 @@ public class StringInterner { * @return strong reference to interned string instance */ public static String strongIntern(String sample) { +if (sample == null) { + return null; +} return strongInterner.intern(sample); } @@ -72,6 +75,9 @@ public class StringInterner { * @return weak reference to interned string instance */ public static String weakIntern(String sample) { +if (sample == null) { + return null; +} return weakInterner.intern(sample); }
svn commit: r1404179 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop: conf/Configuration.java util/StringInterner.java
Author: tgraves Date: Wed Oct 31 14:59:47 2012 New Revision: 1404179 URL: http://svn.apache.org/viewvc?rev=1404179view=rev Log: MAPREDUCE-4752. Reduce MR AM memory usage through String Interning (Robert Evans via tgraves) Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1404179r1=1404178r2=1404179view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java Wed Oct 31 14:59:47 2012 @@ -74,6 +74,7 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.util.StringUtils; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; @@ -1916,13 +1917,16 @@ public class Configuration implements It continue; Element field = (Element)fieldNode; if (name.equals(field.getTagName()) field.hasChildNodes()) -attr = ((Text)field.getFirstChild()).getData().trim(); +attr = StringInterner.weakIntern( +((Text)field.getFirstChild()).getData().trim()); if (value.equals(field.getTagName()) field.hasChildNodes()) -value = ((Text)field.getFirstChild()).getData(); +value = StringInterner.weakIntern( +((Text)field.getFirstChild()).getData()); if (final.equals(field.getTagName()) field.hasChildNodes()) finalParameter = true.equals(((Text)field.getFirstChild()).getData()); if (source.equals(field.getTagName()) field.hasChildNodes()) -source.add(((Text)field.getFirstChild()).getData()); +source.add(StringInterner.weakIntern( +((Text)field.getFirstChild()).getData())); } source.add(name); Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java?rev=1404179r1=1404178r2=1404179view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java Wed Oct 31 14:59:47 2012 @@ -59,6 +59,9 @@ public class StringInterner { * @return strong reference to interned string instance */ public static String strongIntern(String sample) { +if (sample == null) { + return null; +} return strongInterner.intern(sample); } @@ -72,6 +75,9 @@ public class StringInterner { * @return weak reference to interned string instance */ public static String weakIntern(String sample) { +if (sample == null) { + return null; +} return weakInterner.intern(sample); }
svn commit: r1404182 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop: conf/Configuration.java util/StringInterner.java
Author: tgraves Date: Wed Oct 31 15:02:59 2012 New Revision: 1404182 URL: http://svn.apache.org/viewvc?rev=1404182view=rev Log: merge -r 1404176:1404177 from trunk. FIXES: MAPREDUCE-4752 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1404182r1=1404181r2=1404182view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java Wed Oct 31 15:02:59 2012 @@ -75,6 +75,7 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.util.StringUtils; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; @@ -1964,13 +1965,16 @@ public class Configuration implements It continue; Element field = (Element)fieldNode; if (name.equals(field.getTagName()) field.hasChildNodes()) -attr = ((Text)field.getFirstChild()).getData().trim(); +attr = StringInterner.weakIntern( +((Text)field.getFirstChild()).getData().trim()); if (value.equals(field.getTagName()) field.hasChildNodes()) -value = ((Text)field.getFirstChild()).getData(); +value = StringInterner.weakIntern( +((Text)field.getFirstChild()).getData()); if (final.equals(field.getTagName()) field.hasChildNodes()) finalParameter = true.equals(((Text)field.getFirstChild()).getData()); if (source.equals(field.getTagName()) field.hasChildNodes()) -source.add(((Text)field.getFirstChild()).getData()); +source.add(StringInterner.weakIntern( +((Text)field.getFirstChild()).getData())); } source.add(name); Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java?rev=1404182r1=1404181r2=1404182view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java Wed Oct 31 15:02:59 2012 @@ -59,6 +59,9 @@ public class StringInterner { * @return strong reference to interned string instance */ public static String strongIntern(String sample) { +if (sample == null) { + return null; +} return strongInterner.intern(sample); } @@ -72,6 +75,9 @@ public class StringInterner { * @return weak reference to interned string instance */ public static String weakIntern(String sample) { +if (sample == null) { + return null; +} return weakInterner.intern(sample); }
svn commit: r1402660 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
Author: tgraves Date: Fri Oct 26 21:03:29 2012 New Revision: 1402660 URL: http://svn.apache.org/viewvc?rev=1402660view=rev Log: HADOOP-8713. TestRPCCompatibility fails intermittently with JDK7 Trevor Robinson via tgraves) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1402660r1=1402659r2=1402660view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Oct 26 21:03:29 2012 @@ -383,6 +383,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-8951. RunJar to fail with user-comprehensible error message if jar missing. (stevel via suresh) +HADOOP-8713. TestRPCCompatibility fails intermittently with JDK7 +(Trevor Robinson via tgraves) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java?rev=1402660r1=1402659r2=1402660view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java Fri Oct 26 21:03:29 2012 @@ -36,6 +36,7 @@ import org.apache.hadoop.ipc.protobuf.Pr import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolSignatureProto; import org.apache.hadoop.net.NetUtils; import org.junit.After; +import org.junit.Before; import org.junit.Test; /** Unit test for supporting method-name based compatible RPCs. */ @@ -114,6 +115,11 @@ public class TestRPCCompatibility { } } + + @Before + public void setUp() { +ProtocolSignature.resetCache(); + } @After public void tearDown() throws IOException { @@ -219,7 +225,6 @@ System.out.println(echo int is NOT supp @Test // equal version client and server public void testVersion2ClientVersion2Server() throws Exception { -ProtocolSignature.resetCache(); // create a server with two handlers TestImpl2 impl = new TestImpl2(); server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
svn commit: r1402662 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
Author: tgraves Date: Fri Oct 26 21:04:57 2012 New Revision: 1402662 URL: http://svn.apache.org/viewvc?rev=1402662view=rev Log: merge -r 1402659:1402660 from trunk. FIXES: HADOOP-8713 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1402662r1=1402661r2=1402662view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Fri Oct 26 21:04:57 2012 @@ -113,6 +113,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-8951. RunJar to fail with user-comprehensible error message if jar missing. (stevel via suresh) +HADOOP-8713. TestRPCCompatibility fails intermittently with JDK7 +(Trevor Robinson via tgraves) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java?rev=1402662r1=1402661r2=1402662view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java Fri Oct 26 21:04:57 2012 @@ -36,6 +36,7 @@ import org.apache.hadoop.ipc.protobuf.Pr import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolSignatureProto; import org.apache.hadoop.net.NetUtils; import org.junit.After; +import org.junit.Before; import org.junit.Test; /** Unit test for supporting method-name based compatible RPCs. */ @@ -114,6 +115,11 @@ public class TestRPCCompatibility { } } + + @Before + public void setUp() { +ProtocolSignature.resetCache(); + } @After public void tearDown() throws IOException { @@ -219,7 +225,6 @@ System.out.println(echo int is NOT supp @Test // equal version client and server public void testVersion2ClientVersion2Server() throws Exception { -ProtocolSignature.resetCache(); // create a server with two handlers TestImpl2 impl = new TestImpl2(); server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
svn commit: r1401668 - in /hadoop/common/trunk/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/sch
Author: tgraves Date: Wed Oct 24 13:21:09 2012 New Revision: 1401668 URL: http://svn.apache.org/viewvc?rev=1401668view=rev Log: YARN-177. CapacityScheduler - adding a queue while the RM is running has wacky results (acmurthy vai tgraves) Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1401668r1=1401667r2=1401668view=diff == --- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Wed Oct 24 13:21:09 2012 @@ -174,6 +174,9 @@ Release 0.23.5 - UNRELEASED YARN-174. Modify NodeManager to pass the user's configuration even when rebooting. (vinodkv) +YARN-177. CapacityScheduler - adding a queue while the RM is running has +wacky results (acmurthy vai tgraves) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java?rev=1401668r1=1401667r2=1401668view=diff == --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java Wed Oct 24 13:21:09 2012 @@ -51,6 +51,12 @@ extends org.apache.hadoop.yarn.server.re public CSQueue getParent(); /** + * Set the parent codeQueue/code. + * @param newParentQueue new parent queue + */ + public void setParent(CSQueue newParentQueue); + + /** * Get the queue name. * @return the queue name */ @@ -195,10 +201,10 @@ extends org.apache.hadoop.yarn.server.re /** * Reinitialize the queue. - * @param queue new queue to re-initalize from + * @param newlyParsedQueue new queue to re-initalize from * @param clusterResource resources in the cluster */ - public void reinitialize(CSQueue queue, Resource clusterResource) + public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) throws IOException; /** Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1401668r1=1401667r2=1401668view=diff == --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Wed Oct 24 13:21:09 2012 @@ -223,7 +223,7 @@ public class LeafQueue implements CSQueu { // Sanity check CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); -float absCapacity = parent.getAbsoluteCapacity() * capacity; +float absCapacity = getParent().getAbsoluteCapacity() * capacity
svn commit: r1401669 - in /hadoop/common/branches/branch-2/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourc
Author: tgraves Date: Wed Oct 24 13:25:19 2012 New Revision: 1401669 URL: http://svn.apache.org/viewvc?rev=1401669view=rev Log: merge -r 1401667:1401668 from trunk. FIXES: YARN-177 Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1401669r1=1401668r2=1401669view=diff == --- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Wed Oct 24 13:25:19 2012 @@ -157,6 +157,9 @@ Release 0.23.5 - UNRELEASED YARN-174. Modify NodeManager to pass the user's configuration even when rebooting. (vinodkv) +YARN-177. CapacityScheduler - adding a queue while the RM is running has +wacky results (acmurthy vai tgraves) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java?rev=1401669r1=1401668r2=1401669view=diff == --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java Wed Oct 24 13:25:19 2012 @@ -51,6 +51,12 @@ extends org.apache.hadoop.yarn.server.re public CSQueue getParent(); /** + * Set the parent codeQueue/code. + * @param newParentQueue new parent queue + */ + public void setParent(CSQueue newParentQueue); + + /** * Get the queue name. * @return the queue name */ @@ -195,10 +201,10 @@ extends org.apache.hadoop.yarn.server.re /** * Reinitialize the queue. - * @param queue new queue to re-initalize from + * @param newlyParsedQueue new queue to re-initalize from * @param clusterResource resources in the cluster */ - public void reinitialize(CSQueue queue, Resource clusterResource) + public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) throws IOException; /** Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1401669r1=1401668r2=1401669view=diff == --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Wed Oct 24 13:25:19 2012 @@ -223,7 +223,7 @@ public class LeafQueue implements CSQueu { // Sanity check CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity
svn commit: r1396705 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
Author: tgraves Date: Wed Oct 10 17:35:18 2012 New Revision: 1396705 URL: http://svn.apache.org/viewvc?rev=1396705view=rev Log: HDFS-3824. TestHftpDelegationToken fails intermittently with JDK7 (Trevor Robinson via tgraves) Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1396705r1=1396704r2=1396705view=diff == --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Oct 10 17:35:18 2012 @@ -1700,6 +1700,9 @@ Release 0.23.5 - UNRELEASED HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor Robinson via tgraves) +HDFS-3824. TestHftpDelegationToken fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java?rev=1396705r1=1396704r2=1396705view=diff == --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java Wed Oct 10 17:35:18 2012 @@ -41,6 +41,8 @@ public class TestHftpDelegationToken { @Test public void testHdfsDelegationToken() throws Exception { +SecurityUtilTestHelper.setTokenServiceUseIp(true); + final Configuration conf = new Configuration(); conf.set(HADOOP_SECURITY_AUTHENTICATION, kerberos); UserGroupInformation.setConfiguration(conf); @@ -265,4 +267,4 @@ public class TestHftpDelegationToken { @Override protected void initDelegationToken() throws IOException {} } -} \ No newline at end of file +}
svn commit: r1396706 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
Author: tgraves Date: Wed Oct 10 17:37:54 2012 New Revision: 1396706 URL: http://svn.apache.org/viewvc?rev=1396706view=rev Log: merge -r 1396704:1396705 from trunk. FIXES: HDFS-3824 Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1396706r1=1396705r2=1396706view=diff == --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Oct 10 17:37:54 2012 @@ -1545,6 +1545,9 @@ Release 0.23.5 - UNRELEASED HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor Robinson via tgraves) +HDFS-3824. TestHftpDelegationToken fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java?rev=1396706r1=1396705r2=1396706view=diff == --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java Wed Oct 10 17:37:54 2012 @@ -41,6 +41,8 @@ public class TestHftpDelegationToken { @Test public void testHdfsDelegationToken() throws Exception { +SecurityUtilTestHelper.setTokenServiceUseIp(true); + final Configuration conf = new Configuration(); conf.set(HADOOP_SECURITY_AUTHENTICATION, kerberos); UserGroupInformation.setConfiguration(conf); @@ -265,4 +267,4 @@ public class TestHftpDelegationToken { @Override protected void initDelegationToken() throws IOException {} } -} \ No newline at end of file +}
svn commit: r1396707 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
Author: tgraves Date: Wed Oct 10 17:41:05 2012 New Revision: 1396707 URL: http://svn.apache.org/viewvc?rev=1396707view=rev Log: merge -r 1396705:1396706 from branch-2. FIXES: HDFS-3824 Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1396707r1=1396706r2=1396707view=diff == --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Oct 10 17:41:05 2012 @@ -21,6 +21,9 @@ Release 0.23.5 - UNRELEASED HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor Robinson via tgraves) +HDFS-3824. TestHftpDelegationToken fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java?rev=1396707r1=1396706r2=1396707view=diff == --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java Wed Oct 10 17:41:05 2012 @@ -43,6 +43,8 @@ public class TestHftpDelegationToken { @Test public void testHdfsDelegationToken() throws Exception { +SecurityUtilTestHelper.setTokenServiceUseIp(true); + final Configuration conf = new Configuration(); conf.set(HADOOP_SECURITY_AUTHENTICATION, kerberos); UserGroupInformation.setConfiguration(conf);
svn commit: r1394609 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
Author: tgraves Date: Fri Oct 5 15:51:10 2012 New Revision: 1394609 URL: http://svn.apache.org/viewvc?rev=1394609view=rev Log: HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor Robinson via tgraves) Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1394609r1=1394608r2=1394609view=diff == --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Oct 5 15:51:10 2012 @@ -1677,6 +1677,9 @@ Release 0.23.5 - UNRELEASED BUG FIXES +HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java?rev=1394609r1=1394608r2=1394609view=diff == --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java Fri Oct 5 15:51:10 2012 @@ -53,19 +53,23 @@ public class TestHftpURLTimeouts { boolean timedout = false; HftpFileSystem fs = (HftpFileSystem)FileSystem.get(uri, conf); -HttpURLConnection conn = fs.openConnection(/, ); -timedout = false; try { - // this will consume the only slot in the backlog - conn.getInputStream(); -} catch (SocketTimeoutException ste) { - timedout = true; - assertEquals(Read timed out, ste.getMessage()); + HttpURLConnection conn = fs.openConnection(/, ); + timedout = false; + try { +// this will consume the only slot in the backlog +conn.getInputStream(); + } catch (SocketTimeoutException ste) { +timedout = true; +assertEquals(Read timed out, ste.getMessage()); + } finally { +if (conn != null) conn.disconnect(); + } + assertTrue(read timedout, timedout); + assertTrue(connect timedout, checkConnectTimeout(fs, false)); } finally { - if (conn != null) conn.disconnect(); + fs.close(); } -assertTrue(read timedout, timedout); -assertTrue(connect timedout, checkConnectTimeout(fs, false)); } @Test @@ -79,20 +83,24 @@ public class TestHftpURLTimeouts { boolean timedout = false; HsftpFileSystem fs = (HsftpFileSystem)FileSystem.get(uri, conf); -HttpURLConnection conn = null; -timedout = false; try { - // this will consume the only slot in the backlog - conn = fs.openConnection(/, ); -} catch (SocketTimeoutException ste) { - // SSL expects a negotiation, so it will timeout on read, unlike hftp - timedout = true; - assertEquals(Read timed out, ste.getMessage()); + HttpURLConnection conn = null; + timedout = false; + try { +// this will consume the only slot in the backlog +conn = fs.openConnection(/, ); + } catch (SocketTimeoutException ste) { +// SSL expects a negotiation, so it will timeout on read, unlike hftp +timedout = true; +assertEquals(Read timed out, ste.getMessage()); + } finally { +if (conn != null) conn.disconnect(); + } + assertTrue(ssl read connect timedout, timedout); + assertTrue(connect timedout, checkConnectTimeout(fs, true)); } finally { - if (conn != null) conn.disconnect(); + fs.close(); } -assertTrue(ssl read connect timedout, timedout); -assertTrue(connect timedout, checkConnectTimeout(fs, true)); } private boolean checkConnectTimeout(HftpFileSystem fs, boolean ignoreReadTimeout)
svn commit: r1394610 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
Author: tgraves Date: Fri Oct 5 15:52:18 2012 New Revision: 1394610 URL: http://svn.apache.org/viewvc?rev=1394610view=rev Log: merge -r 1394608:1394609 from trunk. FIXES: HDFS-3829 Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1394610r1=1394609r2=1394610view=diff == --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Oct 5 15:52:18 2012 @@ -1525,6 +1525,9 @@ Release 0.23.5 - UNRELEASED BUG FIXES +HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java?rev=1394610r1=1394609r2=1394610view=diff == --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java Fri Oct 5 15:52:18 2012 @@ -53,19 +53,23 @@ public class TestHftpURLTimeouts { boolean timedout = false; HftpFileSystem fs = (HftpFileSystem)FileSystem.get(uri, conf); -HttpURLConnection conn = fs.openConnection(/, ); -timedout = false; try { - // this will consume the only slot in the backlog - conn.getInputStream(); -} catch (SocketTimeoutException ste) { - timedout = true; - assertEquals(Read timed out, ste.getMessage()); + HttpURLConnection conn = fs.openConnection(/, ); + timedout = false; + try { +// this will consume the only slot in the backlog +conn.getInputStream(); + } catch (SocketTimeoutException ste) { +timedout = true; +assertEquals(Read timed out, ste.getMessage()); + } finally { +if (conn != null) conn.disconnect(); + } + assertTrue(read timedout, timedout); + assertTrue(connect timedout, checkConnectTimeout(fs, false)); } finally { - if (conn != null) conn.disconnect(); + fs.close(); } -assertTrue(read timedout, timedout); -assertTrue(connect timedout, checkConnectTimeout(fs, false)); } @Test @@ -79,20 +83,24 @@ public class TestHftpURLTimeouts { boolean timedout = false; HsftpFileSystem fs = (HsftpFileSystem)FileSystem.get(uri, conf); -HttpURLConnection conn = null; -timedout = false; try { - // this will consume the only slot in the backlog - conn = fs.openConnection(/, ); -} catch (SocketTimeoutException ste) { - // SSL expects a negotiation, so it will timeout on read, unlike hftp - timedout = true; - assertEquals(Read timed out, ste.getMessage()); + HttpURLConnection conn = null; + timedout = false; + try { +// this will consume the only slot in the backlog +conn = fs.openConnection(/, ); + } catch (SocketTimeoutException ste) { +// SSL expects a negotiation, so it will timeout on read, unlike hftp +timedout = true; +assertEquals(Read timed out, ste.getMessage()); + } finally { +if (conn != null) conn.disconnect(); + } + assertTrue(ssl read connect timedout, timedout); + assertTrue(connect timedout, checkConnectTimeout(fs, true)); } finally { - if (conn != null) conn.disconnect(); + fs.close(); } -assertTrue(ssl read connect timedout, timedout); -assertTrue(connect timedout, checkConnectTimeout(fs, true)); } private boolean checkConnectTimeout(HftpFileSystem fs, boolean ignoreReadTimeout)
svn commit: r1394617 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
Author: tgraves Date: Fri Oct 5 16:01:09 2012 New Revision: 1394617 URL: http://svn.apache.org/viewvc?rev=1394617view=rev Log: merge -r 1394609:1394610 from branch-2. FIXES: HDFS-3829 Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1394617r1=1394616r2=1394617view=diff == --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Oct 5 16:01:09 2012 @@ -18,6 +18,9 @@ Release 0.23.5 - UNRELEASED HDFS-3905. Secure cluster cannot use hftp to an insecure cluster (Daryn Sharp via tgraves) +HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java?rev=1394617r1=1394616r2=1394617view=diff == --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java Fri Oct 5 16:01:09 2012 @@ -54,19 +54,23 @@ public class TestHftpURLTimeouts { boolean timedout = false; HftpFileSystem fs = (HftpFileSystem)FileSystem.get(uri, conf); -HttpURLConnection conn = fs.openConnection(/, ); -timedout = false; try { - // this will consume the only slot in the backlog - conn.getInputStream(); -} catch (SocketTimeoutException ste) { - timedout = true; - assertEquals(Read timed out, ste.getMessage()); + HttpURLConnection conn = fs.openConnection(/, ); + timedout = false; + try { +// this will consume the only slot in the backlog +conn.getInputStream(); + } catch (SocketTimeoutException ste) { +timedout = true; +assertEquals(Read timed out, ste.getMessage()); + } finally { +if (conn != null) conn.disconnect(); + } + assertTrue(read timedout, timedout); + assertTrue(connect timedout, checkConnectTimeout(fs, false)); } finally { - if (conn != null) conn.disconnect(); + fs.close(); } -assertTrue(read timedout, timedout); -assertTrue(connect timedout, checkConnectTimeout(fs, false)); } @Test @@ -80,20 +84,24 @@ public class TestHftpURLTimeouts { boolean timedout = false; HsftpFileSystem fs = (HsftpFileSystem)FileSystem.get(uri, conf); -HttpURLConnection conn = null; -timedout = false; try { - // this will consume the only slot in the backlog - conn = fs.openConnection(/, ); -} catch (SocketTimeoutException ste) { - // SSL expects a negotiation, so it will timeout on read, unlike hftp - timedout = true; - assertEquals(Read timed out, ste.getMessage()); + HttpURLConnection conn = null; + timedout = false; + try { +// this will consume the only slot in the backlog +conn = fs.openConnection(/, ); + } catch (SocketTimeoutException ste) { +// SSL expects a negotiation, so it will timeout on read, unlike hftp +timedout = true; +assertEquals(Read timed out, ste.getMessage()); + } finally { +if (conn != null) conn.disconnect(); + } + assertTrue(ssl read connect timedout, timedout); + assertTrue(connect timedout, checkConnectTimeout(fs, true)); } finally { - if (conn != null) conn.disconnect(); + fs.close(); } -assertTrue(ssl read connect timedout, timedout); -assertTrue(connect timedout, checkConnectTimeout(fs, true)); } private boolean checkConnectTimeout(HftpFileSystem fs, boolean ignoreReadTimeout)
svn commit: r1393699 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
Author: tgraves Date: Wed Oct 3 19:18:12 2012 New Revision: 1393699 URL: http://svn.apache.org/viewvc?rev=1393699view=rev Log: HDFS-3905. Secure cluster cannot use hftp to an insecure cluster (Daryn Sharp via tgraves) Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1393699r1=1393698r2=1393699view=diff == --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Oct 3 19:18:12 2012 @@ -15,6 +15,9 @@ Release 0.23.5 - UNRELEASED HDFS-3919. MiniDFSCluster:waitClusterUp can hang forever. (Andy Isaacson via eli) +HDFS-3905. Secure cluster cannot use hftp to an insecure cluster +(Daryn Sharp via tgraves) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=1393699r1=1393698r2=1393699view=diff == --- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java Wed Oct 3 19:18:12 2012 @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintStream; +import java.net.ConnectException; import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.URL; @@ -385,10 +386,24 @@ public class DelegationTokenFetcher { return ugi.doAs( new PrivilegedExceptionActionURLConnection() { public URLConnection run() throws IOException { - SecurityUtil.fetchServiceTicket(url); - URLConnection connection = URLUtils.openConnection(url); - connection.connect(); - return connection; + // might not be fatal if secure port doesn't connect because + // security may be disabled on remote cluster + IOException ioeForTGS = null; + try { +SecurityUtil.fetchServiceTicket(url); + } catch (IOException ioe) { +ioeForTGS = ioe; + } + try { +URLConnection connection = URLUtils.openConnection(url); +connection.connect(); +return connection; + } catch (ConnectException e) { +throw e; + } catch (IOException ioe) { +// throw TGS exception instead if negotiation fails +throw (ioeForTGS != null) ? ioeForTGS : ioe; + } } }); } catch (InterruptedException ie) {
svn commit: r1390557 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapred
Author: tgraves Date: Wed Sep 26 15:22:21 2012 New Revision: 1390557 URL: http://svn.apache.org/viewvc?rev=1390557view=rev Log: MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory in it. (Robert Evans via tgraves) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1390557r1=1390556r2=1390557view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Sep 26 15:22:21 2012 @@ -550,6 +550,9 @@ Release 0.23.4 - UNRELEASED BUG FIXES +MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory +in it. (Robert Evans via tgraves) + Release 0.23.3 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1390557r1=1390556r2=1390557view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Wed Sep 26 15:22:21 2012 @@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.JobContext; import org.apache.hadoop.mapred.MapReduceChildJVM; import org.apache.hadoop.mapred.ShuffleHandler; import org.apache.hadoop.mapred.Task; @@ -610,10 +611,12 @@ public abstract class TaskAttemptImpl im if (jobJar != null) { Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS .getUri(), remoteFS.getWorkingDirectory()); -localResources.put( -MRJobConfig.JOB_JAR, -createLocalResource(remoteFS, remoteJobJar, -LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION)); +LocalResource rc = createLocalResource(remoteFS, remoteJobJar, +LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION); +String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, +JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern(); +rc.setPattern(pattern); +localResources.put(MRJobConfig.JOB_JAR, rc); LOG.info(The job-jar file on the remote FS is + remoteJobJar.toUri().toASCIIString()); } else { Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1390557r1=1390556r2=1390557view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java Wed Sep 26 15:22:21 2012 @@ -152,6 +152,10 @@ class
svn commit: r1390560 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl
Author: tgraves Date: Wed Sep 26 15:24:03 2012 New Revision: 1390560 URL: http://svn.apache.org/viewvc?rev=1390560view=rev Log: MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory in it. (Robert Evans via tgraves) Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1390560r1=1390559r2=1390560view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Wed Sep 26 15:24:03 2012 @@ -23,6 +23,9 @@ Release 0.23.4 - UNRELEASED MAPREDUCE-4193. broken doc link for yarn-default.xml in site.xml. (phunt via tomwhite) +MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory +in it. (Robert Evans via tgraves) + Release 0.23.3 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1390560r1=1390559r2=1390560view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Wed Sep 26 15:24:03 2012 @@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.JobContext; import org.apache.hadoop.mapred.MapReduceChildJVM; import org.apache.hadoop.mapred.ShuffleHandler; import org.apache.hadoop.mapred.Task; @@ -601,10 +602,12 @@ public abstract class TaskAttemptImpl im if (jobJar != null) { Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS .getUri(), remoteFS.getWorkingDirectory()); -localResources.put( -MRJobConfig.JOB_JAR, -createLocalResource(remoteFS, remoteJobJar, -LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION)); +LocalResource rc = createLocalResource(remoteFS, remoteJobJar, +LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION); +String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, +JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern(); +rc.setPattern(pattern); +localResources.put(MRJobConfig.JOB_JAR, rc); LOG.info(The job-jar file on the remote FS is + remoteJobJar.toUri().toASCIIString()); } else { Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1390560r1=1390559r2=1390560view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce
svn commit: r1390563 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ h
Author: tgraves Date: Wed Sep 26 15:25:05 2012 New Revision: 1390563 URL: http://svn.apache.org/viewvc?rev=1390563view=rev Log: merge -r 1390556:1390557 from trunk. FIXES: MAPREDUCE-4647 Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1390563r1=1390562r2=1390563view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Wed Sep 26 15:25:05 2012 @@ -422,6 +422,9 @@ Release 0.23.4 - UNRELEASED BUG FIXES +MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory +in it. (Robert Evans via tgraves) + Release 0.23.3 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1390563r1=1390562r2=1390563view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Wed Sep 26 15:25:05 2012 @@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.JobContext; import org.apache.hadoop.mapred.MapReduceChildJVM; import org.apache.hadoop.mapred.ShuffleHandler; import org.apache.hadoop.mapred.Task; @@ -610,10 +611,12 @@ public abstract class TaskAttemptImpl im if (jobJar != null) { Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS .getUri(), remoteFS.getWorkingDirectory()); -localResources.put( -MRJobConfig.JOB_JAR, -createLocalResource(remoteFS, remoteJobJar, -LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION)); +LocalResource rc = createLocalResource(remoteFS, remoteJobJar, +LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION); +String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, +JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern(); +rc.setPattern(pattern); +localResources.put(MRJobConfig.JOB_JAR, rc); LOG.info(The job-jar file on the remote FS is + remoteJobJar.toUri().toASCIIString()); } else { Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1390563r1=1390562r2=1390563view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client
svn commit: r1390557 - in /hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn: hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/a
Author: tgraves Date: Wed Sep 26 15:22:21 2012 New Revision: 1390557 URL: http://svn.apache.org/viewvc?rev=1390557view=rev Log: MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory in it. (Robert Evans via tgraves) Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java?rev=1390557r1=1390556r2=1390557view=diff == --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java Wed Sep 26 15:22:21 2012 @@ -106,4 +106,20 @@ public interface LocalResource { * localized */ public void setVisibility(LocalResourceVisibility visibility); + + /** + * Get the empattern/em that should be used to extract entries from the + * archive (only used when type is codePATTERN/code). + * @return empattern/em that should be used to extract entries from the + * archive. + */ + public String getPattern(); + + /** + * Set the empattern/em that should be used to extract entries from the + * archive (only used when type is codePATTERN/code). + * @param pattern empattern/em that should be used to extract entries + * from the archive. + */ + public void setPattern(String pattern); } Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java?rev=1390557r1=1390556r2=1390557view=diff == --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java
svn commit: r1390479 - in /hadoop/common/site/main: author/src/documentation/content/xdocs/who.xml publish/who.html publish/who.pdf
Author: tgraves Date: Wed Sep 26 13:40:34 2012 New Revision: 1390479 URL: http://svn.apache.org/viewvc?rev=1390479view=rev Log: add tgraves to PMC list Modified: hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml hadoop/common/site/main/publish/who.html hadoop/common/site/main/publish/who.pdf Modified: hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml URL: http://svn.apache.org/viewvc/hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml?rev=1390479r1=1390478r2=1390479view=diff == --- hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml (original) +++ hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml Wed Sep 26 13:40:34 2012 @@ -257,6 +257,14 @@ /tr tr + tdtgraves/td + tdThomas Graves/td + tdYahoo!/td + td/td + td-6/td +/tr + +tr tdtodd/td tda href=http://people.apache.org/~todd;Todd Lipcon/a/td tdCloudera/td Modified: hadoop/common/site/main/publish/who.html URL: http://svn.apache.org/viewvc/hadoop/common/site/main/publish/who.html?rev=1390479r1=1390478r2=1390479view=diff == --- hadoop/common/site/main/publish/who.html (original) +++ hadoop/common/site/main/publish/who.html Wed Sep 26 13:40:34 2012 @@ -572,6 +572,17 @@ document.write(Last Published: + docu tr +td colspan=1 rowspan=1tgraves/td + td colspan=1 rowspan=1Thomas Graves/td + td colspan=1 rowspan=1Yahoo!/td + td colspan=1 rowspan=1/td + td colspan=1 rowspan=1-6/td + +/tr + + +tr + td colspan=1 rowspan=1todd/td td colspan=1 rowspan=1a href=http://people.apache.org/~todd;Todd Lipcon/a/td td colspan=1 rowspan=1Cloudera/td @@ -640,7 +651,7 @@ document.write(Last Published: + docu /div -a name=N103C6/aa name=Emeritus+Hadoop+PMC+Members/a +a name=N103E1/aa name=Emeritus+Hadoop+PMC+Members/a h2 class=h3Emeritus Hadoop PMC Members/h2 div class=section ul @@ -655,7 +666,7 @@ document.write(Last Published: + docu /div -a name=N103D9/aa name=Hadoop+Committers/a +a name=N103F4/aa name=Hadoop+Committers/a h2 class=h3Hadoop Committers/h2 div class=section pHadoop's active committers include:/p @@ -1270,7 +1281,7 @@ document.write(Last Published: + docu /div -a name=N10A10/aa name=Emeritus+Hadoop+Committers/a +a name=N10A2B/aa name=Emeritus+Hadoop+Committers/a h2 class=h3Emeritus Hadoop Committers/h2 div class=section pHadoop committers who are no longer active include:/p Modified: hadoop/common/site/main/publish/who.pdf URL: http://svn.apache.org/viewvc/hadoop/common/site/main/publish/who.pdf?rev=1390479r1=1390478r2=1390479view=diff == Binary files - no diff available.
svn commit: r1389888 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapred/JobHistory.java
Author: tgraves Date: Tue Sep 25 14:38:17 2012 New Revision: 1389888 URL: http://svn.apache.org/viewvc?rev=1389888view=rev Log: MAPREDUCE-4662. JobHistoryFilesManager thread pool never expands (Kihwal Lee via tgraves) Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobHistory.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1389888r1=1389887r2=1389888view=diff == --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Tue Sep 25 14:38:17 2012 @@ -254,6 +254,9 @@ Release 1.2.0 - unreleased MAPREDUCE-4652. ValueAggregatorJob sets the wrong job jar. (Ahmed Radwan via tomwhite) +MAPREDUCE-4662. JobHistoryFilesManager thread pool never expands +(Kihwal Lee via tgraves) + Release 1.1.0 - unreleased INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobHistory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=1389888r1=1389887r2=1389888view=diff == --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobHistory.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobHistory.java Tue Sep 25 14:38:17 2012 @@ -221,8 +221,11 @@ public class JobHistory { void start() { - executor = new ThreadPoolExecutor(1, 3, 1, + executor = new ThreadPoolExecutor(5, 5, 1, TimeUnit.HOURS, new LinkedBlockingQueueRunnable()); + // make core threads to terminate if there has been no work + // for the keppalive period. + executor.allowCoreThreadTimeOut(true); } private FilesHolder getFileHolder(JobID id) {
svn commit: r1383430 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
Author: tgraves Date: Tue Sep 11 14:34:27 2012 New Revision: 1383430 URL: http://svn.apache.org/viewvc?rev=1383430view=rev Log: MAPREDUCE-4576. Large dist cache can block tasktracker heartbeat (Robert Evans via tgraves). Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1383430r1=1383429r2=1383430view=diff == --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Tue Sep 11 14:34:27 2012 @@ -233,6 +233,9 @@ Release 1.2.0 - unreleased HADOOP-8781. hadoop-config.sh should add JAVA_LIBRARY_PATH to LD_LIBRARY_PATH. (tucu) +MAPREDUCE-4576. Large dist cache can block tasktracker heartbeat +(Robert Evans via tgraves) + Release 1.1.0 - unreleased INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java?rev=1383430r1=1383429r2=1383430view=diff == --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java Tue Sep 11 14:34:27 2012 @@ -35,6 +35,7 @@ import java.util.Random; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -566,7 +567,8 @@ public class TrackerDistributedCacheMana // long size; //the size of this cache. boolean inited = false; // is it initialized ? - +private final ReentrantLock lock = new ReentrantLock(); + // // The following five fields are Immutable. // @@ -598,14 +600,20 @@ public class TrackerDistributedCacheMana this.key = key; } -public synchronized void incRefCount() { - refcount.incrementAndGet() ; - LOG.debug(localizedLoadPath + : refcount= + refcount.get()); +public void incRefCount() { + lock.lock(); + try { +refcount.incrementAndGet() ; +LOG.debug(localizedLoadPath + : refcount= + refcount.get()); + } finally { +lock.unlock(); + } } public void decRefCount() { synchronized (cachedArchives) { -synchronized (this) { +lock.lock(); +try { refcount.decrementAndGet() ; LOG.debug(localizedLoadPath + : refcount= + refcount.get()); if(refcount.get() = 0) { @@ -613,6 +621,8 @@ public class TrackerDistributedCacheMana cachedArchives.remove(key); cachedArchives.put(key, this); } +} finally { + lock.unlock(); } } } @@ -621,9 +631,14 @@ public class TrackerDistributedCacheMana return refcount.get(); } -public synchronized boolean isUsed() { - LOG.debug(localizedLoadPath + : refcount= + refcount.get()); - return refcount.get() 0; +public boolean isUsed() { + lock.lock(); + try { +LOG.debug(localizedLoadPath + : refcount= + refcount.get()); +return refcount.get() 0; + } finally { +lock.unlock(); + } } Path getBaseDir(){ @@ -1027,19 +1042,24 @@ public class TrackerDistributedCacheMana CacheDir leftToClean = toBeCleanedBaseDir.get(cacheStatus.getBaseDir()); if (leftToClean != null (leftToClean.size 0 || leftToClean.subdirs 0)) { -synchronized (cacheStatus) { - // if reference count is zero mark the cache for deletion - boolean isUsed = cacheStatus.isUsed(); - long cacheSize = cacheStatus.size; - LOG.debug(cacheStatus.getLocalizedUniqueDir() + : isUsed= + isUsed + - size= + cacheSize + leftToClean.size= + leftToClean.size); - if (!isUsed) { -leftToClean.size -= cacheSize; -leftToClean.subdirs--; -// delete this cache entry from the global list -// and mark the localized file for deletion -toBeDeletedCache.add(cacheStatus); -it.remove(); +boolean gotLock = cacheStatus.lock.tryLock(); +if (gotLock) { + try { +// if reference count is zero mark the cache for deletion +boolean isUsed
svn commit: r1381459 - in /hadoop/common/trunk/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containerma
Author: tgraves Date: Thu Sep 6 02:08:37 2012 New Revision: 1381459 URL: http://svn.apache.org/viewvc?rev=1381459view=rev Log: YARN-87. NM ResourceLocalizationService does not set permissions of local cache directories (Jason Lowe via tgraves) Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1381459r1=1381458r2=1381459view=diff == --- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Thu Sep 6 02:08:37 2012 @@ -114,3 +114,6 @@ Release 0.23.3 - Unreleased YARN-68. NodeManager will refuse to shutdown indefinitely due to container log aggregation (daryn via bobby) + +YARN-87. NM ResourceLocalizationService does not set permissions of local +cache directories (Jason Lowe via tgraves) Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1381459r1=1381458r2=1381459view=diff == --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Thu Sep 6 02:08:37 2012 @@ -181,6 +181,7 @@ public class ResourceLocalizationService try { // TODO queue deletions here, rather than NM init? FileContext lfs = getLocalFileContext(conf); + lfs.setUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK)); ListString localDirs = dirsHandler.getLocalDirs(); for (String localDir : localDirs) { // $local/usercache Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java?rev=1381459r1=1381458r2=1381459view=diff == --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java Thu Sep 6 02:08:37 2012 @@ -50,6 +50,7 @@ import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Options.ChecksumOpt; @@ -116,6 +117,7 @@ public class TestResourceLocalizationSer @Test public void testLocalizationInit() throws Exception { final Configuration conf = new Configuration(); +conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, 077); AsyncDispatcher dispatcher = new AsyncDispatcher(); dispatcher.init(new Configuration()); @@ -151,14 +153,18 @@ public class TestResourceLocalizationSer // initialize ResourceLocalizationService locService.init(conf); + final FsPermission defaultPerm = new FsPermission((short)0755); + // verify directory
svn commit: r1379599 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ hadoop-mapreduce-c
Author: tgraves Date: Fri Aug 31 20:43:46 2012 New Revision: 1379599 URL: http://svn.apache.org/viewvc?rev=1379599view=rev Log: MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert Evans via tgraves) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1379599r1=1379598r2=1379599view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Aug 31 20:43:46 2012 @@ -858,6 +858,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby) +MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert +Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1379599r1=1379598r2=1379599view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java Fri Aug 31 20:43:46 2012 @@ -99,8 +99,8 @@ public class JobHistoryEventHandler exte protected static final MapJobId, MetaInfo fileMap = Collections.JobId,MetaInfosynchronizedMap(new HashMapJobId,MetaInfo()); - // Has a signal (SIGTERM etc) been issued? - protected volatile boolean isSignalled = false; + // should job completion be force when the AM shuts down? + protected volatile boolean forceJobCompletion = false; public JobHistoryEventHandler(AppContext context, int startCount) { super(JobHistoryEventHandler); @@ -322,7 +322,7 @@ public class JobHistoryEventHandler exte // Process JobUnsuccessfulCompletionEvent for jobIds which still haven't // closed their event writers IteratorJobId jobIt = fileMap.keySet().iterator(); -if(isSignalled) { +if(forceJobCompletion) { while (jobIt.hasNext()) { JobId toClose = jobIt.next(); MetaInfo mi = fileMap.get(toClose); @@ -911,9 +911,9 @@ public class JobHistoryEventHandler exte return tmpFileName.substring(0, tmpFileName.length()-4); } - public void setSignalled(boolean isSignalled) { -this.isSignalled = isSignalled; -LOG.info(JobHistoryEventHandler notified that isSignalled was - + isSignalled); + public void setForcejobCompletion(boolean forceJobCompletion) { +this.forceJobCompletion = forceJobCompletion; +LOG.info(JobHistoryEventHandler notified that forceJobCompletion is + + forceJobCompletion); } } Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1379599r1=1379598r2=1379599view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original) +++ hadoop/common/trunk/hadoop-mapreduce
svn commit: r1379602 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ hadoop
Author: tgraves Date: Fri Aug 31 20:45:34 2012 New Revision: 1379602 URL: http://svn.apache.org/viewvc?rev=1379602view=rev Log: merge -r 1379598:1379599 from trunk. FIXES: MAPREDUCE-4611 Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1379602r1=1379601r2=1379602view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Fri Aug 31 20:45:34 2012 @@ -734,6 +734,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby) +MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert +Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1379602r1=1379601r2=1379602view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java Fri Aug 31 20:45:34 2012 @@ -99,8 +99,8 @@ public class JobHistoryEventHandler exte protected static final MapJobId, MetaInfo fileMap = Collections.JobId,MetaInfosynchronizedMap(new HashMapJobId,MetaInfo()); - // Has a signal (SIGTERM etc) been issued? - protected volatile boolean isSignalled = false; + // should job completion be force when the AM shuts down? + protected volatile boolean forceJobCompletion = false; public JobHistoryEventHandler(AppContext context, int startCount) { super(JobHistoryEventHandler); @@ -322,7 +322,7 @@ public class JobHistoryEventHandler exte // Process JobUnsuccessfulCompletionEvent for jobIds which still haven't // closed their event writers IteratorJobId jobIt = fileMap.keySet().iterator(); -if(isSignalled) { +if(forceJobCompletion) { while (jobIt.hasNext()) { JobId toClose = jobIt.next(); MetaInfo mi = fileMap.get(toClose); @@ -911,9 +911,9 @@ public class JobHistoryEventHandler exte return tmpFileName.substring(0, tmpFileName.length()-4); } - public void setSignalled(boolean isSignalled) { -this.isSignalled = isSignalled; -LOG.info(JobHistoryEventHandler notified that isSignalled was - + isSignalled); + public void setForcejobCompletion(boolean forceJobCompletion) { +this.forceJobCompletion = forceJobCompletion; +LOG.info(JobHistoryEventHandler notified that forceJobCompletion is + + forceJobCompletion); } } Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1379602r1=1379601r2=1379602view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project
svn commit: r1379604 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ had
Author: tgraves Date: Fri Aug 31 20:48:22 2012 New Revision: 1379604 URL: http://svn.apache.org/viewvc?rev=1379604view=rev Log: merge -r 1379601:1379602 from branch-2. FIXES: MAPREDUCE-4611 Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1379604r1=1379603r2=1379604view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri Aug 31 20:48:22 2012 @@ -426,6 +426,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby) +MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert +Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1379604r1=1379603r2=1379604view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java Fri Aug 31 20:48:22 2012 @@ -99,8 +99,8 @@ public class JobHistoryEventHandler exte protected static final MapJobId, MetaInfo fileMap = Collections.JobId,MetaInfosynchronizedMap(new HashMapJobId,MetaInfo()); - // Has a signal (SIGTERM etc) been issued? - protected volatile boolean isSignalled = false; + // should job completion be force when the AM shuts down? + protected volatile boolean forceJobCompletion = false; public JobHistoryEventHandler(AppContext context, int startCount) { super(JobHistoryEventHandler); @@ -322,7 +322,7 @@ public class JobHistoryEventHandler exte // Process JobUnsuccessfulCompletionEvent for jobIds which still haven't // closed their event writers IteratorJobId jobIt = fileMap.keySet().iterator(); -if(isSignalled) { +if(forceJobCompletion) { while (jobIt.hasNext()) { JobId toClose = jobIt.next(); MetaInfo mi = fileMap.get(toClose); @@ -911,9 +911,9 @@ public class JobHistoryEventHandler exte return tmpFileName.substring(0, tmpFileName.length()-4); } - public void setSignalled(boolean isSignalled) { -this.isSignalled = isSignalled; -LOG.info(JobHistoryEventHandler notified that isSignalled was - + isSignalled); + public void setForcejobCompletion(boolean forceJobCompletion) { +this.forceJobCompletion = forceJobCompletion; +LOG.info(JobHistoryEventHandler notified that forceJobCompletion is + + forceJobCompletion); } } Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1379604r1=1379603r2=1379604view=diff
svn commit: r1378614 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Author: tgraves Date: Wed Aug 29 15:38:40 2012 New Revision: 1378614 URL: http://svn.apache.org/viewvc?rev=1378614view=rev Log: MAPREDUCE-4499. Looking for speculative tasks is very expensive in 1.x (Koji Noguchi via tgraves) Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1378614r1=1378613r2=1378614view=diff == --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Wed Aug 29 15:38:40 2012 @@ -103,6 +103,9 @@ Release 1.2.0 - unreleased MAPREDUCE-4408. allow jobs to set a JAR that is in the distributed cached (rkanter via tucu) +MAPREDUCE-4499. Looking for speculative tasks is very expensive in 1.x +(Koji Noguchi via tgraves) + OPTIMIZATIONS HDFS-2533. Backport: Remove needless synchronization on some FSDataSet Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1378614r1=1378613r2=1378614view=diff == --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java Wed Aug 29 15:38:40 2012 @@ -2246,21 +2246,19 @@ public class JobInProgress { continue; } - if (!tip.hasRunOnMachine(ttStatus.getHost(), + if (tip.hasSpeculativeTask(currentTime, avgProgress)) { +// Check if this tip can be removed from the list. +// If the list is shared then we should not remove. +if(shouldRemove){ + iter.remove(); +} +if (!tip.hasRunOnMachine(ttStatus.getHost(), ttStatus.getTrackerName())) { -if (tip.hasSpeculativeTask(currentTime, avgProgress)) { - // In case of shared list we don't remove it. Since the TIP failed - // on this tracker can be scheduled on some other tracker. - if (shouldRemove) { -iter.remove(); //this tracker is never going to run it again - } return tip; -} +} } else { -// Check if this tip can be removed from the list. -// If the list is shared then we should not remove. -if (shouldRemove) { - // This tracker will never speculate this tip +if (shouldRemove tip.hasRunOnMachine(ttStatus.getHost(), + ttStatus.getTrackerName())) { iter.remove(); } }
svn commit: r1378357 - in /hadoop/common/branches/branch-1: CHANGES.txt src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
Author: tgraves Date: Tue Aug 28 21:28:39 2012 New Revision: 1378357 URL: http://svn.apache.org/viewvc?rev=1378357view=rev Log: MAPREDUCE-1684. ClusterStatus can be cached in CapacityTaskScheduler.assignTasks() (Koji Noguchi via tgraves) Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1378357r1=1378356r2=1378357view=diff == --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Tue Aug 28 21:28:39 2012 @@ -213,6 +213,9 @@ Release 1.2.0 - unreleased MAPREDUCE-4595. TestLostTracker failing - possibly due to a race in JobHistory.JobHistoryFilesManager#run() (kkambatl via tucu) +MAPREDUCE-1684. ClusterStatus can be cached in +CapacityTaskScheduler.assignTasks() (Koji Noguchi via tgraves) + Release 1.1.0 - unreleased INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-1/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java?rev=1378357r1=1378356r2=1378357view=diff == --- hadoop/common/branches/branch-1/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java (original) +++ hadoop/common/branches/branch-1/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java Tue Aug 28 21:28:39 2012 @@ -154,7 +154,8 @@ class CapacityTaskScheduler extends Task protected TaskType type = null; abstract TaskLookupResult obtainNewTask(TaskTrackerStatus taskTracker, -JobInProgress job, boolean assignOffSwitch) throws IOException; +JobInProgress job, boolean assignOffSwitch, +ClusterStatus clusterStatus) throws IOException; int getSlotsOccupied(JobInProgress job) { return (getNumReservedTaskTrackers(job) + getRunningTasks(job)) * @@ -293,7 +294,8 @@ class CapacityTaskScheduler extends Task private TaskLookupResult getTaskFromQueue(TaskTracker taskTracker, int availableSlots, CapacitySchedulerQueue queue, - boolean assignOffSwitch) + boolean assignOffSwitch, + ClusterStatus clusterStatus) throws IOException { TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus(); // we only look at jobs in the running queues, as these are the ones @@ -320,7 +322,8 @@ class CapacityTaskScheduler extends Task availableSlots)) { // We found a suitable job. Get task from it. TaskLookupResult tlr = -obtainNewTask(taskTrackerStatus, j, assignOffSwitch); +obtainNewTask(taskTrackerStatus, j, assignOffSwitch, + clusterStatus); //if there is a task return it immediately. if (tlr.getLookUpStatus() == TaskLookupResult.LookUpStatus.LOCAL_TASK_FOUND || @@ -379,6 +382,11 @@ class CapacityTaskScheduler extends Task printQueues(); + //MAPREDUCE-1684: somehow getClusterStatus seems to be expensive. Caching + //here to reuse during the scheduling + ClusterStatus clusterStatus = +scheduler.taskTrackerManager.getClusterStatus(); + // Check if this tasktracker has been reserved for a job... JobInProgress job = taskTracker.getJobForFallowSlot(type); if (job != null) { @@ -397,7 +405,7 @@ class CapacityTaskScheduler extends Task // Don't care about locality! job.overrideSchedulingOpportunities(); } - return obtainNewTask(taskTrackerStatus, job, true); + return obtainNewTask(taskTrackerStatus, job, true, clusterStatus); } else { // Re-reserve the current tasktracker taskTracker.reserveSlots(type, job, availableSlots); @@ -420,7 +428,8 @@ class CapacityTaskScheduler extends Task } TaskLookupResult tlr = - getTaskFromQueue(taskTracker, availableSlots, queue, assignOffSwitch); + getTaskFromQueue(taskTracker, availableSlots, queue, assignOffSwitch, + clusterStatus); TaskLookupResult.LookUpStatus lookUpStatus = tlr.getLookUpStatus(); if (lookUpStatus
svn commit: r1375834 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/util/
Author: tgraves Date: Tue Aug 21 22:38:46 2012 New Revision: 1375834 URL: http://svn.apache.org/viewvc?rev=1375834view=rev Log: HADOOP-8239. Add subclasses of MD5MD5CRC32FileChecksum to support file checksum with CRC32C. (Kihwal Lee via szetszwo) Added: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32GzipFileChecksum.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1375834r1=1375833r2=1375834view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Tue Aug 21 22:38:46 2012 @@ -36,6 +36,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8240. Add a new API to allow users to specify a checksum type on FileSystem.create(..). (Kihwal Lee via szetszwo) +HADOOP-8239. Add subclasses of MD5MD5CRC32FileChecksum to support file +checksum with CRC32C. (Kihwal Lee via szetszwo) + OPTIMIZATIONS BUG FIXES Added: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java?rev=1375834view=auto == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java (added) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java Tue Aug 21 22:38:46 2012 @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs; + +import org.apache.hadoop.io.MD5Hash; +import org.apache.hadoop.util.DataChecksum; + +/** For CRC32 with the Castagnoli polynomial */ +public class MD5MD5CRC32CastagnoliFileChecksum extends MD5MD5CRC32FileChecksum { + /** Same as this(0, 0, null) */ + public MD5MD5CRC32CastagnoliFileChecksum() { +this(0, 0, null); + } + + /** Create a MD5FileChecksum */ + public MD5MD5CRC32CastagnoliFileChecksum(int bytesPerCRC, long crcPerBlock, MD5Hash md5) { +super(bytesPerCRC, crcPerBlock, md5); + } + + @Override + public DataChecksum.Type getCrcType() { +// default to the one that is understood by all releases. +return DataChecksum.Type.CRC32C; + } +} Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java?rev=1375834r1=1375833r2=1375834view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java Tue Aug 21 22:38:46 2012 @@ -23,12 +23,17 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience
svn commit: r1374407 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/ had
Author: tgraves Date: Fri Aug 17 19:47:35 2012 New Revision: 1374407 URL: http://svn.apache.org/viewvc?rev=1374407view=rev Log: MAPREDUCE-4549. Distributed cache conflicts breaks backwards compatability (Robert Evans via tgraves) Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1374407r1=1374406r2=1374407view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri Aug 17 19:47:35 2012 @@ -392,6 +392,9 @@ Release 0.23.3 - UNRELEASED for compatibility reasons is creating incorrect counter name. (Jarek Jarcec Cecho via tomwhite) +MAPREDUCE-4549. Distributed cache conflicts breaks backwards compatability +(Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1374407r1=1374406r2=1374407view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java Fri Aug 17 19:47:35 2012 @@ -30,6 +30,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -48,14 +50,15 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.yarn.ContainerLogAppender; import org.apache.hadoop.yarn.YarnException; -import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; /** * Helper class for MR applications @@ -63,6 +66,8 @@ import org.apache.hadoop.yarn.util.Build @Private @Unstable public class MRApps extends Apps { + private static final Log LOG = LogFactory.getLog(MRApps.class); + public static String toString(JobId jid) { return jid.toString(); } @@ -263,6 +268,23 @@ public class MRApps extends Apps { DistributedCache.getFileClassPaths(conf)); } + private static String getResourceDescription(LocalResourceType type) { +if(type == LocalResourceType.ARCHIVE) { + return cache archive ( + MRJobConfig.CACHE_ARCHIVES + ) ; +} +return cache file ( + MRJobConfig.CACHE_FILES + ) ; + } + + private static String toString(org.apache.hadoop.yarn.api.records.URL url) { +StringBuffer b = new StringBuffer(); +b.append(url.getScheme()).append(://).append(url.getHost()); +if(url.getPort() = 0) { + b.append(:).append(url.getPort()); +} +b.append(url.getFile()); +return b.toString(); + } + // TODO - Move this to MR! // Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[], // long[], boolean[], Path[], FileType) @@ -308,6 +330,17 @@ public class MRApps extends Apps { throw new IllegalArgumentException(Resource name must be relative); } String linkName = name.toUri().getPath(); +LocalResource orig
svn commit: r1374424 - /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
Author: tgraves Date: Fri Aug 17 20:29:38 2012 New Revision: 1374424 URL: http://svn.apache.org/viewvc?rev=1374424view=rev Log: YARN-25. remove old aggregated logs (Robert Evans via tgraves) Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1374424r1=1374423r2=1374424view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java Fri Aug 17 20:29:38 2012 @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.YarnExcept import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService; import org.apache.hadoop.yarn.service.CompositeService; /** @@ -53,6 +54,7 @@ public class JobHistoryServer extends Co private HistoryClientService clientService; private JobHistory jobHistoryService; private JHSDelegationTokenSecretManager jhsDTSecretManager; + private AggregatedLogDeletionService aggLogDelService; public JobHistoryServer() { super(JobHistoryServer.class.getName()); @@ -74,8 +76,10 @@ public class JobHistoryServer extends Co this.jhsDTSecretManager = createJHSSecretManager(conf); clientService = new HistoryClientService(historyContext, this.jhsDTSecretManager); +aggLogDelService = new AggregatedLogDeletionService(); addService(jobHistoryService); addService(clientService); +addService(aggLogDelService); super.init(config); }
svn commit: r1374425 - /hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.jav
Author: tgraves Date: Fri Aug 17 20:31:09 2012 New Revision: 1374425 URL: http://svn.apache.org/viewvc?rev=1374425view=rev Log: merge -r 1374423:1374424 from trunk. FIXES: YARN-25 Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1374425r1=1374424r2=1374425view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java Fri Aug 17 20:31:09 2012 @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.YarnExcept import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService; import org.apache.hadoop.yarn.service.CompositeService; /** @@ -53,6 +54,7 @@ public class JobHistoryServer extends Co private HistoryClientService clientService; private JobHistory jobHistoryService; private JHSDelegationTokenSecretManager jhsDTSecretManager; + private AggregatedLogDeletionService aggLogDelService; public JobHistoryServer() { super(JobHistoryServer.class.getName()); @@ -74,8 +76,10 @@ public class JobHistoryServer extends Co this.jhsDTSecretManager = createJHSSecretManager(conf); clientService = new HistoryClientService(historyContext, this.jhsDTSecretManager); +aggLogDelService = new AggregatedLogDeletionService(); addService(jobHistoryService); addService(clientService); +addService(aggLogDelService); super.init(config); }
svn commit: r1374426 - /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.
Author: tgraves Date: Fri Aug 17 20:32:22 2012 New Revision: 1374426 URL: http://svn.apache.org/viewvc?rev=1374426view=rev Log: merge -r 1374424:1374425 from branch-2. FIXES: YARN-25 Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1374426r1=1374425r2=1374426view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java Fri Aug 17 20:32:22 2012 @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.YarnExcept import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService; import org.apache.hadoop.yarn.service.CompositeService; /** @@ -53,6 +54,7 @@ public class JobHistoryServer extends Co private HistoryClientService clientService; private JobHistory jobHistoryService; private JHSDelegationTokenSecretManager jhsDTSecretManager; + private AggregatedLogDeletionService aggLogDelService; public JobHistoryServer() { super(JobHistoryServer.class.getName()); @@ -74,8 +76,10 @@ public class JobHistoryServer extends Co this.jhsDTSecretManager = createJHSSecretManager(conf); clientService = new HistoryClientService(historyContext, this.jhsDTSecretManager); +aggLogDelService = new AggregatedLogDeletionService(); addService(jobHistoryService); addService(clientService); +addService(aggLogDelService); super.init(config); }
svn commit: r1374427 - /hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory
Author: tgraves Date: Fri Aug 17 20:32:42 2012 New Revision: 1374427 URL: http://svn.apache.org/viewvc?rev=1374427view=rev Log: merge -r 1374423:1374424 from trunk. FIXES: YARN-25 Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1374427r1=1374426r2=1374427view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java Fri Aug 17 20:32:42 2012 @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.YarnExcept import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService; import org.apache.hadoop.yarn.service.CompositeService; /** @@ -53,6 +54,7 @@ public class JobHistoryServer extends Co private HistoryClientService clientService; private JobHistory jobHistoryService; private JHSDelegationTokenSecretManager jhsDTSecretManager; + private AggregatedLogDeletionService aggLogDelService; public JobHistoryServer() { super(JobHistoryServer.class.getName()); @@ -74,8 +76,10 @@ public class JobHistoryServer extends Co this.jhsDTSecretManager = createJHSSecretManager(conf); clientService = new HistoryClientService(historyContext, this.jhsDTSecretManager); +aggLogDelService = new AggregatedLogDeletionService(); addService(jobHistoryService); addService(clientService); +addService(aggLogDelService); super.init(config); }
svn commit: r1374314 - in /hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Author: tgraves Date: Fri Aug 17 15:20:03 2012 New Revision: 1374314 URL: http://svn.apache.org/viewvc?rev=1374314view=rev Log: merge -r 1374311:1374312 from trunk. FIXES: HADOOP-8692 Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374314r1=1374313r2=1374314view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 15:20:03 2012 @@ -654,6 +654,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor Robinson via tgraves) +HADOOP-8692. TestLocalDirAllocator fails intermittently with JDK7 +(Trevor Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java?rev=1374314r1=1374313r2=1374314view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java Fri Aug 17 15:20:03 2012 @@ -343,14 +343,18 @@ public class TestLocalDirAllocator { @Test public void testRemoveContext() throws IOException { String dir = buildBufferDir(ROOT, 0); -String contextCfgItemName = application_1340842292563_0004.app.cache.dirs; -conf.set(contextCfgItemName, dir); -LocalDirAllocator localDirAllocator = new LocalDirAllocator( -contextCfgItemName); -localDirAllocator.getLocalPathForWrite(p1/x, SMALL_FILE_SIZE, conf); -assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName)); -LocalDirAllocator.removeContext(contextCfgItemName); -assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName)); +try { + String contextCfgItemName = application_1340842292563_0004.app.cache.dirs; + conf.set(contextCfgItemName, dir); + LocalDirAllocator localDirAllocator = new LocalDirAllocator( + contextCfgItemName); + localDirAllocator.getLocalPathForWrite(p1/x, SMALL_FILE_SIZE, conf); + assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName)); + LocalDirAllocator.removeContext(contextCfgItemName); + assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName)); +} finally { + rmBufferDirs(); +} } }
svn commit: r1374316 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Author: tgraves Date: Fri Aug 17 15:21:48 2012 New Revision: 1374316 URL: http://svn.apache.org/viewvc?rev=1374316view=rev Log: merge -r 1374312:1374313 from branch-2. FIXES: HADOOP-8692 Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374316r1=1374315r2=1374316view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 15:21:48 2012 @@ -159,6 +159,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor Robinson via tgraves) +HADOOP-8692. TestLocalDirAllocator fails intermittently with JDK7 +(Trevor Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java?rev=1374316r1=1374315r2=1374316view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java Fri Aug 17 15:21:48 2012 @@ -340,14 +340,18 @@ public class TestLocalDirAllocator { @Test public void testRemoveContext() throws IOException { String dir = buildBufferDir(ROOT, 0); -String contextCfgItemName = application_1340842292563_0004.app.cache.dirs; -conf.set(contextCfgItemName, dir); -LocalDirAllocator localDirAllocator = new LocalDirAllocator( -contextCfgItemName); -localDirAllocator.getLocalPathForWrite(p1/x, SMALL_FILE_SIZE, conf); -assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName)); -LocalDirAllocator.removeContext(contextCfgItemName); -assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName)); +try { + String contextCfgItemName = application_1340842292563_0004.app.cache.dirs; + conf.set(contextCfgItemName, dir); + LocalDirAllocator localDirAllocator = new LocalDirAllocator( + contextCfgItemName); + localDirAllocator.getLocalPathForWrite(p1/x, SMALL_FILE_SIZE, conf); + assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName)); + LocalDirAllocator.removeContext(contextCfgItemName); + assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName)); +} finally { + rmBufferDirs(); +} } }
svn commit: r1374322 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Author: tgraves Date: Fri Aug 17 15:32:50 2012 New Revision: 1374322 URL: http://svn.apache.org/viewvc?rev=1374322view=rev Log: HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor Robinson via tgraves) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374322r1=1374321r2=1374322view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 15:32:50 2012 @@ -919,6 +919,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8692. TestLocalDirAllocator fails intermittently with JDK7 (Trevor Robinson via tgraves) +HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java?rev=1374322r1=1374321r2=1374322view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java Fri Aug 17 15:32:50 2012 @@ -137,6 +137,7 @@ public class TestSecurityUtil { @Test public void testBuildDTServiceName() { +SecurityUtil.setTokenServiceUseIp(true); assertEquals(127.0.0.1:123, SecurityUtil.buildDTServiceName(URI.create(test://LocalHost), 123) ); @@ -153,6 +154,7 @@ public class TestSecurityUtil { @Test public void testBuildTokenServiceSockAddr() { +SecurityUtil.setTokenServiceUseIp(true); assertEquals(127.0.0.1:123, SecurityUtil.buildTokenService(new InetSocketAddress(LocalHost, 123)).toString() );
svn commit: r1374323 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Author: tgraves Date: Fri Aug 17 15:37:00 2012 New Revision: 1374323 URL: http://svn.apache.org/viewvc?rev=1374323view=rev Log: merge -r 1374321:1374322 from trunk. FIXES: HADOOP-8693 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374323r1=1374322r2=1374323view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 15:37:00 2012 @@ -739,6 +739,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8692. TestLocalDirAllocator fails intermittently with JDK7 (Trevor Robinson via tgraves) +HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java?rev=1374323r1=1374322r2=1374323view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java Fri Aug 17 15:37:00 2012 @@ -137,6 +137,7 @@ public class TestSecurityUtil { @Test public void testBuildDTServiceName() { +SecurityUtil.setTokenServiceUseIp(true); assertEquals(127.0.0.1:123, SecurityUtil.buildDTServiceName(URI.create(test://LocalHost), 123) ); @@ -153,6 +154,7 @@ public class TestSecurityUtil { @Test public void testBuildTokenServiceSockAddr() { +SecurityUtil.setTokenServiceUseIp(true); assertEquals(127.0.0.1:123, SecurityUtil.buildTokenService(new InetSocketAddress(LocalHost, 123)).toString() );
svn commit: r1374325 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Author: tgraves Date: Fri Aug 17 15:38:10 2012 New Revision: 1374325 URL: http://svn.apache.org/viewvc?rev=1374325view=rev Log: merge -r 1374322:1374323 from branch-2. FIXES: HADOOP-8693 Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374325r1=1374324r2=1374325view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 15:38:10 2012 @@ -162,6 +162,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8692. TestLocalDirAllocator fails intermittently with JDK7 (Trevor Robinson via tgraves) +HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java?rev=1374325r1=1374324r2=1374325view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java Fri Aug 17 15:38:10 2012 @@ -137,6 +137,7 @@ public class TestSecurityUtil { @Test public void testBuildDTServiceName() { +SecurityUtil.setTokenServiceUseIp(true); assertEquals(127.0.0.1:123, SecurityUtil.buildDTServiceName(URI.create(test://LocalHost), 123) ); @@ -153,6 +154,7 @@ public class TestSecurityUtil { @Test public void testBuildTokenServiceSockAddr() { +SecurityUtil.setTokenServiceUseIp(true); assertEquals(127.0.0.1:123, SecurityUtil.buildTokenService(new InetSocketAddress(LocalHost, 123)).toString() );
svn commit: r1374349 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/io/TestWritableName.java
Author: tgraves Date: Fri Aug 17 16:38:53 2012 New Revision: 1374349 URL: http://svn.apache.org/viewvc?rev=1374349view=rev Log: HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor Robinson via tgraves) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374349r1=1374348r2=1374349view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 16:38:53 2012 @@ -922,6 +922,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor Robinson via tgraves) +HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java?rev=1374349r1=1374348r2=1374349view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java Fri Aug 17 16:38:53 2012 @@ -83,6 +83,7 @@ public class TestWritableName extends Te Configuration conf = new Configuration(); String altName = testName + .alt; +WritableName.setName(SimpleWritable.class, testName); WritableName.addName(SimpleWritable.class, altName); Class? test = WritableName.getClass(altName, conf);
svn commit: r1374351 - in /hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/io/TestWritableName.java
Author: tgraves Date: Fri Aug 17 16:41:26 2012 New Revision: 1374351 URL: http://svn.apache.org/viewvc?rev=1374351view=rev Log: merge -r 1374348:1374349 from trunk. FIXES: HADOOP-8697 Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374351r1=1374350r2=1374351view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 16:41:26 2012 @@ -660,6 +660,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor Robinson via tgraves) +HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java?rev=1374351r1=1374350r2=1374351view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java Fri Aug 17 16:41:26 2012 @@ -83,6 +83,7 @@ public class TestWritableName extends Te Configuration conf = new Configuration(); String altName = testName + .alt; +WritableName.setName(SimpleWritable.class, testName); WritableName.addName(SimpleWritable.class, altName); Class? test = WritableName.getClass(altName, conf);
svn commit: r1374447 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Author: tgraves Date: Fri Aug 17 21:43:31 2012 New Revision: 1374447 URL: http://svn.apache.org/viewvc?rev=1374447view=rev Log: HADOOP-8695. TestPathData fails intermittently with JDK7 (Trevor Robinson via tgraves) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374447r1=1374446r2=1374447view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 21:43:31 2012 @@ -925,6 +925,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor Robinson via tgraves) +HADOOP-8695. TestPathData fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java?rev=1374447r1=1374446r2=1374447view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java Fri Aug 17 21:43:31 2012 @@ -26,23 +26,17 @@ import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.Test; public class TestPathData { - protected static Configuration conf; - protected static FileSystem fs; - protected static String dirString; - protected static Path testDir; - protected static PathData item; - - protected static String[] d1Paths = -new String[] { d1/f1, d1/f1.1, d1/f2 }; - protected static String[] d2Paths = -new String[] { d2/f3 }; - - @BeforeClass - public static void initialize() throws Exception { + protected Configuration conf; + protected FileSystem fs; + protected Path testDir; + + @Before + public void initialize() throws Exception { conf = new Configuration(); fs = FileSystem.getLocal(conf); testDir = new Path( @@ -60,23 +54,28 @@ public class TestPathData { fs.create(new Path(d2,f3)); } + @After + public void cleanup() throws Exception { +fs.close(); + } + @Test public void testWithDirStringAndConf() throws Exception { -dirString = d1; -item = new PathData(dirString, conf); -checkPathData(); +String dirString = d1; +PathData item = new PathData(dirString, conf); +checkPathData(dirString, item); // properly implementing symlink support in various commands will require // trailing slashes to be retained dirString = d1/; item = new PathData(dirString, conf); -checkPathData(); +checkPathData(dirString, item); } @Test public void testUnqualifiedUriContents() throws Exception { -dirString = d1; -item = new PathData(dirString, conf); +String dirString = d1; +PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(d1/f1, d1/f1.1, d1/f2), @@ -86,8 +85,8 @@ public class TestPathData { @Test public void testQualifiedUriContents() throws Exception { -dirString = fs.makeQualified(new Path(d1)).toString(); -item = new PathData(dirString, conf); +String dirString = fs.makeQualified(new Path(d1)).toString(); +PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(dirString+/f1, dirString+/f1.1, dirString+/f2), @@ -97,8 +96,8 @@ public class TestPathData { @Test public void testCwdContents() throws Exception { -dirString = Path.CUR_DIR; -item = new PathData(dirString, conf); +String dirString = Path.CUR_DIR; +PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(d1, d2), @@ -106,17 +105,16 @@ public class TestPathData { ); } - - @Test - public void testToFile() throws Exception { -item = new PathData(., conf); + @Test + public void testToFile() throws Exception { +PathData item = new PathData(., conf
svn commit: r1374449 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Author: tgraves Date: Fri Aug 17 21:44:35 2012 New Revision: 1374449 URL: http://svn.apache.org/viewvc?rev=1374449view=rev Log: merge -r 1374446:1374447 from trunk. FIXES: HADOOP-8695 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374449r1=1374448r2=1374449view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 21:44:35 2012 @@ -747,6 +747,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor Robinson via tgraves) +HADOOP-8695. TestPathData fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java?rev=1374449r1=1374448r2=1374449view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java Fri Aug 17 21:44:35 2012 @@ -26,23 +26,17 @@ import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.Test; public class TestPathData { - protected static Configuration conf; - protected static FileSystem fs; - protected static String dirString; - protected static Path testDir; - protected static PathData item; - - protected static String[] d1Paths = -new String[] { d1/f1, d1/f1.1, d1/f2 }; - protected static String[] d2Paths = -new String[] { d2/f3 }; - - @BeforeClass - public static void initialize() throws Exception { + protected Configuration conf; + protected FileSystem fs; + protected Path testDir; + + @Before + public void initialize() throws Exception { conf = new Configuration(); fs = FileSystem.getLocal(conf); testDir = new Path( @@ -60,23 +54,28 @@ public class TestPathData { fs.create(new Path(d2,f3)); } + @After + public void cleanup() throws Exception { +fs.close(); + } + @Test public void testWithDirStringAndConf() throws Exception { -dirString = d1; -item = new PathData(dirString, conf); -checkPathData(); +String dirString = d1; +PathData item = new PathData(dirString, conf); +checkPathData(dirString, item); // properly implementing symlink support in various commands will require // trailing slashes to be retained dirString = d1/; item = new PathData(dirString, conf); -checkPathData(); +checkPathData(dirString, item); } @Test public void testUnqualifiedUriContents() throws Exception { -dirString = d1; -item = new PathData(dirString, conf); +String dirString = d1; +PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(d1/f1, d1/f1.1, d1/f2), @@ -86,8 +85,8 @@ public class TestPathData { @Test public void testQualifiedUriContents() throws Exception { -dirString = fs.makeQualified(new Path(d1)).toString(); -item = new PathData(dirString, conf); +String dirString = fs.makeQualified(new Path(d1)).toString(); +PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(dirString+/f1, dirString+/f1.1, dirString+/f2), @@ -97,8 +96,8 @@ public class TestPathData { @Test public void testCwdContents() throws Exception { -dirString = Path.CUR_DIR; -item = new PathData(dirString, conf); +String dirString = Path.CUR_DIR; +PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(d1, d2), @@ -106,17 +105,16 @@ public class TestPathData { ); } - - @Test - public void testToFile() throws Exception { -item = new PathData(., conf); + @Test
svn commit: r1374450 - in /hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Author: tgraves Date: Fri Aug 17 21:45:41 2012 New Revision: 1374450 URL: http://svn.apache.org/viewvc?rev=1374450view=rev Log: merge -r 1374446:1374447 from trunk. FIXES: HADOOP-8695 Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374450r1=1374449r2=1374450view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 21:45:41 2012 @@ -665,6 +665,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor Robinson via tgraves) +HADOOP-8695. TestPathData fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java?rev=1374450r1=1374449r2=1374450view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java Fri Aug 17 21:45:41 2012 @@ -26,23 +26,17 @@ import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.Test; public class TestPathData { - protected static Configuration conf; - protected static FileSystem fs; - protected static String dirString; - protected static Path testDir; - protected static PathData item; - - protected static String[] d1Paths = -new String[] { d1/f1, d1/f1.1, d1/f2 }; - protected static String[] d2Paths = -new String[] { d2/f3 }; - - @BeforeClass - public static void initialize() throws Exception { + protected Configuration conf; + protected FileSystem fs; + protected Path testDir; + + @Before + public void initialize() throws Exception { conf = new Configuration(); fs = FileSystem.getLocal(conf); testDir = new Path( @@ -60,23 +54,28 @@ public class TestPathData { fs.create(new Path(d2,f3)); } + @After + public void cleanup() throws Exception { +fs.close(); + } + @Test public void testWithDirStringAndConf() throws Exception { -dirString = d1; -item = new PathData(dirString, conf); -checkPathData(); +String dirString = d1; +PathData item = new PathData(dirString, conf); +checkPathData(dirString, item); // properly implementing symlink support in various commands will require // trailing slashes to be retained dirString = d1/; item = new PathData(dirString, conf); -checkPathData(); +checkPathData(dirString, item); } @Test public void testUnqualifiedUriContents() throws Exception { -dirString = d1; -item = new PathData(dirString, conf); +String dirString = d1; +PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(d1/f1, d1/f1.1, d1/f2), @@ -86,8 +85,8 @@ public class TestPathData { @Test public void testQualifiedUriContents() throws Exception { -dirString = fs.makeQualified(new Path(d1)).toString(); -item = new PathData(dirString, conf); +String dirString = fs.makeQualified(new Path(d1)).toString(); +PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(dirString+/f1, dirString+/f1.1, dirString+/f2), @@ -97,8 +96,8 @@ public class TestPathData { @Test public void testCwdContents() throws Exception { -dirString = Path.CUR_DIR; -item = new PathData(dirString, conf); +String dirString = Path.CUR_DIR; +PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(d1, d2), @@ -106,17 +105,16 @@ public class TestPathData { ); } - - @Test
svn commit: r1374451 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Author: tgraves Date: Fri Aug 17 21:46:05 2012 New Revision: 1374451 URL: http://svn.apache.org/viewvc?rev=1374451view=rev Log: merge -r 1374448:1374449 from branch-2. FIXES: HADOOP-8695 Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374451r1=1374450r2=1374451view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 21:46:05 2012 @@ -168,6 +168,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor Robinson via tgraves) +HADOOP-8695. TestPathData fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java?rev=1374451r1=1374450r2=1374451view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java Fri Aug 17 21:46:05 2012 @@ -26,23 +26,17 @@ import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.Test; public class TestPathData { - protected static Configuration conf; - protected static FileSystem fs; - protected static String dirString; - protected static Path testDir; - protected static PathData item; - - protected static String[] d1Paths = -new String[] { d1/f1, d1/f1.1, d1/f2 }; - protected static String[] d2Paths = -new String[] { d2/f3 }; - - @BeforeClass - public static void initialize() throws Exception { + protected Configuration conf; + protected FileSystem fs; + protected Path testDir; + + @Before + public void initialize() throws Exception { conf = new Configuration(); fs = FileSystem.getLocal(conf); testDir = new Path( @@ -60,23 +54,28 @@ public class TestPathData { fs.create(new Path(d2,f3)); } + @After + public void cleanup() throws Exception { +fs.close(); + } + @Test public void testWithDirStringAndConf() throws Exception { -dirString = d1; -item = new PathData(dirString, conf); -checkPathData(); +String dirString = d1; +PathData item = new PathData(dirString, conf); +checkPathData(dirString, item); // properly implementing symlink support in various commands will require // trailing slashes to be retained dirString = d1/; item = new PathData(dirString, conf); -checkPathData(); +checkPathData(dirString, item); } @Test public void testUnqualifiedUriContents() throws Exception { -dirString = d1; -item = new PathData(dirString, conf); +String dirString = d1; +PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(d1/f1, d1/f1.1, d1/f2), @@ -86,8 +85,8 @@ public class TestPathData { @Test public void testQualifiedUriContents() throws Exception { -dirString = fs.makeQualified(new Path(d1)).toString(); -item = new PathData(dirString, conf); +String dirString = fs.makeQualified(new Path(d1)).toString(); +PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(dirString+/f1, dirString+/f1.1, dirString+/f2), @@ -97,8 +96,8 @@ public class TestPathData { @Test public void testCwdContents() throws Exception { -dirString = Path.CUR_DIR; -item = new PathData(dirString, conf); +String dirString = Path.CUR_DIR; +PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(d1, d2), @@ -106,17 +105,16 @@ public class TestPathData { ); } - - @Test - public void testToFile() throws Exception { -item = new
svn commit: r1374084 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Author: tgraves Date: Thu Aug 16 22:16:19 2012 New Revision: 1374084 URL: http://svn.apache.org/viewvc?rev=1374084view=rev Log: HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor Robinson via tgraves) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374084r1=1374083r2=1374084view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 16 22:16:19 2012 @@ -911,6 +911,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8703. distcpV2: turn CRC checking off for 0 byte size (Dave Thompson via bobby) +HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java?rev=1374084r1=1374083r2=1374084view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java Thu Aug 16 22:16:19 2012 @@ -18,18 +18,20 @@ package org.apache.hadoop.fs; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import java.io.IOException; import java.net.URI; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.NetUtilsTestResolver; import org.apache.hadoop.util.Progressable; +import org.junit.BeforeClass; import org.junit.Test; -public class TestFileSystemCanonicalization extends TestCase { +public class TestFileSystemCanonicalization { static String[] authorities = { myfs://host, myfs://host.a, @@ -41,8 +43,8 @@ public class TestFileSystemCanonicalizat }; - @Test - public void testSetupResolver() throws Exception { + @BeforeClass + public static void initialize() throws Exception { NetUtilsTestResolver.install(); }
svn commit: r1374085 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Author: tgraves Date: Thu Aug 16 22:19:23 2012 New Revision: 1374085 URL: http://svn.apache.org/viewvc?rev=1374085view=rev Log: merge -r 1374083:1374084 from trunk. FIXES: HADOOP-8390 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374085r1=1374084r2=1374085view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 16 22:19:23 2012 @@ -733,6 +733,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8703. distcpV2: turn CRC checking off for 0 byte size (Dave Thompson via bobby) +HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java?rev=1374085r1=1374084r2=1374085view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java Thu Aug 16 22:19:23 2012 @@ -18,18 +18,20 @@ package org.apache.hadoop.fs; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import java.io.IOException; import java.net.URI; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.NetUtilsTestResolver; import org.apache.hadoop.util.Progressable; +import org.junit.BeforeClass; import org.junit.Test; -public class TestFileSystemCanonicalization extends TestCase { +public class TestFileSystemCanonicalization { static String[] authorities = { myfs://host, myfs://host.a, @@ -41,8 +43,8 @@ public class TestFileSystemCanonicalizat }; - @Test - public void testSetupResolver() throws Exception { + @BeforeClass + public static void initialize() throws Exception { NetUtilsTestResolver.install(); }
svn commit: r1374087 - in /hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Author: tgraves Date: Thu Aug 16 22:21:56 2012 New Revision: 1374087 URL: http://svn.apache.org/viewvc?rev=1374087view=rev Log: merge -r 1374083:1374084 from trunk. FIXES: HADOOP-8390 Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374087r1=1374086r2=1374087view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 16 22:21:56 2012 @@ -651,6 +651,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8703. distcpV2: turn CRC checking off for 0 byte size (Dave Thompson via bobby) +HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java?rev=1374087r1=1374086r2=1374087view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java Thu Aug 16 22:21:56 2012 @@ -18,18 +18,20 @@ package org.apache.hadoop.fs; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import java.io.IOException; import java.net.URI; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.NetUtilsTestResolver; import org.apache.hadoop.util.Progressable; +import org.junit.BeforeClass; import org.junit.Test; -public class TestFileSystemCanonicalization extends TestCase { +public class TestFileSystemCanonicalization { static String[] authorities = { myfs://host, myfs://host.a, @@ -41,8 +43,8 @@ public class TestFileSystemCanonicalizat }; - @Test - public void testSetupResolver() throws Exception { + @BeforeClass + public static void initialize() throws Exception { NetUtilsTestResolver.install(); }
svn commit: r1374088 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Author: tgraves Date: Thu Aug 16 22:23:31 2012 New Revision: 1374088 URL: http://svn.apache.org/viewvc?rev=1374088view=rev Log: merge -r 1374084:1374085 from branch-2. FIXES: HADOOP-8390 Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374088r1=1374087r2=1374088view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 16 22:23:31 2012 @@ -156,6 +156,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8703. distcpV2: turn CRC checking off for 0 byte size (Dave Thompson via bobby) +HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java?rev=1374088r1=1374087r2=1374088view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java Thu Aug 16 22:23:31 2012 @@ -18,18 +18,20 @@ package org.apache.hadoop.fs; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import java.io.IOException; import java.net.URI; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.NetUtilsTestResolver; import org.apache.hadoop.util.Progressable; +import org.junit.BeforeClass; import org.junit.Test; -public class TestFileSystemCanonicalization extends TestCase { +public class TestFileSystemCanonicalization { static String[] authorities = { myfs://host, myfs://host.a, @@ -41,8 +43,8 @@ public class TestFileSystemCanonicalizat }; - @Test - public void testSetupResolver() throws Exception { + @BeforeClass + public static void initialize() throws Exception { NetUtilsTestResolver.install(); }
svn commit: r1372636 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/ hadoop-mapreduce-cl
Author: tgraves Date: Mon Aug 13 21:48:33 2012 New Revision: 1372636 URL: http://svn.apache.org/viewvc?rev=1372636view=rev Log: MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over group names deprecated names don't show up (Robert Evans via tgraves) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1372636r1=1372635r2=1372636view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Mon Aug 13 21:48:33 2012 @@ -809,6 +809,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason Lowe via bobby) +MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over +group names deprecated names don't show up (Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1372636r1=1372635r2=1372636view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java Mon Aug 13 21:48:33 2012 @@ -24,6 +24,7 @@ import static org.apache.hadoop.mapreduc import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentSkipListMap; @@ -185,7 +186,15 @@ public abstract class AbstractCountersC * @return Set of counter names. */ public synchronized IterableString getGroupNames() { -return Iterables.concat(fgroups.keySet(), groups.keySet()); +HashSetString deprecated = new HashSetString(); +for(Map.EntryString, String entry : legacyMap.entrySet()) { + String newGroup = entry.getValue(); + boolean isFGroup = isFrameworkGroup(newGroup); + if(isFGroup ? fgroups.containsKey(newGroup) : groups.containsKey(newGroup)) { +deprecated.add(entry.getKey()); + } +} +return Iterables.concat(fgroups.keySet(), groups.keySet(), deprecated); } @Override Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java?rev=1372636r1=1372635r2=1372636view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java Mon Aug 13 21:48:33 2012 @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTru import java.io.IOException; import java.text.ParseException; +import java.util.HashSet; import java.util.Iterator; import java.util.Random; @@ -225,6 +226,23 @@ public class TestCounters { } @Test + public void testLegacyGetGroupNames() { +Counters counters = new Counters(); +// create 2 filesystem counter groups +counters.findCounter(fs1, FileSystemCounter.BYTES_READ).increment(1); +counters.findCounter(fs2, FileSystemCounter.BYTES_READ).increment(1); +counters.incrCounter(group1, counter1, 1); + +HashSetString groups = new HashSetString(counters.getGroupNames()); +HashSetString expectedGroups = new HashSetString(); +expectedGroups.add(group1); +expectedGroups.add(FileSystemCounter); //Legacy Name
svn commit: r1372639 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/ hado
Author: tgraves Date: Mon Aug 13 21:51:06 2012 New Revision: 1372639 URL: http://svn.apache.org/viewvc?rev=1372639view=rev Log: merge -r 1372637:1372638 from branch-2. FIXES: MAPREDUCE-4053 Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1372639r1=1372638r2=1372639view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon Aug 13 21:51:06 2012 @@ -385,6 +385,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason Lowe via bobby) +MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over +group names deprecated names don't show up (Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1372639r1=1372638r2=1372639view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java Mon Aug 13 21:51:06 2012 @@ -24,6 +24,7 @@ import static org.apache.hadoop.mapreduc import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentSkipListMap; @@ -185,7 +186,15 @@ public abstract class AbstractCountersC * @return Set of counter names. */ public synchronized IterableString getGroupNames() { -return Iterables.concat(fgroups.keySet(), groups.keySet()); +HashSetString deprecated = new HashSetString(); +for(Map.EntryString, String entry : legacyMap.entrySet()) { + String newGroup = entry.getValue(); + boolean isFGroup = isFrameworkGroup(newGroup); + if(isFGroup ? fgroups.containsKey(newGroup) : groups.containsKey(newGroup)) { +deprecated.add(entry.getKey()); + } +} +return Iterables.concat(fgroups.keySet(), groups.keySet(), deprecated); } @Override Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java?rev=1372639r1=1372638r2=1372639view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java Mon Aug 13 21:51:06 2012 @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTru import java.io.IOException; import java.text.ParseException; +import java.util.HashSet; import java.util.Iterator; import java.util.Random; @@ -225,6 +226,23 @@ public class TestCounters { } @Test + public void testLegacyGetGroupNames() { +Counters counters = new Counters(); +// create 2 filesystem counter groups +counters.findCounter(fs1, FileSystemCounter.BYTES_READ).increment(1); +counters.findCounter(fs2, FileSystemCounter.BYTES_READ).increment(1); +counters.incrCounter(group1, counter1, 1); + +HashSetString groups = new HashSetString(counters.getGroupNames
svn commit: r1372640 - in /hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counter
Author: tgraves Date: Mon Aug 13 21:53:50 2012 New Revision: 1372640 URL: http://svn.apache.org/viewvc?rev=1372640view=rev Log: merge -r 1372635:1372636 from trunk. FIXES: MAPREDUCE-4053 Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt?rev=1372640r1=1372639r2=1372640view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt Mon Aug 13 21:53:50 2012 @@ -584,6 +584,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason Lowe via bobby) +MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over +group names deprecated names don't show up (Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1372640r1=1372639r2=1372640view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java Mon Aug 13 21:53:50 2012 @@ -24,6 +24,7 @@ import static org.apache.hadoop.mapreduc import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentSkipListMap; @@ -185,7 +186,15 @@ public abstract class AbstractCountersC * @return Set of counter names. */ public synchronized IterableString getGroupNames() { -return Iterables.concat(fgroups.keySet(), groups.keySet()); +HashSetString deprecated = new HashSetString(); +for(Map.EntryString, String entry : legacyMap.entrySet()) { + String newGroup = entry.getValue(); + boolean isFGroup = isFrameworkGroup(newGroup); + if(isFGroup ? fgroups.containsKey(newGroup) : groups.containsKey(newGroup)) { +deprecated.add(entry.getKey()); + } +} +return Iterables.concat(fgroups.keySet(), groups.keySet(), deprecated); } @Override Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java?rev=1372640r1=1372639r2=1372640view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java Mon Aug 13 21:53:50 2012 @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTru import java.io.IOException; import java.text.ParseException; +import java.util.HashSet; import java.util.Iterator; import java.util.Random; @@ -225,6 +226,23 @@ public class TestCounters { } @Test + public void testLegacyGetGroupNames() { +Counters counters = new Counters(); +// create 2 filesystem counter groups +counters.findCounter(fs1, FileSystemCounter.BYTES_READ).increment(1); +counters.findCounter(fs2, FileSystemCounter.BYTES_READ).increment(1); +counters.incrCounter
svn commit: r1368294 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ src/test/mapred/org/apache
Author: tgraves Date: Wed Aug 1 22:14:13 2012 New Revision: 1368294 URL: http://svn.apache.org/viewvc?rev=1368294view=rev Log: MAPREDUCE-4504. SortValidator writes to wrong directory (Robert Evans via tgraves) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1368294r1=1368293r2=1368294view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Aug 1 22:14:13 2012 @@ -789,6 +789,9 @@ Release 0.23.3 - UNRELEASED parallel use (due to a /tmp file with static name) (Robert Evans via jeagles) +MAPREDUCE-4504. SortValidator writes to wrong directory (Robert Evans +via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java?rev=1368294r1=1368293r2=1368294view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java Wed Aug 1 22:14:13 2012 @@ -344,7 +344,7 @@ public class SortValidator extends Confi FileInputFormat.setInputPaths(jobConf, sortInput); FileInputFormat.addInputPath(jobConf, sortOutput); - Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir, /tmp), + Path outputPath = new Path(new Path(/tmp, sortvalidate), UUID.randomUUID().toString()); if (defaultfs.exists(outputPath)) { defaultfs.delete(outputPath, true); Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java?rev=1368294r1=1368293r2=1368294view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java Wed Aug 1 22:14:13 2012 @@ -344,7 +344,7 @@ public class SortValidator extends Confi FileInputFormat.setInputPaths(jobConf, sortInput); FileInputFormat.addInputPath(jobConf, sortOutput); - Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir, /tmp), + Path outputPath = new Path(new Path(/tmp, sortvalidate), UUID.randomUUID().toString()); if (defaultfs.exists(outputPath)) { defaultfs.delete(outputPath, true);
svn commit: r1368296 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ src/test/mapre
Author: tgraves Date: Wed Aug 1 22:20:59 2012 New Revision: 1368296 URL: http://svn.apache.org/viewvc?rev=1368296view=rev Log: merge -r 1368293:1368294 from trunk. FIXES: MAPREDUCE-4504 Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1368296r1=1368295r2=1368296view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Wed Aug 1 22:20:59 2012 @@ -665,6 +665,9 @@ Release 0.23.3 - UNRELEASED parallel use (due to a /tmp file with static name) (Robert Evans via jeagles) +MAPREDUCE-4504. SortValidator writes to wrong directory (Robert Evans +via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java?rev=1368296r1=1368295r2=1368296view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java Wed Aug 1 22:20:59 2012 @@ -344,7 +344,7 @@ public class SortValidator extends Confi FileInputFormat.setInputPaths(jobConf, sortInput); FileInputFormat.addInputPath(jobConf, sortOutput); - Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir, /tmp), + Path outputPath = new Path(new Path(/tmp, sortvalidate), UUID.randomUUID().toString()); if (defaultfs.exists(outputPath)) { defaultfs.delete(outputPath, true); Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java?rev=1368296r1=1368295r2=1368296view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java Wed Aug 1 22:20:59 2012 @@ -344,7 +344,7 @@ public class SortValidator extends Confi FileInputFormat.setInputPaths(jobConf, sortInput); FileInputFormat.addInputPath(jobConf, sortOutput); - Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir, /tmp), + Path outputPath = new Path(new Path(/tmp, sortvalidate), UUID.randomUUID().toString()); if (defaultfs.exists(outputPath)) { defaultfs.delete(outputPath, true);
svn commit: r1368298 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ src/test/ma
Author: tgraves Date: Wed Aug 1 22:22:23 2012 New Revision: 1368298 URL: http://svn.apache.org/viewvc?rev=1368298view=rev Log: merge -r 1368295:1368296 from branch-2. FIXES: MAPREDUCE-4504 Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1368298r1=1368297r2=1368298view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Wed Aug 1 22:22:23 2012 @@ -379,6 +379,9 @@ Release 0.23.3 - UNRELEASED parallel use (due to a /tmp file with static name) (Robert Evans via jeagles) +MAPREDUCE-4504. SortValidator writes to wrong directory (Robert Evans +via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java?rev=1368298r1=1368297r2=1368298view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java Wed Aug 1 22:22:23 2012 @@ -344,7 +344,7 @@ public class SortValidator extends Confi FileInputFormat.setInputPaths(jobConf, sortInput); FileInputFormat.addInputPath(jobConf, sortOutput); - Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir, /tmp), + Path outputPath = new Path(new Path(/tmp, sortvalidate), UUID.randomUUID().toString()); if (defaultfs.exists(outputPath)) { defaultfs.delete(outputPath, true); Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java?rev=1368298r1=1368297r2=1368298view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java Wed Aug 1 22:22:23 2012 @@ -344,7 +344,7 @@ public class SortValidator extends Confi FileInputFormat.setInputPaths(jobConf, sortInput); FileInputFormat.addInputPath(jobConf, sortOutput); - Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir, /tmp), + Path outputPath = new Path(new Path(/tmp, sortvalidate), UUID.randomUUID().toString()); if (defaultfs.exists(outputPath)) { defaultfs.delete(outputPath, true);
svn commit: r1368002 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/shell/ src/test/java/org/apache/hadoop/fs/shell/
Author: tgraves Date: Wed Aug 1 14:02:22 2012 New Revision: 1368002 URL: http://svn.apache.org/viewvc?rev=1368002view=rev Log: HADOOP-8633. Interrupted FsShell copies may leave tmp files (Daryn Sharp via tgraves) Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1368002r1=1368001r2=1368002view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Aug 1 14:02:22 2012 @@ -873,6 +873,9 @@ Release 0.23.3 - UNRELEASED org.apache.hadoop.classification.InterfaceAudience not found (Trevor Robinson via tgraves) +HADOOP-8633. Interrupted FsShell copies may leave tmp files (Daryn Sharp +via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1368002r1=1368001r2=1368002view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Wed Aug 1 14:02:22 2012 @@ -24,6 +24,8 @@ import java.io.InputStream; import java.util.LinkedList; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException; import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; @@ -232,31 +234,65 @@ abstract class CommandWithDestination ex if (target.exists (target.stat.isDirectory() || !overwrite)) { throw new PathExistsException(target.toString()); } -target.fs.setWriteChecksum(writeChecksum); -PathData tempFile = null; +TargetFileSystem targetFs = new TargetFileSystem(target.fs); try { - tempFile = target.createTempFile(target+._COPYING_); - FSDataOutputStream out = target.fs.create(tempFile.path, true); - IOUtils.copyBytes(in, out, getConf(), true); + PathData tempTarget = target.suffix(._COPYING_); + targetFs.setWriteChecksum(writeChecksum); + targetFs.writeStreamToFile(in, tempTarget); + targetFs.rename(tempTarget, target); +} finally { + targetFs.close(); // last ditch effort to ensure temp file is removed +} + } + + // Helper filter filesystem that registers created files as temp files to + // be deleted on exit unless successfully renamed + private static class TargetFileSystem extends FilterFileSystem { +TargetFileSystem(FileSystem fs) { + super(fs); +} + +void writeStreamToFile(InputStream in, PathData target) throws IOException { + FSDataOutputStream out = null; + try { +out = create(target); +IOUtils.copyBytes(in, out, getConf(), true); + } finally { +IOUtils.closeStream(out); // just in case copyBytes didn't + } +} + +// tag created files as temp files +FSDataOutputStream create(PathData item) throws IOException { + try { +return create(item.path, true); + } finally { // might have been created but stream was interrupted +deleteOnExit(item.path); + } +} + +void rename(PathData src, PathData target) throws IOException { // the rename method with an option to delete the target is deprecated - if (target.exists !target.fs.delete(target.path, false)) { + if (target.exists !delete(target.path, false)) { // too bad we don't know why it failed PathIOException e = new PathIOException(target.toString()); e.setOperation(delete); throw e; } - if (!tempFile.fs.rename(tempFile.path, target.path)) { + if (!rename(src.path, target.path)) { // too bad we don't know why it failed -PathIOException e = new PathIOException(tempFile.toString()); +PathIOException e = new
svn commit: r1368003 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/shell/ src/test/java/org/apache/hadoop/fs/shell/
Author: tgraves Date: Wed Aug 1 14:03:01 2012 New Revision: 1368003 URL: http://svn.apache.org/viewvc?rev=1368003view=rev Log: merge -r 1368001:1368002 from trunk. FIXES: HADOOP-8633 Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java - copied unchanged from r1368002, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1368003r1=1368002r2=1368003view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Aug 1 14:03:01 2012 @@ -692,6 +692,9 @@ Release 0.23.3 - UNRELEASED org.apache.hadoop.classification.InterfaceAudience not found (Trevor Robinson via tgraves) +HADOOP-8633. Interrupted FsShell copies may leave tmp files (Daryn Sharp +via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1368003r1=1368002r2=1368003view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Wed Aug 1 14:03:01 2012 @@ -24,6 +24,8 @@ import java.io.InputStream; import java.util.LinkedList; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException; import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; @@ -232,31 +234,65 @@ abstract class CommandWithDestination ex if (target.exists (target.stat.isDirectory() || !overwrite)) { throw new PathExistsException(target.toString()); } -target.fs.setWriteChecksum(writeChecksum); -PathData tempFile = null; +TargetFileSystem targetFs = new TargetFileSystem(target.fs); try { - tempFile = target.createTempFile(target+._COPYING_); - FSDataOutputStream out = target.fs.create(tempFile.path, true); - IOUtils.copyBytes(in, out, getConf(), true); + PathData tempTarget = target.suffix(._COPYING_); + targetFs.setWriteChecksum(writeChecksum); + targetFs.writeStreamToFile(in, tempTarget); + targetFs.rename(tempTarget, target); +} finally { + targetFs.close(); // last ditch effort to ensure temp file is removed +} + } + + // Helper filter filesystem that registers created files as temp files to + // be deleted on exit unless successfully renamed + private static class TargetFileSystem extends FilterFileSystem { +TargetFileSystem(FileSystem fs) { + super(fs); +} + +void writeStreamToFile(InputStream in, PathData target) throws IOException { + FSDataOutputStream out = null; + try { +out = create(target); +IOUtils.copyBytes(in, out, getConf(), true); + } finally { +IOUtils.closeStream(out); // just in case copyBytes didn't + } +} + +// tag created files as temp files +FSDataOutputStream create(PathData item) throws IOException { + try { +return create(item.path, true); + } finally { // might have been created but stream was interrupted +deleteOnExit(item.path); + } +} + +void rename(PathData src, PathData target) throws IOException { // the rename method with an option to delete the target is deprecated - if (target.exists !target.fs.delete(target.path, false)) { + if (target.exists !delete(target.path, false)) { // too bad we don't know why it failed PathIOException e = new PathIOException(target.toString()); e.setOperation(delete); throw e
svn commit: r1368004 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/shell/ src/test/java/org/apache/hadoop/fs/shell/
Author: tgraves Date: Wed Aug 1 14:03:35 2012 New Revision: 1368004 URL: http://svn.apache.org/viewvc?rev=1368004view=rev Log: merge -r 1368002:1368003 from branch-2. FIXES: HADOOP-8633 Added: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java - copied unchanged from r1368003, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1368004r1=1368003r2=1368004view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Wed Aug 1 14:03:35 2012 @@ -146,6 +146,9 @@ Release 0.23.3 - UNRELEASED org.apache.hadoop.classification.InterfaceAudience not found (Trevor Robinson via tgraves) +HADOOP-8633. Interrupted FsShell copies may leave tmp files (Daryn Sharp +via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1368004r1=1368003r2=1368004view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Wed Aug 1 14:03:35 2012 @@ -24,6 +24,8 @@ import java.io.InputStream; import java.util.LinkedList; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException; import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; @@ -232,31 +234,65 @@ abstract class CommandWithDestination ex if (target.exists (target.stat.isDirectory() || !overwrite)) { throw new PathExistsException(target.toString()); } -target.fs.setWriteChecksum(writeChecksum); -PathData tempFile = null; +TargetFileSystem targetFs = new TargetFileSystem(target.fs); try { - tempFile = target.createTempFile(target+._COPYING_); - FSDataOutputStream out = target.fs.create(tempFile.path, true); - IOUtils.copyBytes(in, out, getConf(), true); + PathData tempTarget = target.suffix(._COPYING_); + targetFs.setWriteChecksum(writeChecksum); + targetFs.writeStreamToFile(in, tempTarget); + targetFs.rename(tempTarget, target); +} finally { + targetFs.close(); // last ditch effort to ensure temp file is removed +} + } + + // Helper filter filesystem that registers created files as temp files to + // be deleted on exit unless successfully renamed + private static class TargetFileSystem extends FilterFileSystem { +TargetFileSystem(FileSystem fs) { + super(fs); +} + +void writeStreamToFile(InputStream in, PathData target) throws IOException { + FSDataOutputStream out = null; + try { +out = create(target); +IOUtils.copyBytes(in, out, getConf(), true); + } finally { +IOUtils.closeStream(out); // just in case copyBytes didn't + } +} + +// tag created files as temp files +FSDataOutputStream create(PathData item) throws IOException { + try { +return create(item.path, true); + } finally { // might have been created but stream was interrupted +deleteOnExit(item.path); + } +} + +void rename(PathData src, PathData target) throws IOException { // the rename method with an option to delete the target is deprecated - if (target.exists !target.fs.delete(target.path, false)) { + if (target.exists !delete(target.path, false)) { // too bad we don't know why it failed PathIOException e = new PathIOException(target.toString
svn commit: r1367539 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapred
Author: tgraves Date: Tue Jul 31 13:39:41 2012 New Revision: 1367539 URL: http://svn.apache.org/viewvc?rev=1367539view=rev Log: MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby via tgraves) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367539r1=1367538r2=1367539view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 13:39:41 2012 @@ -505,6 +505,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4267. mavenize pipes (tgraves via bobby) +MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby +via tgraves) + OPTIMIZATIONS MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367539r1=1367538r2=1367539view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue Jul 31 13:39:41 2012 @@ -1561,7 +1561,7 @@ public class JobImpl implements org.apac Path confPath = getConfFile(); FileContext fc = FileContext.getFileContext(confPath.toUri(), conf); Configuration jobConf = new Configuration(false); -jobConf.addResource(fc.open(confPath)); +jobConf.addResource(fc.open(confPath), confPath.toString()); return jobConf; } } Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1367539r1=1367538r2=1367539view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java Tue Jul 31 13:39:41 2012 @@ -78,14 +78,29 @@ public class ConfBlock extends HtmlBlock tr(). th(_TH, key). th(_TH, value). +th(_TH, source chain). _(). _(). tbody(); for (ConfEntryInfo entry : info.getProperties()) { +StringBuffer buffer = new StringBuffer(); +String[] sources = entry.getSource(); +//Skip the last entry, because it is always the same HDFS file, and +// output them in reverse order so most recent is output first +boolean first = true; +for(int i
svn commit: r1367540 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ h
Author: tgraves Date: Tue Jul 31 13:40:39 2012 New Revision: 1367540 URL: http://svn.apache.org/viewvc?rev=1367540view=rev Log: merge -r 1367538:1367539 from trunk. FIXES: MAPREDUCE-4375 Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367540r1=1367539r2=1367540view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 13:40:39 2012 @@ -381,6 +381,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4267. mavenize pipes (tgraves via bobby) +MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby +via tgraves) + OPTIMIZATIONS MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367540r1=1367539r2=1367540view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue Jul 31 13:40:39 2012 @@ -1561,7 +1561,7 @@ public class JobImpl implements org.apac Path confPath = getConfFile(); FileContext fc = FileContext.getFileContext(confPath.toUri(), conf); Configuration jobConf = new Configuration(false); -jobConf.addResource(fc.open(confPath)); +jobConf.addResource(fc.open(confPath), confPath.toString()); return jobConf; } } Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1367540r1=1367539r2=1367540view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java Tue Jul 31 13:40:39 2012 @@ -78,14 +78,29 @@ public class ConfBlock extends HtmlBlock tr(). th(_TH, key). th(_TH, value). +th(_TH, source chain). _(). _(). tbody(); for (ConfEntryInfo entry : info.getProperties()) { +StringBuffer buffer = new StringBuffer
svn commit: r1367544 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl
Author: tgraves Date: Tue Jul 31 13:44:50 2012 New Revision: 1367544 URL: http://svn.apache.org/viewvc?rev=1367544view=rev Log: merge -r 1367539:1367540 from branch-2. FIXES: MAPREDUCE-4375 Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367544r1=1367543r2=1367544view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 13:44:50 2012 @@ -56,6 +56,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-2220. Fix new API FileOutputFormat-related typos in mapred-default.xml (Rui Kubo via harsh) +MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby +via tgraves) + OPTIMIZATIONS MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367544r1=1367543r2=1367544view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue Jul 31 13:44:50 2012 @@ -1481,7 +1481,7 @@ public class JobImpl implements org.apac Path confPath = getConfFile(); FileContext fc = FileContext.getFileContext(confPath.toUri(), conf); Configuration jobConf = new Configuration(false); -jobConf.addResource(fc.open(confPath)); +jobConf.addResource(fc.open(confPath), confPath.toString()); return jobConf; } } Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1367544r1=1367543r2=1367544view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java Tue Jul 31 13:44:50 2012 @@ -78,14 +78,29 @@ public class ConfBlock extends HtmlBlock tr(). th(_TH, key). th(_TH, value). +th(_TH, source chain). _(). _(). tbody
svn commit: r1367581 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoo
Author: tgraves Date: Tue Jul 31 14:46:57 2012 New Revision: 1367581 URL: http://svn.apache.org/viewvc?rev=1367581view=rev Log: MAPREDUCE-4456. LocalDistributedCacheManager can get an ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves) Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367581r1=1367580r2=1367581view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 14:46:57 2012 @@ -766,6 +766,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4423. Potential infinite fetching of map output (Robert Evans via tgraves) +MAPREDUCE-4456. LocalDistributedCacheManager can get an +ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367581r1=1367580r2=1367581view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java Tue Jul 31 14:46:57 2012 @@ -18,12 +18,9 @@ package org.apache.hadoop.mapred; -import com.google.common.collect.Maps; - import java.io.File; import java.io.IOException; import java.net.MalformedURLException; -import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLClassLoader; @@ -34,6 +31,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -60,6 +58,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.FSDownload; +import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; /** @@ -85,6 +84,9 @@ class LocalDistributedCacheManager { * @throws IOException */ public void setup(JobConf conf) throws IOException { +boolean mkLinks = DistributedCache.getSymlink(conf); +File workDir = new File(System.getProperty(user.dir)); + // Generate YARN local resources objects corresponding to the distributed // cache configuration MapString, LocalResource localResources = @@ -132,7 +134,8 @@ class LocalDistributedCacheManager { FuturePath future = exec.submit(download); resourcesToPaths.put(resource, future); } - for (LocalResource resource : localResources.values()) { + for (EntryString, LocalResource entry : localResources.entrySet()) { +LocalResource resource = entry.getValue(); Path path; try { path = resourcesToPaths.get(resource).get(); @@ -142,6 +145,12 @@ class LocalDistributedCacheManager { throw new IOException(e); } String pathString = path.toUri().toString(); +if(mkLinks) { + String link = entry.getKey(); + String target = new File(path.toUri()).getPath(); + symlink(workDir, target, link); +} + if (resource.getType() == LocalResourceType.ARCHIVE) { localArchives.add(pathString); } else if (resource.getType() == LocalResourceType.FILE) { @@ -175,27 +184,6 @@ class LocalDistributedCacheManager { .arrayToString(localFiles.toArray(new String[localArchives .size
svn commit: r1367585 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-
Author: tgraves Date: Tue Jul 31 14:49:07 2012 New Revision: 1367585 URL: http://svn.apache.org/viewvc?rev=1367585view=rev Log: merge -r 1367580:1367581 from trunk. FIXES: MAPREDUCE-4456 Added: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java - copied unchanged from r1367581, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367585r1=1367584r2=1367585view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 14:49:07 2012 @@ -642,6 +642,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4423. Potential infinite fetching of map output (Robert Evans via tgraves) +MAPREDUCE-4456. LocalDistributedCacheManager can get an +ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367585r1=1367584r2=1367585view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java Tue Jul 31 14:49:07 2012 @@ -18,12 +18,9 @@ package org.apache.hadoop.mapred; -import com.google.common.collect.Maps; - import java.io.File; import java.io.IOException; import java.net.MalformedURLException; -import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLClassLoader; @@ -34,6 +31,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -60,6 +58,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.FSDownload; +import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; /** @@ -85,6 +84,9 @@ class LocalDistributedCacheManager { * @throws IOException */ public void setup(JobConf conf) throws IOException { +boolean mkLinks = DistributedCache.getSymlink(conf); +File workDir = new File(System.getProperty(user.dir)); + // Generate YARN local resources objects corresponding to the distributed // cache configuration MapString, LocalResource localResources = @@ -132,7 +134,8 @@ class LocalDistributedCacheManager { FuturePath future = exec.submit(download); resourcesToPaths.put(resource, future); } - for (LocalResource resource : localResources.values()) { + for (EntryString, LocalResource entry : localResources.entrySet()) { +LocalResource resource = entry.getValue(); Path path; try { path = resourcesToPaths.get(resource).get(); @@ -142,6 +145,12 @@ class LocalDistributedCacheManager { throw new IOException(e); } String pathString = path.toUri().toString(); +if(mkLinks) { + String link = entry.getKey(); + String target = new File(path.toUri()).getPath(); + symlink(workDir, target, link); +} + if (resource.getType() == LocalResourceType.ARCHIVE
svn commit: r1367587 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapredu
Author: tgraves Date: Tue Jul 31 14:50:27 2012 New Revision: 1367587 URL: http://svn.apache.org/viewvc?rev=1367587view=rev Log: merge -r 1367584:1367585 from branch-2. FIXES: MAPREDUCE-4456 Added: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java - copied unchanged from r1367585, hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367587r1=1367586r2=1367587view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 14:50:27 2012 @@ -358,6 +358,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4010. TestWritableJobConf fails on trunk (tucu via bobby) +MAPREDUCE-4456. LocalDistributedCacheManager can get an +ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367587r1=1367586r2=1367587view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java Tue Jul 31 14:50:27 2012 @@ -18,12 +18,9 @@ package org.apache.hadoop.mapred; -import com.google.common.collect.Maps; - import java.io.File; import java.io.IOException; import java.net.MalformedURLException; -import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLClassLoader; @@ -34,6 +31,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -60,6 +58,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.FSDownload; +import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; /** @@ -85,6 +84,9 @@ class LocalDistributedCacheManager { * @throws IOException */ public void setup(JobConf conf) throws IOException { +boolean mkLinks = DistributedCache.getSymlink(conf); +File workDir = new File(System.getProperty(user.dir)); + // Generate YARN local resources objects corresponding to the distributed // cache configuration MapString, LocalResource localResources = @@ -132,7 +134,8 @@ class LocalDistributedCacheManager { FuturePath future = exec.submit(download); resourcesToPaths.put(resource, future); } - for (LocalResource resource : localResources.values()) { + for (EntryString, LocalResource entry : localResources.entrySet()) { +LocalResource resource = entry.getValue(); Path path; try { path = resourcesToPaths.get(resource).get(); @@ -142,6 +145,12 @@ class LocalDistributedCacheManager { throw new IOException(e); } String pathString = path.toUri().toString(); +if(mkLinks) { + String link = entry.getKey(); + String target = new File(path.toUri()).getPath(); + symlink(workDir, target, link); +} + if (resource.getType
svn commit: r1367715 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-
Author: tgraves Date: Tue Jul 31 19:21:27 2012 New Revision: 1367715 URL: http://svn.apache.org/viewvc?rev=1367715view=rev Log: merge -r 1367712:1367713 from trunk. FIXES: MAPREDUCE-4493 Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367715r1=1367714r2=1367715view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 19:21:27 2012 @@ -647,6 +647,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4496. AM logs link is missing user name (Jason Lowe via bobby) +MAPREDUCE-4493. Distibuted Cache Compatability Issues (Robert Evans +via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367715r1=1367714r2=1367715view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java Tue Jul 31 19:21:27
svn commit: r1367717 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapredu
Author: tgraves Date: Tue Jul 31 19:22:25 2012 New Revision: 1367717 URL: http://svn.apache.org/viewvc?rev=1367717view=rev Log: merge -r 1367714:1367715 from branch-2. FIXES: MAPREDUCE-4493 Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367717r1=1367716r2=1367717view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 19:22:25 2012 @@ -363,6 +363,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4496. AM logs link is missing user name (Jason Lowe via bobby) +MAPREDUCE-4493. Distibuted Cache Compatability Issues (Robert Evans +via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367717r1=1367716r2=1367717view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java
svn commit: r1367771 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapred
Author: tgraves Date: Tue Jul 31 20:52:41 2012 New Revision: 1367771 URL: http://svn.apache.org/viewvc?rev=1367771view=rev Log: MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at FAILED (Robert Evans via tgraves) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367771r1=1367770r2=1367771view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 20:52:41 2012 @@ -779,6 +779,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4492. Configuring total queue capacity between 100.5 and 99.5 at perticular level is sucessfull (Mayank Bansal via bobby) +MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at +FAILED (Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367771r1=1367770r2=1367771view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue Jul 31 20:52:41 2012 @@ -1370,7 +1370,8 @@ public class JobImpl implements org.apac } } -float failureRate = (float) fetchFailures / runningReduceTasks; +float failureRate = runningReduceTasks == 0 ? 1.0f : + (float) fetchFailures / runningReduceTasks; // declare faulty if fetch-failures = max-allowed-failures boolean isMapFaulty = (failureRate = MAX_ALLOWED_FETCH_FAILURES_FRACTION); Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1367771r1=1367770r2=1367771view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Tue Jul 31 20:52:41 2012 @@ -435,7 +435,8 @@ public abstract class TaskAttemptImpl im TaskAttemptEventType.TA_CONTAINER_CLEANED, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, - TaskAttemptEventType.TA_FAILMSG)) + TaskAttemptEventType.TA_FAILMSG, + TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE)) // Transitions from KILLED state .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED, Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java?rev=1367771r1=1367770r2=1367771view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop
svn commit: r1367772 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ h
Author: tgraves Date: Tue Jul 31 20:53:45 2012 New Revision: 1367772 URL: http://svn.apache.org/viewvc?rev=1367772view=rev Log: merge -r 1367770:1367771 from trunk. FIXES: MAPREDUCE-4457 Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367772r1=1367771r2=1367772view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 20:53:45 2012 @@ -655,6 +655,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4492. Configuring total queue capacity between 100.5 and 99.5 at perticular level is sucessfull (Mayank Bansal via bobby) +MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at +FAILED (Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367772r1=1367771r2=1367772view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue Jul 31 20:53:45 2012 @@ -1370,7 +1370,8 @@ public class JobImpl implements org.apac } } -float failureRate = (float) fetchFailures / runningReduceTasks; +float failureRate = runningReduceTasks == 0 ? 1.0f : + (float) fetchFailures / runningReduceTasks; // declare faulty if fetch-failures = max-allowed-failures boolean isMapFaulty = (failureRate = MAX_ALLOWED_FETCH_FAILURES_FRACTION); Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1367772r1=1367771r2=1367772view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Tue Jul 31 20:53:45 2012 @@ -435,7 +435,8 @@ public abstract class TaskAttemptImpl im TaskAttemptEventType.TA_CONTAINER_CLEANED, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, - TaskAttemptEventType.TA_FAILMSG)) + TaskAttemptEventType.TA_FAILMSG, + TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE)) // Transitions from KILLED state .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED, Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java?rev=1367772r1=1367771r2=1367772view
svn commit: r1367774 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl
Author: tgraves Date: Tue Jul 31 20:54:45 2012 New Revision: 1367774 URL: http://svn.apache.org/viewvc?rev=1367774view=rev Log: merge -r 1367771:1367772 from branch-2. FIXES: MAPREDUCE-4457 Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367774r1=1367773r2=1367774view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 20:54:45 2012 @@ -369,6 +369,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4492. Configuring total queue capacity between 100.5 and 99.5 at perticular level is sucessfull (Mayank Bansal via bobby) +MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at +FAILED (Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367774r1=1367773r2=1367774view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue Jul 31 20:54:45 2012 @@ -1305,7 +1305,8 @@ public class JobImpl implements org.apac } } -float failureRate = (float) fetchFailures / runningReduceTasks; +float failureRate = runningReduceTasks == 0 ? 1.0f : + (float) fetchFailures / runningReduceTasks; // declare faulty if fetch-failures = max-allowed-failures boolean isMapFaulty = (failureRate = MAX_ALLOWED_FETCH_FAILURES_FRACTION); Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1367774r1=1367773r2=1367774view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Tue Jul 31 20:54:45 2012 @@ -431,7 +431,8 @@ public abstract class TaskAttemptImpl im TaskAttemptEventType.TA_CONTAINER_CLEANED, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, - TaskAttemptEventType.TA_FAILMSG)) + TaskAttemptEventType.TA_FAILMSG, + TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE)) // Transitions from KILLED state .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED, Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl
svn commit: r1367713 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm
Author: tgraves Date: Tue Jul 31 19:20:03 2012 New Revision: 1367713 URL: http://svn.apache.org/viewvc?rev=1367713view=rev Log: MAPREDUCE-4493. Distibuted Cache Compatability Issues (Robert Evans via tgraves) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm?rev=1367713r1=1367712r2=1367713view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm Tue Jul 31 19:20:03 2012 @@ -196,7 +196,9 @@ Deprecated Properties *---+---+ |mapred.compress.map.output | mapreduce.map.output.compress *---+---+ -|mapred.create.symlink | mapreduce.job.cache.symlink.create +|mapred.create.symlink | NONE - symlinking is always on +*---+---+ +|mapreduce.job.cache.symlink.create | NONE - symlinking is always on *---+---+ |mapred.data.field.separator | mapreduce.fieldsel.data.field.separator *---+---+
svn commit: r1367713 - /hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Author: tgraves Date: Tue Jul 31 19:20:03 2012 New Revision: 1367713 URL: http://svn.apache.org/viewvc?rev=1367713view=rev Log: MAPREDUCE-4493. Distibuted Cache Compatability Issues (Robert Evans via tgraves) Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1367713r1=1367712r2=1367713view=diff == --- hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java (original) +++ hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Tue Jul 31 19:20:03 2012 @@ -958,7 +958,6 @@ public class StreamJob implements Tool { if (!b) fail(LINK_URI); } -DistributedCache.createSymlink(jobConf_); // set the jobconf for the caching parameters if (cacheArchives != null) DistributedCache.setCacheArchives(archiveURIs, jobConf_);
svn commit: r1367715 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm
Author: tgraves Date: Tue Jul 31 19:21:27 2012 New Revision: 1367715 URL: http://svn.apache.org/viewvc?rev=1367715view=rev Log: merge -r 1367712:1367713 from trunk. FIXES: MAPREDUCE-4493 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm?rev=1367715r1=1367714r2=1367715view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm Tue Jul 31 19:21:27 2012 @@ -198,7 +198,9 @@ Deprecated Properties *---+---+ |mapred.compress.map.output | mapreduce.map.output.compress *---+---+ -|mapred.create.symlink | mapreduce.job.cache.symlink.create +|mapred.create.symlink | NONE - symlinking is always on +*---+---+ +|mapreduce.job.cache.symlink.create | NONE - symlinking is always on *---+---+ |mapred.data.field.separator | mapreduce.fieldsel.data.field.separator *---+---+
svn commit: r1367715 - /hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Author: tgraves Date: Tue Jul 31 19:21:27 2012 New Revision: 1367715 URL: http://svn.apache.org/viewvc?rev=1367715view=rev Log: merge -r 1367712:1367713 from trunk. FIXES: MAPREDUCE-4493 Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1367715r1=1367714r2=1367715view=diff == --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Tue Jul 31 19:21:27 2012 @@ -958,7 +958,6 @@ public class StreamJob implements Tool { if (!b) fail(LINK_URI); } -DistributedCache.createSymlink(jobConf_); // set the jobconf for the caching parameters if (cacheArchives != null) DistributedCache.setCacheArchives(archiveURIs, jobConf_);
svn commit: r1367717 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm
Author: tgraves Date: Tue Jul 31 19:22:25 2012 New Revision: 1367717 URL: http://svn.apache.org/viewvc?rev=1367717view=rev Log: merge -r 1367714:1367715 from branch-2. FIXES: MAPREDUCE-4493 Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm?rev=1367717r1=1367716r2=1367717view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm Tue Jul 31 19:22:25 2012 @@ -198,7 +198,9 @@ Deprecated Properties *---+---+ |mapred.compress.map.output | mapreduce.map.output.compress *---+---+ -|mapred.create.symlink | mapreduce.job.cache.symlink.create +|mapred.create.symlink | NONE - symlinking is always on +*---+---+ +|mapreduce.job.cache.symlink.create | NONE - symlinking is always on *---+---+ |mapred.data.field.separator | mapreduce.fieldsel.data.field.separator *---+---+
svn commit: r1367717 - /hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Author: tgraves Date: Tue Jul 31 19:22:25 2012 New Revision: 1367717 URL: http://svn.apache.org/viewvc?rev=1367717view=rev Log: merge -r 1367714:1367715 from branch-2. FIXES: MAPREDUCE-4493 Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1367717r1=1367716r2=1367717view=diff == --- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Tue Jul 31 19:22:25 2012 @@ -958,7 +958,6 @@ public class StreamJob implements Tool { if (!b) fail(LINK_URI); } -DistributedCache.createSymlink(jobConf_); // set the jobconf for the caching parameters if (cacheArchives != null) DistributedCache.setCacheArchives(archiveURIs, jobConf_);
svn commit: r1367764 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt pom.xml
Author: tgraves Date: Tue Jul 31 20:43:21 2012 New Revision: 1367764 URL: http://svn.apache.org/viewvc?rev=1367764view=rev Log: HADOOP-8370. Native build failure: javah: class file for org.apache.hadoop.classification.InterfaceAudience not found (Trevor Robinson via tgraves) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1367764r1=1367763r2=1367764view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Jul 31 20:43:21 2012 @@ -869,6 +869,10 @@ Release 0.23.3 - UNRELEASED HADOOP-8637. FilterFileSystem#setWriteChecksum is broken (daryn via bobby) +HADOOP-8370. Native build failure: javah: class file for +org.apache.hadoop.classification.InterfaceAudience not found (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml?rev=1367764r1=1367763r2=1367764view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml Tue Jul 31 20:43:21 2012 @@ -44,7 +44,7 @@ dependency groupIdorg.apache.hadoop/groupId artifactIdhadoop-annotations/artifactId - scopeprovided/scope + scopecompile/scope /dependency dependency groupIdcom.google.guava/groupId
svn commit: r1367765 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt pom.xml
Author: tgraves Date: Tue Jul 31 20:44:12 2012 New Revision: 1367765 URL: http://svn.apache.org/viewvc?rev=1367765view=rev Log: merge -r 1367763:1367764 from trunk. FIXES: HADOOP-8370 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1367765r1=1367764r2=1367765view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Tue Jul 31 20:44:12 2012 @@ -688,6 +688,10 @@ Release 0.23.3 - UNRELEASED HADOOP-8637. FilterFileSystem#setWriteChecksum is broken (daryn via bobby) +HADOOP-8370. Native build failure: javah: class file for +org.apache.hadoop.classification.InterfaceAudience not found (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml?rev=1367765r1=1367764r2=1367765view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml Tue Jul 31 20:44:12 2012 @@ -43,7 +43,7 @@ dependency groupIdorg.apache.hadoop/groupId artifactIdhadoop-annotations/artifactId - scopeprovided/scope + scopecompile/scope /dependency dependency groupIdcom.google.guava/groupId