svn commit: r1367539 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapred

[tgraves]

Author: tgraves
Date: Tue Jul 31 13:39:41 2012
New Revision: 1367539

URL: http://svn.apache.org/viewvc?rev=1367539view=rev
Log:
MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby via tgraves)

Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367539r1=1367538r2=1367539view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 
13:39:41 2012
@@ -505,6 +505,9 @@ Release 0.23.3 - UNRELEASED
 
 MAPREDUCE-4267. mavenize pipes (tgraves via bobby)
 
+MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby 
+via tgraves)
+
   OPTIMIZATIONS
 
 MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367539r1=1367538r2=1367539view=diff
==
--- 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
 (original)
+++ 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
 Tue Jul 31 13:39:41 2012
@@ -1561,7 +1561,7 @@ public class JobImpl implements org.apac
 Path confPath = getConfFile();
 FileContext fc = FileContext.getFileContext(confPath.toUri(), conf);
 Configuration jobConf = new Configuration(false);
-jobConf.addResource(fc.open(confPath));
+jobConf.addResource(fc.open(confPath), confPath.toString());
 return jobConf;
   }
 }

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1367539r1=1367538r2=1367539view=diff
==
--- 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
 (original)
+++ 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
 Tue Jul 31 13:39:41 2012
@@ -78,14 +78,29 @@ public class ConfBlock extends HtmlBlock
   tr().
 th(_TH, key).
 th(_TH, value).
+th(_TH, source chain).
   _().
 _().
   tbody();
   for (ConfEntryInfo entry : info.getProperties()) {
+StringBuffer buffer = new StringBuffer();
+String[] sources = entry.getSource();
+//Skip the last entry, because it is always the same HDFS file, and
+// output them in reverse order so most recent is output first
+boolean first = true;
+for(int i = 

svn commit: r1367540 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ h

[tgraves]

Author: tgraves
Date: Tue Jul 31 13:40:39 2012
New Revision: 1367540

URL: http://svn.apache.org/viewvc?rev=1367540view=rev
Log:
merge -r 1367538:1367539 from trunk. FIXES: MAPREDUCE-4375

Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367540r1=1367539r2=1367540view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 13:40:39 2012
@@ -381,6 +381,9 @@ Release 0.23.3 - UNRELEASED
 
 MAPREDUCE-4267. mavenize pipes (tgraves via bobby)
 
+MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby 
+via tgraves)
+
   OPTIMIZATIONS
 
 MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367540r1=1367539r2=1367540view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
 Tue Jul 31 13:40:39 2012
@@ -1561,7 +1561,7 @@ public class JobImpl implements org.apac
 Path confPath = getConfFile();
 FileContext fc = FileContext.getFileContext(confPath.toUri(), conf);
 Configuration jobConf = new Configuration(false);
-jobConf.addResource(fc.open(confPath));
+jobConf.addResource(fc.open(confPath), confPath.toString());
 return jobConf;
   }
 }

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1367540r1=1367539r2=1367540view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
 Tue Jul 31 13:40:39 2012
@@ -78,14 +78,29 @@ public class ConfBlock extends HtmlBlock
   tr().
 th(_TH, key).
 th(_TH, value).
+th(_TH, source chain).
   _().
 _().
   tbody();
   for (ConfEntryInfo entry : info.getProperties()) {
+StringBuffer buffer = new StringBuffer();
+

svn commit: r1367544 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl

[tgraves]

Author: tgraves
Date: Tue Jul 31 13:44:50 2012
New Revision: 1367544

URL: http://svn.apache.org/viewvc?rev=1367544view=rev
Log:
merge -r 1367539:1367540 from branch-2. FIXES: MAPREDUCE-4375

Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367544r1=1367543r2=1367544view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 13:44:50 2012
@@ -56,6 +56,9 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-2220. Fix new API FileOutputFormat-related typos in
 mapred-default.xml (Rui Kubo via harsh)
 
+MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby 
+via tgraves)
+
   OPTIMIZATIONS
 
 MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367544r1=1367543r2=1367544view=diff
==
--- 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
 (original)
+++ 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
 Tue Jul 31 13:44:50 2012
@@ -1481,7 +1481,7 @@ public class JobImpl implements org.apac
 Path confPath = getConfFile();
 FileContext fc = FileContext.getFileContext(confPath.toUri(), conf);
 Configuration jobConf = new Configuration(false);
-jobConf.addResource(fc.open(confPath));
+jobConf.addResource(fc.open(confPath), confPath.toString());
 return jobConf;
   }
 }

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1367544r1=1367543r2=1367544view=diff
==
--- 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
 (original)
+++ 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
 Tue Jul 31 13:44:50 2012
@@ -78,14 +78,29 @@ public class ConfBlock extends HtmlBlock
   tr().
 th(_TH, key).
 th(_TH, value).
+th(_TH, source chain).
   _().
 _().
   tbody();
  

svn commit: r1367581 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoo

[tgraves]

Author: tgraves
Date: Tue Jul 31 14:46:57 2012
New Revision: 1367581

URL: http://svn.apache.org/viewvc?rev=1367581view=rev
Log:
MAPREDUCE-4456. LocalDistributedCacheManager can get an ArrayIndexOutOfBounds 
when creating symlinks (Robert Evans via tgraves)

Added:

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367581r1=1367580r2=1367581view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 
14:46:57 2012
@@ -766,6 +766,9 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-4423. Potential infinite fetching of map output (Robert Evans
 via tgraves)
 
+MAPREDUCE-4456. LocalDistributedCacheManager can get an 
+ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367581r1=1367580r2=1367581view=diff
==
--- 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
 (original)
+++ 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
 Tue Jul 31 14:46:57 2012
@@ -18,12 +18,9 @@
 
 package org.apache.hadoop.mapred;
 
-import com.google.common.collect.Maps;
-
 import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;
-import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLClassLoader;
@@ -34,6 +31,7 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Random;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
@@ -60,6 +58,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.FSDownload;
 
+import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 /**
@@ -85,6 +84,9 @@ class LocalDistributedCacheManager {
* @throws IOException
*/
   public void setup(JobConf conf) throws IOException {
+boolean mkLinks = DistributedCache.getSymlink(conf);
+File workDir = new File(System.getProperty(user.dir));
+
 // Generate YARN local resources objects corresponding to the distributed
 // cache configuration
 MapString, LocalResource localResources = 
@@ -132,7 +134,8 @@ class LocalDistributedCacheManager {
 FuturePath future = exec.submit(download);
 resourcesToPaths.put(resource, future);
   }
-  for (LocalResource resource : localResources.values()) {
+  for (EntryString, LocalResource entry : localResources.entrySet()) {
+LocalResource resource = entry.getValue();
 Path path;
 try {
   path = resourcesToPaths.get(resource).get();
@@ -142,6 +145,12 @@ class LocalDistributedCacheManager {
   throw new IOException(e);
 }
 String pathString = path.toUri().toString();
+if(mkLinks) {
+  String link = entry.getKey();
+  String target = new File(path.toUri()).getPath();
+  symlink(workDir, target, link);
+}
+
 if (resource.getType() == LocalResourceType.ARCHIVE) {
   localArchives.add(pathString);
 } else if (resource.getType() == LocalResourceType.FILE) {
@@ -175,27 +184,6 @@ class LocalDistributedCacheManager {
   .arrayToString(localFiles.toArray(new String[localArchives
   .size()])));
 }
-if 

svn commit: r1367585 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-

[tgraves]

Author: tgraves
Date: Tue Jul 31 14:49:07 2012
New Revision: 1367585

URL: http://svn.apache.org/viewvc?rev=1367585view=rev
Log:
merge -r 1367580:1367581 from trunk. FIXES: MAPREDUCE-4456

Added:

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
  - copied unchanged from r1367581, 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367585r1=1367584r2=1367585view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 14:49:07 2012
@@ -642,6 +642,9 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-4423. Potential infinite fetching of map output (Robert Evans
 via tgraves)
 
+MAPREDUCE-4456. LocalDistributedCacheManager can get an 
+ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367585r1=1367584r2=1367585view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
 Tue Jul 31 14:49:07 2012
@@ -18,12 +18,9 @@
 
 package org.apache.hadoop.mapred;
 
-import com.google.common.collect.Maps;
-
 import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;
-import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLClassLoader;
@@ -34,6 +31,7 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Random;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
@@ -60,6 +58,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.FSDownload;
 
+import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 /**
@@ -85,6 +84,9 @@ class LocalDistributedCacheManager {
* @throws IOException
*/
   public void setup(JobConf conf) throws IOException {
+boolean mkLinks = DistributedCache.getSymlink(conf);
+File workDir = new File(System.getProperty(user.dir));
+
 // Generate YARN local resources objects corresponding to the distributed
 // cache configuration
 MapString, LocalResource localResources = 
@@ -132,7 +134,8 @@ class LocalDistributedCacheManager {
 FuturePath future = exec.submit(download);
 resourcesToPaths.put(resource, future);
   }
-  for (LocalResource resource : localResources.values()) {
+  for (EntryString, LocalResource entry : localResources.entrySet()) {
+LocalResource resource = entry.getValue();
 Path path;
 try {
   path = resourcesToPaths.get(resource).get();
@@ -142,6 +145,12 @@ class LocalDistributedCacheManager {
   throw new IOException(e);
 }
 String pathString = path.toUri().toString();
+if(mkLinks) {
+  String link = entry.getKey();
+  String target = new File(path.toUri()).getPath();
+  symlink(workDir, target, link);
+}
+
 if (resource.getType() == LocalResourceType.ARCHIVE) {
   

svn commit: r1367587 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapredu

[tgraves]

Author: tgraves
Date: Tue Jul 31 14:50:27 2012
New Revision: 1367587

URL: http://svn.apache.org/viewvc?rev=1367587view=rev
Log:
merge -r 1367584:1367585 from branch-2. FIXES: MAPREDUCE-4456

Added:

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
  - copied unchanged from r1367585, 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367587r1=1367586r2=1367587view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 14:50:27 2012
@@ -358,6 +358,9 @@ Release 0.23.3 - UNRELEASED
 
 MAPREDUCE-4010.  TestWritableJobConf fails on trunk (tucu via bobby)
 
+MAPREDUCE-4456. LocalDistributedCacheManager can get an 
+ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367587r1=1367586r2=1367587view=diff
==
--- 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
 (original)
+++ 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
 Tue Jul 31 14:50:27 2012
@@ -18,12 +18,9 @@
 
 package org.apache.hadoop.mapred;
 
-import com.google.common.collect.Maps;
-
 import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;
-import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLClassLoader;
@@ -34,6 +31,7 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Random;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
@@ -60,6 +58,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.FSDownload;
 
+import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 /**
@@ -85,6 +84,9 @@ class LocalDistributedCacheManager {
* @throws IOException
*/
   public void setup(JobConf conf) throws IOException {
+boolean mkLinks = DistributedCache.getSymlink(conf);
+File workDir = new File(System.getProperty(user.dir));
+
 // Generate YARN local resources objects corresponding to the distributed
 // cache configuration
 MapString, LocalResource localResources = 
@@ -132,7 +134,8 @@ class LocalDistributedCacheManager {
 FuturePath future = exec.submit(download);
 resourcesToPaths.put(resource, future);
   }
-  for (LocalResource resource : localResources.values()) {
+  for (EntryString, LocalResource entry : localResources.entrySet()) {
+LocalResource resource = entry.getValue();
 Path path;
 try {
   path = resourcesToPaths.get(resource).get();
@@ -142,6 +145,12 @@ class LocalDistributedCacheManager {
   throw new IOException(e);
 }
 String pathString = path.toUri().toString();
+if(mkLinks) {
+  String link = entry.getKey();
+  String target = new File(path.toUri()).getPath();
+  symlink(workDir, target, link);
+}
+
 if (resource.getType() == 

svn commit: r1367601 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ hadoop-mapr

[bobby]

Author: bobby
Date: Tue Jul 31 15:08:47 2012
New Revision: 1367601

URL: http://svn.apache.org/viewvc?rev=1367601view=rev
Log:
MAPREDUCE-4496. AM logs link is missing user name (Jason Lowe via bobby)

Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367601r1=1367600r2=1367601view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 
15:08:47 2012
@@ -769,6 +769,8 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-4456. LocalDistributedCacheManager can get an 
 ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves)
 
+MAPREDUCE-4496. AM logs link is missing user name (Jason Lowe via bobby)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java?rev=1367601r1=1367600r2=1367601view=diff
==
--- 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java
 (original)
+++ 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java
 Tue Jul 31 15:08:47 2012
@@ -64,7 +64,7 @@ public class AMAttemptInfo {
 if (containerId != null) {
   this.containerId = containerId.toString();
   this.logsLink = join(http://; + nodeHttpAddress,
-  ujoin(node, containerlogs, this.containerId));
+  ujoin(node, containerlogs, this.containerId, user));
 }
   }
 

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java?rev=1367601r1=1367600r2=1367601view=diff
==
--- 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
 (original)
+++ 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
 Tue Jul 31 15:08:47 2012
@@ -972,7 +972,8 @@ public class TestAMWebServicesJobs exten
 WebServicesTestUtils.checkStringMatch(containerId, amInfo
 .getContainerId().toString(), containerId);
 
-String localLogsLink = ujoin(node, containerlogs, containerId);
+String localLogsLink =ujoin(node, containerlogs, containerId,
+job.getUserName());
 
 assertTrue(logsLink, logsLink.contains(localLogsLink));
   }

svn commit: r1367602 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/

[bobby]

Author: bobby
Date: Tue Jul 31 15:09:49 2012
New Revision: 1367602

URL: http://svn.apache.org/viewvc?rev=1367602view=rev
Log:
svn merge -c 1367601 FIXES: MAPREDUCE-4496. AM logs link is missing user name 
(Jason Lowe via bobby)

Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367602r1=1367601r2=1367602view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 15:09:49 2012
@@ -645,6 +645,8 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-4456. LocalDistributedCacheManager can get an 
 ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves)
 
+MAPREDUCE-4496. AM logs link is missing user name (Jason Lowe via bobby)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java?rev=1367602r1=1367601r2=1367602view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java
 Tue Jul 31 15:09:49 2012
@@ -64,7 +64,7 @@ public class AMAttemptInfo {
 if (containerId != null) {
   this.containerId = containerId.toString();
   this.logsLink = join(http://; + nodeHttpAddress,
-  ujoin(node, containerlogs, this.containerId));
+  ujoin(node, containerlogs, this.containerId, user));
 }
   }
 

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java?rev=1367602r1=1367601r2=1367602view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
 Tue Jul 31 15:09:49 2012
@@ -972,7 +972,8 @@ public class TestAMWebServicesJobs exten
 WebServicesTestUtils.checkStringMatch(containerId, amInfo
 .getContainerId().toString(), containerId);
 
-String localLogsLink = ujoin(node, containerlogs, containerId);
+String localLogsLink =ujoin(node, containerlogs, containerId,
+job.getUserName());
 
 assertTrue(logsLink, logsLink.contains(localLogsLink));
   }

svn commit: r1367604 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/d

[bobby]

Author: bobby
Date: Tue Jul 31 15:10:31 2012
New Revision: 1367604

URL: http://svn.apache.org/viewvc?rev=1367604view=rev
Log:
svn merge -c 1367601 FIXES: MAPREDUCE-4496. AM logs link is missing user name 
(Jason Lowe via bobby)

Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367604r1=1367603r2=1367604view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 15:10:31 2012
@@ -361,6 +361,8 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-4456. LocalDistributedCacheManager can get an 
 ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves)
 
+MAPREDUCE-4496. AM logs link is missing user name (Jason Lowe via bobby)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java?rev=1367604r1=1367603r2=1367604view=diff
==
--- 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java
 (original)
+++ 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/AMAttemptInfo.java
 Tue Jul 31 15:10:31 2012
@@ -64,7 +64,7 @@ public class AMAttemptInfo {
 if (containerId != null) {
   this.containerId = containerId.toString();
   this.logsLink = join(http://; + nodeHttpAddress,
-  ujoin(node, containerlogs, this.containerId));
+  ujoin(node, containerlogs, this.containerId, user));
 }
   }
 

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java?rev=1367604r1=1367603r2=1367604view=diff
==
--- 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
 (original)
+++ 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
 Tue Jul 31 15:10:31 2012
@@ -972,7 +972,8 @@ public class TestAMWebServicesJobs exten
 WebServicesTestUtils.checkStringMatch(containerId, amInfo
 .getContainerId().toString(), containerId);
 
-String localLogsLink = ujoin(node, containerlogs, containerId);
+String localLogsLink =ujoin(node, containerlogs, containerId,
+job.getUserName());
 
 assertTrue(logsLink, logsLink.contains(localLogsLink));
   }

svn commit: r1367715 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-

[tgraves]

Author: tgraves
Date: Tue Jul 31 19:21:27 2012
New Revision: 1367715

URL: http://svn.apache.org/viewvc?rev=1367715view=rev
Log:
merge -r 1367712:1367713 from trunk. FIXES: MAPREDUCE-4493

Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367715r1=1367714r2=1367715view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 19:21:27 2012
@@ -647,6 +647,9 @@ Release 0.23.3 - UNRELEASED
 
 MAPREDUCE-4496. AM logs link is missing user name (Jason Lowe via bobby)
 
+MAPREDUCE-4493. Distibuted Cache Compatability Issues (Robert Evans
+via tgraves)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367715r1=1367714r2=1367715view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
 Tue Jul 31 19:21:27 

svn commit: r1367717 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapredu

[tgraves]

Author: tgraves
Date: Tue Jul 31 19:22:25 2012
New Revision: 1367717

URL: http://svn.apache.org/viewvc?rev=1367717view=rev
Log:
merge -r 1367714:1367715 from branch-2. FIXES: MAPREDUCE-4493

Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367717r1=1367716r2=1367717view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 19:22:25 2012
@@ -363,6 +363,9 @@ Release 0.23.3 - UNRELEASED
 
 MAPREDUCE-4496. AM logs link is missing user name (Jason Lowe via bobby)
 
+MAPREDUCE-4493. Distibuted Cache Compatability Issues (Robert Evans
+via tgraves)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367717r1=1367716r2=1367717view=diff
==
--- 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
 (original)
+++ 

svn commit: r1367719 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanage

[bobby]

Author: bobby
Date: Tue Jul 31 19:34:29 2012
New Revision: 1367719

URL: http://svn.apache.org/viewvc?rev=1367719view=rev
Log:
MAPREDUCE-4492. Configuring total queue capacity between 100.5 and 99.5 at 
perticular level is sucessfull (Mayank Bansal via bobby)

Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367719r1=1367718r2=1367719view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 
19:34:29 2012
@@ -774,6 +774,9 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-4493. Distibuted Cache Compatability Issues (Robert Evans
 via tgraves)
 
+MAPREDUCE-4492. Configuring total queue capacity between 100.5 and 99.5 at
+perticular level is sucessfull (Mayank Bansal via bobby)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1367719r1=1367718r2=1367719view=diff
==
--- 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
 (original)
+++ 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
 Tue Jul 31 19:34:29 2012
@@ -193,7 +193,7 @@ public class ParentQueue implements CSQu
 , acls= + aclsString);
   }
 
-  private static float PRECISION = 0.005f; // 0.05% precision
+  private static float PRECISION = 0.0005f; // 0.05% precision
   void setChildQueues(CollectionCSQueue childQueues) {
 
 // Validate

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1367719r1=1367718r2=1367719view=diff
==
--- 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
 (original)
+++ 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
 Tue Jul 31 19:34:29 2012
@@ -34,6 +34,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import junit.framework.Assert;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -270,6 +272,61 @@ public class TestParentQueue {
 verifyQueueMetrics(b, 9*GB, clusterResource);
   }
 
+  @Test
+  public void testSingleLevelQueuesPrecision() throws Exception {
+// Setup queue configs
+setupSingleLevelQueues(csConf);
+final String Q_A = CapacitySchedulerConfiguration.ROOT + . + a;
+csConf.setCapacity(Q_A, 30);
+final String Q_B = CapacitySchedulerConfiguration.ROOT + . + b;
+csConf.setCapacity(Q_B, 70.5F);
+
+MapString, CSQueue queues = new HashMapString, CSQueue();
+boolean exceptionOccured = false;
+try {
+  CapacityScheduler.parseQueue(csContext, csConf, null,
+  CapacitySchedulerConfiguration.ROOT, queues, queues,
+  CapacityScheduler.queueComparator,
+  

svn commit: r1367720 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/re

[bobby]

Author: bobby
Date: Tue Jul 31 19:35:39 2012
New Revision: 1367720

URL: http://svn.apache.org/viewvc?rev=1367720view=rev
Log:
svn merge -c 1367719 FIXES: MAPREDUCE-4492. Configuring total queue capacity 
between 100.5 and 99.5 at perticular level is sucessfull (Mayank Bansal via 
bobby)

Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367720r1=1367719r2=1367720view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 19:35:39 2012
@@ -650,6 +650,9 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-4493. Distibuted Cache Compatability Issues (Robert Evans
 via tgraves)
 
+MAPREDUCE-4492. Configuring total queue capacity between 100.5 and 99.5 at
+perticular level is sucessfull (Mayank Bansal via bobby)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1367720r1=1367719r2=1367720view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
 Tue Jul 31 19:35:39 2012
@@ -193,7 +193,7 @@ public class ParentQueue implements CSQu
 , acls= + aclsString);
   }
 
-  private static float PRECISION = 0.005f; // 0.05% precision
+  private static float PRECISION = 0.0005f; // 0.05% precision
   void setChildQueues(CollectionCSQueue childQueues) {
 
 // Validate

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1367720r1=1367719r2=1367720view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
 Tue Jul 31 19:35:39 2012
@@ -34,6 +34,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import junit.framework.Assert;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -270,6 +272,61 @@ public class TestParentQueue {
 verifyQueueMetrics(b, 9*GB, clusterResource);
   }
 
+  @Test
+  public void testSingleLevelQueuesPrecision() throws Exception {
+// Setup queue configs
+setupSingleLevelQueues(csConf);
+final String Q_A = CapacitySchedulerConfiguration.ROOT + . + a;
+csConf.setCapacity(Q_A, 30);
+final String Q_B = CapacitySchedulerConfiguration.ROOT + . + b;
+csConf.setCapacity(Q_B, 70.5F);
+
+MapString, CSQueue queues = new HashMapString, CSQueue();
+boolean exceptionOccured = false;
+

svn commit: r1367726 - in /hadoop/common/trunk/hadoop-mapreduce-project: CHANGES.txt hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml

[bobby]

Author: bobby
Date: Tue Jul 31 19:43:51 2012
New Revision: 1367726

URL: http://svn.apache.org/viewvc?rev=1367726view=rev
Log:
MAPREDUCE-4483. 2.0 build does not work (John George via bobby)

Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367726r1=1367725r2=1367726view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 
19:43:51 2012
@@ -461,6 +461,8 @@ Release 2.0.0-alpha - 05-23-2012
 MAPREDUCE-4231. Update RAID to use the new BlockCollection interface.
 (szetszwo)
 
+MAPREDUCE-4483. 2.0 build does not work (John George via bobby)
+
 Release 0.23.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml?rev=1367726r1=1367725r2=1367726view=diff
==
--- 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
 (original)
+++ 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
 Tue Jul 31 19:43:51 2012
@@ -64,7 +64,6 @@
 dependency
   groupIdorg.apache.hadoop/groupId
   artifactIdhadoop-yarn-applications-distributedshell/artifactId
-  version3.0.0-SNAPSHOT/version
   scopetest/scope
 /dependency
 dependency

svn commit: r1367729 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: CHANGES.txt hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml

[bobby]

Author: bobby
Date: Tue Jul 31 19:50:11 2012
New Revision: 1367729

URL: http://svn.apache.org/viewvc?rev=1367729view=rev
Log:
svn merge -c 1367726. FIXES: MAPREDUCE-4483. 2.0 build does not work (John 
George via bobby)

Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367729r1=1367728r2=1367729view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 19:50:11 2012
@@ -337,6 +337,8 @@ Release 2.0.0-alpha - 05-23-2012
 MAPREDUCE-4231. Update RAID to use the new BlockCollection interface.
 (szetszwo)
 
+MAPREDUCE-4483. 2.0 build does not work (John George via bobby)
+
 Release 0.23.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml?rev=1367729r1=1367728r2=1367729view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
 Tue Jul 31 19:50:11 2012
@@ -19,12 +19,12 @@
   parent
 artifactIdhadoop-yarn-applications/artifactId
 groupIdorg.apache.hadoop/groupId
-version3.0.0-SNAPSHOT/version
+version2.0.1-SNAPSHOT/version
   /parent
   modelVersion4.0.0/modelVersion
   groupIdorg.apache.hadoop/groupId
   artifactIdhadoop-yarn-applications-unmanaged-am-launcher/artifactId
-  version3.0.0-SNAPSHOT/version
+  version2.0.1-SNAPSHOT/version
   namehadoop-yarn-applications-unmanaged-am-launcher/name
 
   properties
@@ -64,7 +64,6 @@
 dependency
   groupIdorg.apache.hadoop/groupId
   artifactIdhadoop-yarn-applications-distributedshell/artifactId
-  version3.0.0-SNAPSHOT/version
   scopetest/scope
 /dependency
 dependency

svn commit: r1367771 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapred

[tgraves]

Author: tgraves
Date: Tue Jul 31 20:52:41 2012
New Revision: 1367771

URL: http://svn.apache.org/viewvc?rev=1367771view=rev
Log:
MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at FAILED  
(Robert Evans via tgraves)

Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367771r1=1367770r2=1367771view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 
20:52:41 2012
@@ -779,6 +779,9 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-4492. Configuring total queue capacity between 100.5 and 99.5 at
 perticular level is sucessfull (Mayank Bansal via bobby)
 
+MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at 
+FAILED  (Robert Evans via tgraves)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367771r1=1367770r2=1367771view=diff
==
--- 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
 (original)
+++ 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
 Tue Jul 31 20:52:41 2012
@@ -1370,7 +1370,8 @@ public class JobImpl implements org.apac
   }
 }
 
-float failureRate = (float) fetchFailures / runningReduceTasks;
+float failureRate = runningReduceTasks == 0 ? 1.0f : 
+  (float) fetchFailures / runningReduceTasks;
 // declare faulty if fetch-failures = max-allowed-failures
 boolean isMapFaulty =
 (failureRate = MAX_ALLOWED_FETCH_FAILURES_FRACTION);

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1367771r1=1367770r2=1367771view=diff
==
--- 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
 (original)
+++ 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
 Tue Jul 31 20:52:41 2012
@@ -435,7 +435,8 @@ public abstract class TaskAttemptImpl im
  TaskAttemptEventType.TA_CONTAINER_CLEANED,
  TaskAttemptEventType.TA_COMMIT_PENDING,
  TaskAttemptEventType.TA_DONE,
- TaskAttemptEventType.TA_FAILMSG))
+ TaskAttemptEventType.TA_FAILMSG,
+ TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE))
 
  // Transitions from KILLED state
  .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED,

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java?rev=1367771r1=1367770r2=1367771view=diff
==
--- 

svn commit: r1367772 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ h

[tgraves]

Author: tgraves
Date: Tue Jul 31 20:53:45 2012
New Revision: 1367772

URL: http://svn.apache.org/viewvc?rev=1367772view=rev
Log:
merge -r 1367770:1367771 from trunk. FIXES: MAPREDUCE-4457

Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367772r1=1367771r2=1367772view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 20:53:45 2012
@@ -655,6 +655,9 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-4492. Configuring total queue capacity between 100.5 and 99.5 at
 perticular level is sucessfull (Mayank Bansal via bobby)
 
+MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at 
+FAILED  (Robert Evans via tgraves)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367772r1=1367771r2=1367772view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
 Tue Jul 31 20:53:45 2012
@@ -1370,7 +1370,8 @@ public class JobImpl implements org.apac
   }
 }
 
-float failureRate = (float) fetchFailures / runningReduceTasks;
+float failureRate = runningReduceTasks == 0 ? 1.0f : 
+  (float) fetchFailures / runningReduceTasks;
 // declare faulty if fetch-failures = max-allowed-failures
 boolean isMapFaulty =
 (failureRate = MAX_ALLOWED_FETCH_FAILURES_FRACTION);

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1367772r1=1367771r2=1367772view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
 Tue Jul 31 20:53:45 2012
@@ -435,7 +435,8 @@ public abstract class TaskAttemptImpl im
  TaskAttemptEventType.TA_CONTAINER_CLEANED,
  TaskAttemptEventType.TA_COMMIT_PENDING,
  TaskAttemptEventType.TA_DONE,
- TaskAttemptEventType.TA_FAILMSG))
+ TaskAttemptEventType.TA_FAILMSG,
+ TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE))
 
  // Transitions from KILLED state
  .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED,

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
URL: 

svn commit: r1367774 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl

[tgraves]

Author: tgraves
Date: Tue Jul 31 20:54:45 2012
New Revision: 1367774

URL: http://svn.apache.org/viewvc?rev=1367774view=rev
Log:
merge -r 1367771:1367772 from branch-2. FIXES: MAPREDUCE-4457

Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367774r1=1367773r2=1367774view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 20:54:45 2012
@@ -369,6 +369,9 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-4492. Configuring total queue capacity between 100.5 and 99.5 at
 perticular level is sucessfull (Mayank Bansal via bobby)
 
+MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at 
+FAILED  (Robert Evans via tgraves)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367774r1=1367773r2=1367774view=diff
==
--- 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
 (original)
+++ 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
 Tue Jul 31 20:54:45 2012
@@ -1305,7 +1305,8 @@ public class JobImpl implements org.apac
   }
 }
 
-float failureRate = (float) fetchFailures / runningReduceTasks;
+float failureRate = runningReduceTasks == 0 ? 1.0f : 
+  (float) fetchFailures / runningReduceTasks;
 // declare faulty if fetch-failures = max-allowed-failures
 boolean isMapFaulty =
 (failureRate = MAX_ALLOWED_FETCH_FAILURES_FRACTION);

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1367774r1=1367773r2=1367774view=diff
==
--- 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
 (original)
+++ 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
 Tue Jul 31 20:54:45 2012
@@ -431,7 +431,8 @@ public abstract class TaskAttemptImpl im
  TaskAttemptEventType.TA_CONTAINER_CLEANED,
  TaskAttemptEventType.TA_COMMIT_PENDING,
  TaskAttemptEventType.TA_DONE,
- TaskAttemptEventType.TA_FAILMSG))
+ TaskAttemptEventType.TA_FAILMSG,
+ TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE))
 
  // Transitions from KILLED state
  .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED,

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
URL: 

svn commit: r1367783 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ hadoo

[bobby]

Author: bobby
Date: Tue Jul 31 21:08:40 2012
New Revision: 1367783

URL: http://svn.apache.org/viewvc?rev=1367783view=rev
Log:
MAPREDUCE-. nodemanager fails to start when one of the local-dirs is  bad 
(Jason Lowe via bobby)

Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367783r1=1367782r2=1367783view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 
21:08:40 2012
@@ -463,6 +463,9 @@ Release 2.0.0-alpha - 05-23-2012
 
 MAPREDUCE-4483. 2.0 build does not work (John George via bobby)
 
+MAPREDUCE-. nodemanager fails to start when one of the local-dirs is
+bad (Jason Lowe via bobby)
+
 Release 0.23.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java?rev=1367783r1=1367782r2=1367783view=diff
==
--- 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
 (original)
+++ 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
 Tue Jul 31 21:08:40 2012
@@ -93,23 +93,7 @@ public class LocalDirsHandlerService ext
 
 @Override
 public void run() {
-  boolean newFailure = false;
-  if (localDirs.checkDirs()) {
-newFailure = true;
-  }
-  if (logDirs.checkDirs()) {
-newFailure = true;
-  }
-
-  if (newFailure) {
-LOG.info(Disk(s) failed.  + getDisksHealthReport());
-updateDirsInConfiguration();
-if (!areDisksHealthy()) {
-  // Just log.
-  LOG.error(Most of the disks failed.  + getDisksHealthReport());
-}
-  }
-  lastDisksCheckTime = System.currentTimeMillis();
+  checkDirs();
 }
   }
 
@@ -135,6 +119,10 @@ public class LocalDirsHandlerService ext
 YarnConfiguration.DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION);
 lastDisksCheckTime = System.currentTimeMillis();
 super.init(conf);
+
+// Check the disk health immediately to weed out bad directories
+// before other init code attempts to use them.
+checkDirs();
   }
 
   /**
@@ -144,10 +132,8 @@ public class LocalDirsHandlerService ext
   public void start() {
 if (isDiskHealthCheckerEnabled) {
   dirsHandlerScheduler = new Timer(DiskHealthMonitor-Timer, true);
-  // Start the timer task for disk health checking immediately and
-  // then run periodically at interval time.
-  dirsHandlerScheduler.scheduleAtFixedRate(monitoringTimerTask, 0,
-   diskHealthCheckInterval);
+  dirsHandlerScheduler.scheduleAtFixedRate(monitoringTimerTask,
+  diskHealthCheckInterval, diskHealthCheckInterval);
 }
 super.start();
   }
@@ -253,6 +239,26 @@ public class LocalDirsHandlerService ext
   logDirs.toArray(new String[logDirs.size()]));
   }
 
+  private void checkDirs() {
+  boolean newFailure = false;
+  if (localDirs.checkDirs()) {
+newFailure = true;
+  }
+  if (logDirs.checkDirs()) {
+newFailure = true;
+  }
+
+  if (newFailure) {
+LOG.info(Disk(s) failed.  + getDisksHealthReport());
+updateDirsInConfiguration();
+if (!areDisksHealthy()) {
+  // Just log.
+  LOG.error(Most of the disks failed.  + getDisksHealthReport());
+}
+  }
+  lastDisksCheckTime = System.currentTimeMillis();
+  }
+
   public Path getLocalPathForWrite(String pathStr) throws IOException {
 return localDirsAllocator.getLocalPathForWrite(pathStr, getConfig());
   }

Modified: 

svn commit: r1367784 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodema

[bobby]

Author: bobby
Date: Tue Jul 31 21:09:43 2012
New Revision: 1367784

URL: http://svn.apache.org/viewvc?rev=1367784view=rev
Log:
svn merge -c 1367783 FIXES: MAPREDUCE-. nodemanager fails to start when one 
of the local-dirs is  bad (Jason Lowe via bobby)

Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367784r1=1367783r2=1367784view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 21:09:43 2012
@@ -339,6 +339,9 @@ Release 2.0.0-alpha - 05-23-2012
 
 MAPREDUCE-4483. 2.0 build does not work (John George via bobby)
 
+MAPREDUCE-. nodemanager fails to start when one of the local-dirs is
+bad (Jason Lowe via bobby)
+
 Release 0.23.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java?rev=1367784r1=1367783r2=1367784view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
 Tue Jul 31 21:09:43 2012
@@ -93,23 +93,7 @@ public class LocalDirsHandlerService ext
 
 @Override
 public void run() {
-  boolean newFailure = false;
-  if (localDirs.checkDirs()) {
-newFailure = true;
-  }
-  if (logDirs.checkDirs()) {
-newFailure = true;
-  }
-
-  if (newFailure) {
-LOG.info(Disk(s) failed.  + getDisksHealthReport());
-updateDirsInConfiguration();
-if (!areDisksHealthy()) {
-  // Just log.
-  LOG.error(Most of the disks failed.  + getDisksHealthReport());
-}
-  }
-  lastDisksCheckTime = System.currentTimeMillis();
+  checkDirs();
 }
   }
 
@@ -135,6 +119,10 @@ public class LocalDirsHandlerService ext
 YarnConfiguration.DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION);
 lastDisksCheckTime = System.currentTimeMillis();
 super.init(conf);
+
+// Check the disk health immediately to weed out bad directories
+// before other init code attempts to use them.
+checkDirs();
   }
 
   /**
@@ -144,10 +132,8 @@ public class LocalDirsHandlerService ext
   public void start() {
 if (isDiskHealthCheckerEnabled) {
   dirsHandlerScheduler = new Timer(DiskHealthMonitor-Timer, true);
-  // Start the timer task for disk health checking immediately and
-  // then run periodically at interval time.
-  dirsHandlerScheduler.scheduleAtFixedRate(monitoringTimerTask, 0,
-   diskHealthCheckInterval);
+  dirsHandlerScheduler.scheduleAtFixedRate(monitoringTimerTask,
+  diskHealthCheckInterval, diskHealthCheckInterval);
 }
 super.start();
   }
@@ -253,6 +239,26 @@ public class LocalDirsHandlerService ext
   logDirs.toArray(new String[logDirs.size()]));
   }
 
+  private void checkDirs() {
+  boolean newFailure = false;
+  if (localDirs.checkDirs()) {
+newFailure = true;
+  }
+  if (logDirs.checkDirs()) {
+newFailure = true;
+  }
+
+  if (newFailure) {
+LOG.info(Disk(s) failed.  + getDisksHealthReport());
+updateDirsInConfiguration();
+if (!areDisksHealthy()) {
+  // Just log.
+  LOG.error(Most of the disks failed.  + getDisksHealthReport());
+}
+  }
+  lastDisksCheckTime = System.currentTimeMillis();
+  }
+
   public Path getLocalPathForWrite(String pathStr) throws IOException {
 return 

svn commit: r1367785 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nod

[bobby]

Author: bobby
Date: Tue Jul 31 21:10:20 2012
New Revision: 1367785

URL: http://svn.apache.org/viewvc?rev=1367785view=rev
Log:
svn merge -c 1367783 FIXES: MAPREDUCE-. nodemanager fails to start when one 
of the local-dirs is  bad (Jason Lowe via bobby)

Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367785r1=1367784r2=1367785view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 21:10:20 2012
@@ -1,5 +1,8 @@
 Hadoop MapReduce Change Log
 
+MAPREDUCE-. nodemanager fails to start when one of the local-dirs is
+bad (Jason Lowe via bobby)
+
 Release 0.23.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java?rev=1367785r1=1367784r2=1367785view=diff
==
--- 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
 (original)
+++ 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
 Tue Jul 31 21:10:20 2012
@@ -93,23 +93,7 @@ public class LocalDirsHandlerService ext
 
 @Override
 public void run() {
-  boolean newFailure = false;
-  if (localDirs.checkDirs()) {
-newFailure = true;
-  }
-  if (logDirs.checkDirs()) {
-newFailure = true;
-  }
-
-  if (newFailure) {
-LOG.info(Disk(s) failed.  + getDisksHealthReport());
-updateDirsInConfiguration();
-if (!areDisksHealthy()) {
-  // Just log.
-  LOG.error(Most of the disks failed.  + getDisksHealthReport());
-}
-  }
-  lastDisksCheckTime = System.currentTimeMillis();
+  checkDirs();
 }
   }
 
@@ -135,6 +119,10 @@ public class LocalDirsHandlerService ext
 YarnConfiguration.DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION);
 lastDisksCheckTime = System.currentTimeMillis();
 super.init(conf);
+
+// Check the disk health immediately to weed out bad directories
+// before other init code attempts to use them.
+checkDirs();
   }
 
   /**
@@ -144,10 +132,8 @@ public class LocalDirsHandlerService ext
   public void start() {
 if (isDiskHealthCheckerEnabled) {
   dirsHandlerScheduler = new Timer(DiskHealthMonitor-Timer, true);
-  // Start the timer task for disk health checking immediately and
-  // then run periodically at interval time.
-  dirsHandlerScheduler.scheduleAtFixedRate(monitoringTimerTask, 0,
-   diskHealthCheckInterval);
+  dirsHandlerScheduler.scheduleAtFixedRate(monitoringTimerTask,
+  diskHealthCheckInterval, diskHealthCheckInterval);
 }
 super.start();
   }
@@ -253,6 +239,26 @@ public class LocalDirsHandlerService ext
   logDirs.toArray(new String[logDirs.size()]));
   }
 
+  private void checkDirs() {
+  boolean newFailure = false;
+  if (localDirs.checkDirs()) {
+newFailure = true;
+  }
+  if (logDirs.checkDirs()) {
+newFailure = true;
+  }
+
+  if (newFailure) {
+LOG.info(Disk(s) failed.  + getDisksHealthReport());
+updateDirsInConfiguration();
+if (!areDisksHealthy()) {
+  // Just log.
+  LOG.error(Most of the disks failed.  + getDisksHealthReport());
+}
+  }
+  lastDisksCheckTime = System.currentTimeMillis();
+  }
+
   public Path getLocalPathForWrite(String pathStr) throws IOException {
 return localDirsAllocator.getLocalPathForWrite(pathStr, getConfig());
   }

svn commit: r1367789 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ src/test/mapred/org/apache

[jeagles]

Author: jeagles
Date: Tue Jul 31 21:26:08 2012
New Revision: 1367789

URL: http://svn.apache.org/viewvc?rev=1367789view=rev
Log:
MAPREDUCE-4234. SortValidator.java is incompatible with multi-user or parallel 
use (due to a /tmp file with static name) (Robert Evans via jeagles)

Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java

hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367789r1=1367788r2=1367789view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 
21:26:08 2012
@@ -785,6 +785,10 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at 
 FAILED  (Robert Evans via tgraves)
 
+MAPREDUCE-4234. SortValidator.java is incompatible with multi-user or
+parallel use (due to a /tmp file with static name) (Robert Evans via
+jeagles)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java?rev=1367789r1=1367788r2=1367789view=diff
==
--- 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
 (original)
+++ 
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
 Tue Jul 31 21:26:08 2012
@@ -33,7 +33,6 @@ import org.apache.hadoop.io.WritableComp
 import org.apache.hadoop.io.WritableComparator;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.mapred.lib.HashPartitioner;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.fs.*;
@@ -345,7 +344,8 @@ public class SortValidator extends Confi
 
   FileInputFormat.setInputPaths(jobConf, sortInput);
   FileInputFormat.addInputPath(jobConf, sortOutput);
-  Path outputPath = new Path(/tmp/sortvalidate/recordstatschecker);
+  Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir, 
/tmp),
+   sortvalidate), UUID.randomUUID().toString());
   if (defaultfs.exists(outputPath)) {
 defaultfs.delete(outputPath, true);
   }
@@ -365,31 +365,44 @@ public class SortValidator extends Confi
   Date startTime = new Date();
   System.out.println(Job started:  + startTime);
   JobClient.runJob(jobConf);
-  Date end_time = new Date();
-  System.out.println(Job ended:  + end_time);
-  System.out.println(The job took  + 
- (end_time.getTime() - startTime.getTime()) /1000 +  
seconds.);
-  
-  // Check to ensure that the statistics of the 
-  // framework's sort-input and sort-output match
-  SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs,
-  new Path(outputPath, 
part-0), defaults);
-  IntWritable k1 = new IntWritable();
-  IntWritable k2 = new IntWritable();
-  RecordStatsWritable v1 = new RecordStatsWritable();
-  RecordStatsWritable v2 = new RecordStatsWritable();
-  if (!stats.next(k1, v1)) {
-throw new IOException(Failed to read record #1 from reduce's output);
-  }
-  if (!stats.next(k2, v2)) {
-throw new IOException(Failed to read record #2 from reduce's output);
-  }
-
-  if ((v1.getBytes() != v2.getBytes()) || (v1.getRecords() != 
v2.getRecords()) || 
-  v1.getChecksum() != v2.getChecksum()) {
-throw new IOException(( + 
-  v1.getBytes() + ,  + v1.getRecords() + ,  + 
v1.getChecksum() + ) v/s ( +
-  v2.getBytes() + ,  + v2.getRecords() + ,  + 
v2.getChecksum() + ));
+  try {
+Date end_time = new Date();
+System.out.println(Job ended:  + end_time);
+System.out.println(The job took  + 
+(end_time.getTime() - startTime.getTime()) /1000 +  seconds.);
+
+// Check to ensure that the statistics of the 
+

svn commit: r1367793 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ src/test/mapre

[jeagles]

Author: jeagles
Date: Tue Jul 31 21:45:45 2012
New Revision: 1367793

URL: http://svn.apache.org/viewvc?rev=1367793view=rev
Log:
svn merge -c 1367789 FIXES: MAPREDUCE-4234. SortValidator.java is incompatible 
with multi-user or parallel use (due to a /tmp file with static name) (Robert 
Evans via jeagles)

Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java

hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367793r1=1367792r2=1367793view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 21:45:45 2012
@@ -661,6 +661,10 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at 
 FAILED  (Robert Evans via tgraves)
 
+MAPREDUCE-4234. SortValidator.java is incompatible with multi-user or
+parallel use (due to a /tmp file with static name) (Robert Evans via
+jeagles)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java?rev=1367793r1=1367792r2=1367793view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
 Tue Jul 31 21:45:45 2012
@@ -33,7 +33,6 @@ import org.apache.hadoop.io.WritableComp
 import org.apache.hadoop.io.WritableComparator;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.mapred.lib.HashPartitioner;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.fs.*;
@@ -345,7 +344,8 @@ public class SortValidator extends Confi
 
   FileInputFormat.setInputPaths(jobConf, sortInput);
   FileInputFormat.addInputPath(jobConf, sortOutput);
-  Path outputPath = new Path(/tmp/sortvalidate/recordstatschecker);
+  Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir, 
/tmp),
+   sortvalidate), UUID.randomUUID().toString());
   if (defaultfs.exists(outputPath)) {
 defaultfs.delete(outputPath, true);
   }
@@ -365,31 +365,44 @@ public class SortValidator extends Confi
   Date startTime = new Date();
   System.out.println(Job started:  + startTime);
   JobClient.runJob(jobConf);
-  Date end_time = new Date();
-  System.out.println(Job ended:  + end_time);
-  System.out.println(The job took  + 
- (end_time.getTime() - startTime.getTime()) /1000 +  
seconds.);
-  
-  // Check to ensure that the statistics of the 
-  // framework's sort-input and sort-output match
-  SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs,
-  new Path(outputPath, 
part-0), defaults);
-  IntWritable k1 = new IntWritable();
-  IntWritable k2 = new IntWritable();
-  RecordStatsWritable v1 = new RecordStatsWritable();
-  RecordStatsWritable v2 = new RecordStatsWritable();
-  if (!stats.next(k1, v1)) {
-throw new IOException(Failed to read record #1 from reduce's output);
-  }
-  if (!stats.next(k2, v2)) {
-throw new IOException(Failed to read record #2 from reduce's output);
-  }
-
-  if ((v1.getBytes() != v2.getBytes()) || (v1.getRecords() != 
v2.getRecords()) || 
-  v1.getChecksum() != v2.getChecksum()) {
-throw new IOException(( + 
-  v1.getBytes() + ,  + v1.getRecords() + ,  + 
v1.getChecksum() + ) v/s ( +
-  v2.getBytes() + ,  + v2.getRecords() + ,  + 
v2.getChecksum() + ));
+  try {
+Date end_time = new Date();
+System.out.println(Job ended:  + end_time);
+

svn commit: r1367795 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ src/test/ma

[jeagles]

Author: jeagles
Date: Tue Jul 31 21:49:02 2012
New Revision: 1367795

URL: http://svn.apache.org/viewvc?rev=1367795view=rev
Log:
svn merge -c 1367789 FIXES: MAPREDUCE-4234. SortValidator.java is incompatible 
with multi-user or parallel use (due to a /tmp file with static name) (Robert 
Evans via jeagles)


Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java

hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367795r1=1367794r2=1367795view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt 
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue 
Jul 31 21:49:02 2012
@@ -375,6 +375,10 @@ Release 0.23.3 - UNRELEASED
 MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at 
 FAILED  (Robert Evans via tgraves)
 
+MAPREDUCE-4234. SortValidator.java is incompatible with multi-user or
+parallel use (due to a /tmp file with static name) (Robert Evans via
+jeagles)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java?rev=1367795r1=1367794r2=1367795view=diff
==
--- 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
 (original)
+++ 
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
 Tue Jul 31 21:49:02 2012
@@ -33,7 +33,6 @@ import org.apache.hadoop.io.WritableComp
 import org.apache.hadoop.io.WritableComparator;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.mapred.lib.HashPartitioner;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.fs.*;
@@ -345,7 +344,8 @@ public class SortValidator extends Confi
 
   FileInputFormat.setInputPaths(jobConf, sortInput);
   FileInputFormat.addInputPath(jobConf, sortOutput);
-  Path outputPath = new Path(/tmp/sortvalidate/recordstatschecker);
+  Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir, 
/tmp),
+   sortvalidate), UUID.randomUUID().toString());
   if (defaultfs.exists(outputPath)) {
 defaultfs.delete(outputPath, true);
   }
@@ -365,31 +365,44 @@ public class SortValidator extends Confi
   Date startTime = new Date();
   System.out.println(Job started:  + startTime);
   JobClient.runJob(jobConf);
-  Date end_time = new Date();
-  System.out.println(Job ended:  + end_time);
-  System.out.println(The job took  + 
- (end_time.getTime() - startTime.getTime()) /1000 +  
seconds.);
-  
-  // Check to ensure that the statistics of the 
-  // framework's sort-input and sort-output match
-  SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs,
-  new Path(outputPath, 
part-0), defaults);
-  IntWritable k1 = new IntWritable();
-  IntWritable k2 = new IntWritable();
-  RecordStatsWritable v1 = new RecordStatsWritable();
-  RecordStatsWritable v2 = new RecordStatsWritable();
-  if (!stats.next(k1, v1)) {
-throw new IOException(Failed to read record #1 from reduce's output);
-  }
-  if (!stats.next(k2, v2)) {
-throw new IOException(Failed to read record #2 from reduce's output);
-  }
-
-  if ((v1.getBytes() != v2.getBytes()) || (v1.getRecords() != 
v2.getRecords()) || 
-  v1.getChecksum() != v2.getChecksum()) {
-throw new IOException(( + 
-  v1.getBytes() + ,  + v1.getRecords() + ,  + 
v1.getChecksum() + ) v/s ( +
-  v2.getBytes() + ,  + v2.getRecords() + ,  + 
v2.getChecksum() + ));
+  try {
+Date end_time = new Date();
+System.out.println(Job ended:  +