svn commit: r1372573 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/ had
Author: bobby
Date: Mon Aug 13 20:04:49 2012
New Revision: 1372573
URL: http://svn.apache.org/viewvc?rev=1372573view=rev
Log:
svn merge -c -1369197 Reverting: MAPREDUCE-4503 in branch-0.23 until
MAPREDUCE-4549 can be addressed.
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1372573r1=1372572r2=1372573view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon
Aug 13 20:04:49 2012
@@ -382,9 +382,6 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4504. SortValidator writes to wrong directory (Robert Evans
via tgraves)
-MAPREDUCE-4503. Should throw InvalidJobConfException if duplicates found in
-cacheArchives or cacheFiles (Robert Evans via jeagles)
-
MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason
Lowe via bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1372573r1=1372572r2=1372573view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
Mon Aug 13 20:04:49 2012
@@ -35,7 +35,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.InvalidJobConfException;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskAttemptID;
@@ -57,7 +56,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.hadoop.yarn.util.ConverterUtils;
/**
* Helper class for MR applications
@@ -265,13 +263,6 @@ public class MRApps extends Apps {
DistributedCache.getFileClassPaths(conf));
}
- private static String getResourceDescription(LocalResourceType type) {
-if(type == LocalResourceType.ARCHIVE) {
- return cache archive ( + MRJobConfig.CACHE_ARCHIVES + ) ;
-}
-return cache file ( + MRJobConfig.CACHE_FILES + ) ;
- }
-
// TODO - Move this to MR!
// Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[],
// long[], boolean[], Path[], FileType)
@@ -317,13 +308,6 @@ public class MRApps extends Apps {
throw new IllegalArgumentException(Resource name must be relative);
}
String linkName = name.toUri().getPath();
-LocalResource orig = localResources.get(linkName);
-if(orig != null !orig.getResource().equals(
-ConverterUtils.getYarnUrlFromURI(p.toUri( {
- throw new InvalidJobConfException(
- getResourceDescription(orig.getType()) + orig.getResource() +
- conflicts with + getResourceDescription(type) + u);
-}
localResources.put(
linkName,
BuilderUtils.newLocalResource(
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1372573r1=1372572r2=1372573view=diff
==
---
svn commit: r1372636 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/ hadoop-mapreduce-cl
Author: tgraves
Date: Mon Aug 13 21:48:33 2012
New Revision: 1372636
URL: http://svn.apache.org/viewvc?rev=1372636view=rev
Log:
MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over group
names deprecated names don't show up (Robert Evans via tgraves)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1372636r1=1372635r2=1372636view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Mon Aug 13
21:48:33 2012
@@ -809,6 +809,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason
Lowe via bobby)
+MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over
+group names deprecated names don't show up (Robert Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1372636r1=1372635r2=1372636view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
Mon Aug 13 21:48:33 2012
@@ -24,6 +24,7 @@ import static org.apache.hadoop.mapreduc
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentSkipListMap;
@@ -185,7 +186,15 @@ public abstract class AbstractCountersC
* @return Set of counter names.
*/
public synchronized IterableString getGroupNames() {
-return Iterables.concat(fgroups.keySet(), groups.keySet());
+HashSetString deprecated = new HashSetString();
+for(Map.EntryString, String entry : legacyMap.entrySet()) {
+ String newGroup = entry.getValue();
+ boolean isFGroup = isFrameworkGroup(newGroup);
+ if(isFGroup ? fgroups.containsKey(newGroup) :
groups.containsKey(newGroup)) {
+deprecated.add(entry.getKey());
+ }
+}
+return Iterables.concat(fgroups.keySet(), groups.keySet(), deprecated);
}
@Override
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java?rev=1372636r1=1372635r2=1372636view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
Mon Aug 13 21:48:33 2012
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
import java.text.ParseException;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Random;
@@ -225,6 +226,23 @@ public class TestCounters {
}
@Test
+ public void testLegacyGetGroupNames() {
+Counters counters = new Counters();
+// create 2 filesystem counter groups
+counters.findCounter(fs1, FileSystemCounter.BYTES_READ).increment(1);
+counters.findCounter(fs2, FileSystemCounter.BYTES_READ).increment(1);
+counters.incrCounter(group1, counter1, 1);
+
+HashSetString groups = new HashSetString(counters.getGroupNames());
+HashSetString expectedGroups = new HashSetString();
+expectedGroups.add(group1);
+expectedGroups.add(FileSystemCounter); //Legacy Name
+
svn commit: r1372639 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/ hado
Author: tgraves
Date: Mon Aug 13 21:51:06 2012
New Revision: 1372639
URL: http://svn.apache.org/viewvc?rev=1372639view=rev
Log:
merge -r 1372637:1372638 from branch-2. FIXES: MAPREDUCE-4053
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1372639r1=1372638r2=1372639view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon
Aug 13 21:51:06 2012
@@ -385,6 +385,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason
Lowe via bobby)
+MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over
+group names deprecated names don't show up (Robert Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1372639r1=1372638r2=1372639view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
Mon Aug 13 21:51:06 2012
@@ -24,6 +24,7 @@ import static org.apache.hadoop.mapreduc
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentSkipListMap;
@@ -185,7 +186,15 @@ public abstract class AbstractCountersC
* @return Set of counter names.
*/
public synchronized IterableString getGroupNames() {
-return Iterables.concat(fgroups.keySet(), groups.keySet());
+HashSetString deprecated = new HashSetString();
+for(Map.EntryString, String entry : legacyMap.entrySet()) {
+ String newGroup = entry.getValue();
+ boolean isFGroup = isFrameworkGroup(newGroup);
+ if(isFGroup ? fgroups.containsKey(newGroup) :
groups.containsKey(newGroup)) {
+deprecated.add(entry.getKey());
+ }
+}
+return Iterables.concat(fgroups.keySet(), groups.keySet(), deprecated);
}
@Override
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java?rev=1372639r1=1372638r2=1372639view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
Mon Aug 13 21:51:06 2012
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
import java.text.ParseException;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Random;
@@ -225,6 +226,23 @@ public class TestCounters {
}
@Test
+ public void testLegacyGetGroupNames() {
+Counters counters = new Counters();
+// create 2 filesystem counter groups
+counters.findCounter(fs1, FileSystemCounter.BYTES_READ).increment(1);
+counters.findCounter(fs2, FileSystemCounter.BYTES_READ).increment(1);
+counters.incrCounter(group1, counter1, 1);
+
+HashSetString groups = new HashSetString(counters.getGroupNames());
+
svn commit: r1372640 - in /hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counter
Author: tgraves
Date: Mon Aug 13 21:53:50 2012
New Revision: 1372640
URL: http://svn.apache.org/viewvc?rev=1372640view=rev
Log:
merge -r 1372635:1372636 from trunk. FIXES: MAPREDUCE-4053
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt?rev=1372640r1=1372639r2=1372640view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt
Mon Aug 13 21:53:50 2012
@@ -584,6 +584,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason
Lowe via bobby)
+MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over
+group names deprecated names don't show up (Robert Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1372640r1=1372639r2=1372640view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
Mon Aug 13 21:53:50 2012
@@ -24,6 +24,7 @@ import static org.apache.hadoop.mapreduc
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentSkipListMap;
@@ -185,7 +186,15 @@ public abstract class AbstractCountersC
* @return Set of counter names.
*/
public synchronized IterableString getGroupNames() {
-return Iterables.concat(fgroups.keySet(), groups.keySet());
+HashSetString deprecated = new HashSetString();
+for(Map.EntryString, String entry : legacyMap.entrySet()) {
+ String newGroup = entry.getValue();
+ boolean isFGroup = isFrameworkGroup(newGroup);
+ if(isFGroup ? fgroups.containsKey(newGroup) :
groups.containsKey(newGroup)) {
+deprecated.add(entry.getKey());
+ }
+}
+return Iterables.concat(fgroups.keySet(), groups.keySet(), deprecated);
}
@Override
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java?rev=1372640r1=1372639r2=1372640view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
Mon Aug 13 21:53:50 2012
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
import java.text.ParseException;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Random;
@@ -225,6 +226,23 @@ public class TestCounters {
}
@Test
+ public void testLegacyGetGroupNames() {
+Counters counters = new Counters();
+// create 2 filesystem counter groups
+counters.findCounter(fs1, FileSystemCounter.BYTES_READ).increment(1);
+counters.findCounter(fs2, FileSystemCounter.BYTES_READ).increment(1);
+
