Author: bobby
Date: Mon Aug 13 20:04:49 2012
New Revision: 1372573
URL: http://svn.apache.org/viewvc?rev=1372573&view=rev
Log:
svn merge -c -1369197 Reverting: MAPREDUCE-4503 in branch-0.23 until
MAPREDUCE-4549 can be addressed.
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1372573&r1=1372572&r2=1372573&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon
Aug 13 20:04:49 2012
@@ -382,9 +382,6 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4504. SortValidator writes to wrong directory (Robert Evans
via tgraves)
- MAPREDUCE-4503. Should throw InvalidJobConfException if duplicates found in
- cacheArchives or cacheFiles (Robert Evans via jeagles)
-
MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason
Lowe via bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1372573&r1=1372572&r2=1372573&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
Mon Aug 13 20:04:49 2012
@@ -35,7 +35,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.InvalidJobConfException;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskAttemptID;
@@ -57,7 +56,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.hadoop.yarn.util.ConverterUtils;
/**
* Helper class for MR applications
@@ -265,13 +263,6 @@ public class MRApps extends Apps {
DistributedCache.getFileClassPaths(conf));
}
- private static String getResourceDescription(LocalResourceType type) {
- if(type == LocalResourceType.ARCHIVE) {
- return "cache archive (" + MRJobConfig.CACHE_ARCHIVES + ") ";
- }
- return "cache file (" + MRJobConfig.CACHE_FILES + ") ";
- }
-
// TODO - Move this to MR!
// Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[],
// long[], boolean[], Path[], FileType)
@@ -317,13 +308,6 @@ public class MRApps extends Apps {
throw new IllegalArgumentException("Resource name must be relative");
}
String linkName = name.toUri().getPath();
- LocalResource orig = localResources.get(linkName);
- if(orig != null && !orig.getResource().equals(
- ConverterUtils.getYarnUrlFromURI(p.toUri()))) {
- throw new InvalidJobConfException(
- getResourceDescription(orig.getType()) + orig.getResource() +
- " conflicts with " + getResourceDescription(type) + u);
- }
localResources.put(
linkName,
BuilderUtils.newLocalResource(
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1372573&r1=1372572&r2=1372573&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
Mon Aug 13 20:04:49 2012
@@ -19,19 +19,13 @@
package org.apache.hadoop.mapreduce.v2.util;
import java.io.IOException;
-import java.net.URI;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FilterFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.InvalidJobConfException;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
@@ -39,12 +33,9 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.junit.Test;
import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
public class TestMRApps {
@@ -177,122 +168,5 @@ public class TestMRApps {
assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking
effect!",
env_str.indexOf("$PWD:job.jar"), 0);
}
-
- @Test
- public void testSetupDistributedCacheEmpty() throws IOException {
- Configuration conf = new Configuration();
- Map<String, LocalResource> localResources = new HashMap<String,
LocalResource>();
- MRApps.setupDistributedCache(conf, localResources);
- assertTrue("Empty Config did not produce an empty list of resources",
- localResources.isEmpty());
- }
-
- @SuppressWarnings("deprecation")
- @Test(expected = InvalidJobConfException.class)
- public void testSetupDistributedCacheConflicts() throws Exception {
- Configuration conf = new Configuration();
- conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
-
- URI mockUri = URI.create("mockfs://mock/");
- FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf))
- .getRawFileSystem();
-
- URI archive = new URI("mockfs://mock/tmp/something.zip#something");
- Path archivePath = new Path(archive);
- URI file = new URI("mockfs://mock/tmp/something.txt#something");
- Path filePath = new Path(file);
-
- when(mockFs.resolvePath(archivePath)).thenReturn(archivePath);
- when(mockFs.resolvePath(filePath)).thenReturn(filePath);
-
- DistributedCache.addCacheArchive(archive, conf);
- conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10");
- conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10");
- conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true");
- DistributedCache.addCacheFile(file, conf);
- conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11");
- conf.set(MRJobConfig.CACHE_FILES_SIZES, "11");
- conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true");
- Map<String, LocalResource> localResources =
- new HashMap<String, LocalResource>();
- MRApps.setupDistributedCache(conf, localResources);
- }
-
- @SuppressWarnings("deprecation")
- @Test(expected = InvalidJobConfException.class)
- public void testSetupDistributedCacheConflictsFiles() throws Exception {
- Configuration conf = new Configuration();
- conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
-
- URI mockUri = URI.create("mockfs://mock/");
- FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf))
- .getRawFileSystem();
-
- URI file = new URI("mockfs://mock/tmp/something.zip#something");
- Path filePath = new Path(file);
- URI file2 = new URI("mockfs://mock/tmp/something.txt#something");
- Path file2Path = new Path(file);
-
- when(mockFs.resolvePath(filePath)).thenReturn(filePath);
- when(mockFs.resolvePath(file2Path)).thenReturn(file2Path);
-
- DistributedCache.addCacheFile(file, conf);
- DistributedCache.addCacheFile(file2, conf);
- conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "10,11");
- conf.set(MRJobConfig.CACHE_FILES_SIZES, "10,11");
- conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true,true");
- Map<String, LocalResource> localResources =
- new HashMap<String, LocalResource>();
- MRApps.setupDistributedCache(conf, localResources);
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void testSetupDistributedCache() throws Exception {
- Configuration conf = new Configuration();
- conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
-
- URI mockUri = URI.create("mockfs://mock/");
- FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf))
- .getRawFileSystem();
-
- URI archive = new URI("mockfs://mock/tmp/something.zip");
- Path archivePath = new Path(archive);
- URI file = new URI("mockfs://mock/tmp/something.txt#something");
- Path filePath = new Path(file);
-
- when(mockFs.resolvePath(archivePath)).thenReturn(archivePath);
- when(mockFs.resolvePath(filePath)).thenReturn(filePath);
-
- DistributedCache.addCacheArchive(archive, conf);
- conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10");
- conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10");
- conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true");
- DistributedCache.addCacheFile(file, conf);
- conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11");
- conf.set(MRJobConfig.CACHE_FILES_SIZES, "11");
- conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true");
- Map<String, LocalResource> localResources =
- new HashMap<String, LocalResource>();
- MRApps.setupDistributedCache(conf, localResources);
- assertEquals(2, localResources.size());
- LocalResource lr = localResources.get("something.zip");
- assertNotNull(lr);
- assertEquals(10l, lr.getSize());
- assertEquals(10l, lr.getTimestamp());
- assertEquals(LocalResourceType.ARCHIVE, lr.getType());
- lr = localResources.get("something");
- assertNotNull(lr);
- assertEquals(11l, lr.getSize());
- assertEquals(11l, lr.getTimestamp());
- assertEquals(LocalResourceType.FILE, lr.getType());
- }
-
- static class MockFileSystem extends FilterFileSystem {
- MockFileSystem() {
- super(mock(FileSystem.class));
- }
- public void initialize(URI name, Configuration conf) throws IOException {}
- }
-
+
}
