Author: suresh
Date: Sat Apr 20 16:57:44 2013
New Revision: 1470194
URL: http://svn.apache.org/r1470194
Log:
HDFS-4434. Reverting change r1470089 that merges trunk to HDFS-2802.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/ (props
changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Reverse-merged /hadoop/common/trunk/hadoop-mapreduce-project:r1469644-1470045
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt?rev=1470194&r1=1470193&r2=1470194&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt Sat
Apr 20 16:57:44 2013
@@ -134,10 +134,6 @@ Trunk (Unreleased)
MAPREDUCE-4885. Streaming tests have multiple failures on Windows. (Chris
Nauroth via bikas)
- MAPREDUCE-4987. TestMRJobs#testDistributedCache fails on Windows due to
- classpath problems and unexpected behavior of symlinks (Chris Nauroth via
- bikas)
-
BREAKDOWN OF HADOOP-8562 SUBTASKS
MAPREDUCE-4739. Some MapReduce tests fail to find winutils.
@@ -324,9 +320,6 @@ Release 2.0.5-beta - UNRELEASED
MAPREDUCE-4932. mapreduce.job#getTaskCompletionEvents incompatible with
Hadoop 1. (rkanter via tucu)
- MAPREDUCE-5163. Update MR App to not use API utility methods for
collections
- after YARN-441. (Xuan Gong via vinodkv)
-
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
@@ -889,9 +882,6 @@ Release 0.23.8 - UNRELEASED
MAPREDUCE-5015. Coverage fix for org.apache.hadoop.mapreduce.tools.CLI
(Aleksey Gorshkov via tgraves)
- MAPREDUCE-5147. Maven build should create
- hadoop-mapreduce-client-app-VERSION.jar directly (Robert Parker via
tgraves)
-
Release 0.23.7 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Reverse-merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1469644-1470045
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Reverse-merged
/hadoop/common/trunk/hadoop-mapreduce-project/conf:r1469644-1470045
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml?rev=1470194&r1=1470193&r2=1470194&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
Sat Apr 20 16:57:44 2013
@@ -76,6 +76,8 @@
</dependencies>
<build>
+ <!-- local name for links -->
+ <finalName>mr-app</finalName>
<plugins>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
@@ -88,6 +90,26 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-mr-app-symlinks</id>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <symlink link="${applink.base}.jar"
+ resource="mr-app.jar" failonerror="false"/>
+ <symlink link="${applink.base}-3.0.0-SNAPSHOT.jar"
+ resource="mr-app.jar" failonerror="false"/>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java?rev=1470194&r1=1470193&r2=1470194&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
Sat Apr 20 16:57:44 2013
@@ -157,9 +157,8 @@ public class ContainerLauncherImpl exten
startRequest.setContainer(event.getAllocatedContainer());
StartContainerResponse response = proxy.startContainer(startRequest);
- ByteBuffer portInfo =
- response.getAllServiceResponse().get(
- ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID);
+ ByteBuffer portInfo = response
+ .getServiceResponse(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID);
int port = -1;
if(portInfo != null) {
port = ShuffleHandler.deserializeMetaData(portInfo);
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java?rev=1470194&r1=1470193&r2=1470194&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
Sat Apr 20 16:57:44 2013
@@ -26,11 +26,7 @@ import static org.mockito.Mockito.when;
import static org.mockito.Mockito.atLeast;
import org.mockito.ArgumentCaptor;
-import java.io.IOException;
import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CyclicBarrier;
@@ -62,7 +58,6 @@ import org.apache.hadoop.yarn.factories.
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.junit.Before;
import org.junit.Test;
public class TestContainerLauncherImpl {
@@ -70,15 +65,6 @@ public class TestContainerLauncherImpl {
private static final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
- private Map<String, ByteBuffer> serviceResponse =
- new HashMap<String, ByteBuffer>();
-
- @Before
- public void setup() throws IOException {
- serviceResponse.clear();
- serviceResponse.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
- ShuffleHandler.serializeMetaData(80));
- }
private static class ContainerLauncherImplUnderTest extends
ContainerLauncherImpl {
@@ -159,7 +145,8 @@ public class TestContainerLauncherImpl {
String cmAddress = "127.0.0.1:8000";
StartContainerResponse startResp =
recordFactory.newRecordInstance(StartContainerResponse.class);
- startResp.setAllServiceResponse(serviceResponse);
+ startResp.setServiceResponse(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
+ ShuffleHandler.serializeMetaData(80));
LOG.info("inserting launch event");
@@ -223,7 +210,8 @@ public class TestContainerLauncherImpl {
String cmAddress = "127.0.0.1:8000";
StartContainerResponse startResp =
recordFactory.newRecordInstance(StartContainerResponse.class);
- startResp.setAllServiceResponse(serviceResponse);
+ startResp.setServiceResponse(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
+ ShuffleHandler.serializeMetaData(80));
LOG.info("inserting cleanup event");
ContainerLauncherEvent mockCleanupEvent =
@@ -287,7 +275,8 @@ public class TestContainerLauncherImpl {
String cmAddress = "127.0.0.1:8000";
StartContainerResponse startResp =
recordFactory.newRecordInstance(StartContainerResponse.class);
- startResp.setAllServiceResponse(serviceResponse);
+ startResp.setServiceResponse(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
+ ShuffleHandler.serializeMetaData(80));
LOG.info("inserting launch event");
ContainerRemoteLaunchEvent mockLaunchEvent =
@@ -344,7 +333,8 @@ public class TestContainerLauncherImpl {
String cmAddress = "127.0.0.1:8000";
StartContainerResponse startResp =
recordFactory.newRecordInstance(StartContainerResponse.class);
- startResp.setAllServiceResponse(serviceResponse);
+ startResp.setServiceResponse(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
+ ShuffleHandler.serializeMetaData(80));
LOG.info("inserting launch event");
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Reverse-merged
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1469644-1470045
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1470194&r1=1470193&r2=1470194&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
Sat Apr 20 16:57:44 2013
@@ -18,13 +18,11 @@
package org.apache.hadoop.mapreduce.v2;
-import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.StringReader;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
@@ -49,7 +47,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
@@ -74,7 +71,6 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.JarFinder;
-import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -97,6 +93,13 @@ public class TestMRJobs {
} catch (IOException io) {
throw new RuntimeException("problem getting local fs", io);
}
+ try {
+ dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
+ .format(true).racks(null).build();
+ remoteFs = dfsCluster.getFileSystem();
+ } catch (IOException io) {
+ throw new RuntimeException("problem starting mini dfs cluster", io);
+ }
}
private static Path TEST_ROOT_DIR = new Path("target",
@@ -107,13 +110,6 @@ public class TestMRJobs {
@BeforeClass
public static void setup() throws IOException {
- try {
- dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
- .format(true).racks(null).build();
- remoteFs = dfsCluster.getFileSystem();
- } catch (IOException io) {
- throw new RuntimeException("problem starting mini dfs cluster", io);
- }
if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
@@ -219,7 +215,7 @@ public class TestMRJobs {
}
}
- @Test (timeout = 60000)
+ @Test (timeout = 30000)
public void testRandomWriter() throws IOException, InterruptedException,
ClassNotFoundException {
@@ -281,7 +277,7 @@ public class TestMRJobs {
&& counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue()
!= 0);
}
- @Test (timeout = 60000)
+ @Test (timeout = 30000)
public void testFailingMapper() throws IOException, InterruptedException,
ClassNotFoundException {
@@ -363,7 +359,7 @@ public class TestMRJobs {
return job;
}
- //@Test (timeout = 60000)
+ //@Test (timeout = 30000)
public void testSleepJobWithSecurityOn() throws IOException,
InterruptedException, ClassNotFoundException {
@@ -471,46 +467,8 @@ public class TestMRJobs {
// Check that the symlink for the Job Jar was created in the cwd and
// points to the extracted directory
File jobJarDir = new File("job.jar");
- if (Shell.WINDOWS) {
- Assert.assertTrue(isWindowsSymlinkedDirectory(jobJarDir));
- } else {
- Assert.assertTrue(FileUtils.isSymlink(jobJarDir));
- Assert.assertTrue(jobJarDir.isDirectory());
- }
- }
-
- /**
- * Used on Windows to determine if the specified file is a symlink that
- * targets a directory. On most platforms, these checks can be done using
- * commons-io. On Windows, the commons-io implementation is unreliable and
- * always returns false. Instead, this method checks the output of the dir
- * command. After migrating to Java 7, this method can be removed in favor
- * of the new method java.nio.file.Files.isSymbolicLink, which is expected
to
- * work cross-platform.
- *
- * @param file File to check
- * @return boolean true if the file is a symlink that targets a directory
- * @throws IOException thrown for any I/O error
- */
- private static boolean isWindowsSymlinkedDirectory(File file)
- throws IOException {
- String dirOut = Shell.execCommand("cmd", "/c", "dir",
- file.getAbsoluteFile().getParent());
- StringReader sr = new StringReader(dirOut);
- BufferedReader br = new BufferedReader(sr);
- try {
- String line = br.readLine();
- while (line != null) {
- line = br.readLine();
- if (line.contains(file.getName()) && line.contains("<SYMLINKD>")) {
- return true;
- }
- }
- return false;
- } finally {
- IOUtils.closeStream(br);
- IOUtils.closeStream(sr);
- }
+ Assert.assertTrue(FileUtils.isSymlink(jobJarDir));
+ Assert.assertTrue(jobJarDir.isDirectory());
}
/**
@@ -584,7 +542,7 @@ public class TestMRJobs {
trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));
}
- @Test (timeout = 600000)
+ @Test (timeout = 300000)
public void testDistributedCache() throws Exception {
// Test with a local (file:///) Job Jar
Path localJobJarPath = makeJobJarWithLib(TEST_ROOT_DIR.toUri().toString());
