Author: tomwhite
Date: Tue Jan 8 16:26:33 2013
New Revision: 1430363
URL: http://svn.apache.org/viewvc?rev=1430363&view=rev
Log:
MAPREDUCE-4278. Cannot run two local jobs in parallel from the same gateway.
Contributed by Sandy Ryza.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1430363&r1=1430362&r2=1430363&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jan 8
16:26:33 2013
@@ -238,6 +238,9 @@ Release 2.0.3-alpha - Unreleased
MAPREDUCE-4856. TestJobOutputCommitter uses same directory as
TestJobCleanup. (Sandy Ryza via tomwhite)
+ MAPREDUCE-4278. Cannot run two local jobs in parallel from the same
+ gateway. (Sandy Ryza via tomwhite)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java?rev=1430363&r1=1430362&r2=1430363&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
Tue Jan 8 16:26:33 2013
@@ -609,8 +609,12 @@ public class LocalJobRunner implements C
// JobSubmissionProtocol methods
private static int jobid = 0;
+ // used for making sure that local jobs run in different jvms don't
+ // collide on staging or job directories
+ private int randid;
+
public synchronized org.apache.hadoop.mapreduce.JobID getNewJobID() {
- return new org.apache.hadoop.mapreduce.JobID("local", ++jobid);
+ return new org.apache.hadoop.mapreduce.JobID("local" + randid, ++jobid);
}
public org.apache.hadoop.mapreduce.JobStatus submitJob(
@@ -739,10 +743,11 @@ public class LocalJobRunner implements C
"/tmp/hadoop/mapred/staging"));
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
String user;
+ randid = rand.nextInt(Integer.MAX_VALUE);
if (ugi != null) {
- user = ugi.getShortUserName() + rand.nextInt();
+ user = ugi.getShortUserName() + randid;
} else {
- user = "dummy" + rand.nextInt();
+ user = "dummy" + randid;
}
return fs.makeQualified(new Path(stagingRootDir,
user+"/.staging")).toString();
}
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java?rev=1430363&r1=1430362&r2=1430363&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java
Tue Jan 8 16:26:33 2013
@@ -32,7 +32,7 @@ import org.apache.hadoop.io.Text;
* the job. JobID consists of two parts. First part
* represents the jobtracker identifier, so that jobID to jobtracker map
* is defined. For cluster setup this string is the jobtracker
- * start time, for local setting, it is "local".
+ * start time, for local setting, it is "local" and a random number.
* Second part of the JobID is the job number. <br>
* An example JobID is :
* <code>job_200707121733_0003</code> , which represents the third job
