Author: cnauroth Date: Fri Aug 23 17:04:57 2013 New Revision: 1516933 URL: http://svn.apache.org/r1516933 Log: MAPREDUCE-5470. LocalJobRunner does not work on Windows. Contributed by Sandy Ryza.
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt?rev=1516933&r1=1516932&r2=1516933&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt Fri Aug 23 17:04:57 2013 @@ -64,6 +64,9 @@ Release 2.1.1-beta - UNRELEASED MAPREDUCE-5476. Changed MR AM recovery code to cleanup staging-directory only after unregistering from the RM. (Jian He via vinodkv) + MAPREDUCE-5470. LocalJobRunner does not work on Windows. (Sandy Ryza via + cnauroth) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java?rev=1516933&r1=1516932&r2=1516933&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java Fri Aug 23 17:04:57 2013 @@ -219,7 +219,7 @@ public class LocalJobRunner implements C info.getSplitIndex(), 1); map.setUser(UserGroupInformation.getCurrentUser(). getShortUserName()); - setupChildMapredLocalDirs(localJobDir, map, localConf); + setupChildMapredLocalDirs(map, localConf); MapOutputFile mapOutput = new MROutputFiles(); mapOutput.setConf(localConf); @@ -414,7 +414,7 @@ public class LocalJobRunner implements C getShortUserName()); JobConf localConf = new JobConf(job); localConf.set("mapreduce.jobtracker.address", "local"); - setupChildMapredLocalDirs(localJobDir, reduce, localConf); + setupChildMapredLocalDirs(reduce, localConf); // move map output to reduce input for (int i = 0; i < mapIds.size(); i++) { if (!this.isInterrupted()) { @@ -841,16 +841,18 @@ public class LocalJobRunner implements C throw new UnsupportedOperationException("Not supported"); } - static void setupChildMapredLocalDirs(Path localJobDir, Task t, JobConf conf) { + static void setupChildMapredLocalDirs(Task t, JobConf conf) { String[] localDirs = conf.getTrimmedStrings(MRConfig.LOCAL_DIR); + String jobId = t.getJobID().toString(); String taskId = t.getTaskID().toString(); boolean isCleanup = t.isTaskCleanupTask(); + String user = t.getUser(); StringBuffer childMapredLocalDir = new StringBuffer(localDirs[0] + Path.SEPARATOR - + getLocalTaskDir(localJobDir, taskId, isCleanup)); + + getLocalTaskDir(user, jobId, taskId, isCleanup)); for (int i = 1; i < localDirs.length; i++) { childMapredLocalDir.append("," + localDirs[i] + Path.SEPARATOR - + getLocalTaskDir(localJobDir, taskId, isCleanup)); + + getLocalTaskDir(user, jobId, taskId, isCleanup)); } LOG.debug(MRConfig.LOCAL_DIR + " for child : " + childMapredLocalDir); conf.set(MRConfig.LOCAL_DIR, childMapredLocalDir.toString()); @@ -859,9 +861,10 @@ public class LocalJobRunner implements C static final String TASK_CLEANUP_SUFFIX = ".cleanup"; static final String JOBCACHE = "jobcache"; - static String getLocalTaskDir(Path localJobDir, String taskid, + static String getLocalTaskDir(String user, String jobid, String taskid, boolean isCleanupAttempt) { - String taskDir = localJobDir.toString() + Path.SEPARATOR + taskid; + String taskDir = jobDir + Path.SEPARATOR + user + Path.SEPARATOR + JOBCACHE + + Path.SEPARATOR + jobid + Path.SEPARATOR + taskid; if (isCleanupAttempt) { taskDir = taskDir + TASK_CLEANUP_SUFFIX; }