Author: wang Date: Thu Aug 7 20:26:52 2014 New Revision: 1616589 URL: http://svn.apache.org/r1616589 Log: Merge from trunk to branch.
Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/ (props changed) hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt (contents, props changed) hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestBlocks.java hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java Propchange: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1615844-1616586 Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt?rev=1616589&r1=1616588&r2=1616589&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt Thu Aug 7 20:26:52 2014 @@ -181,6 +181,12 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-5756. CombineFileInputFormat.getSplits() including directories in its results (Jason Dere via jlowe) + MAPREDUCE-6014. New task status field in task attempts table can lead to + an empty web page (Mit Desai via jlowe) + + MAPREDUCE-6021. MR AM should have working directory in LD_LIBRARY_PATH + (jlowe) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1615844-1616586 Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java?rev=1616589&r1=1616588&r2=1616589&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java Thu Aug 7 20:26:52 2014 @@ -85,7 +85,8 @@ public class TaskPage extends AppView { .append(ta.getId()).append("\",\"") .append(progress).append("\",\"") .append(ta.getState().toString()).append("\",\"") - .append(ta.getStatus()).append("\",\"") + .append(StringEscapeUtils.escapeJavaScript( + StringEscapeUtils.escapeHtml(ta.getStatus()))).append("\",\"") .append(nodeHttpAddr == null ? "N/A" : "<a class='nodelink' href='" + MRWebAppUtil.getYARNWebappScheme() + nodeHttpAddr + "'>" Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java?rev=1616589&r1=1616588&r2=1616589&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java Thu Aug 7 20:26:52 2014 @@ -25,6 +25,7 @@ import static org.apache.hadoop.yarn.uti import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskInfo; @@ -102,7 +103,8 @@ public class TasksBlock extends HtmlBloc .append(join(pct, '%')).append("'> ").append("<div class='") .append(C_PROGRESSBAR_VALUE).append("' style='") .append(join("width:", pct, '%')).append("'> </div> </div>\",\"") - .append(info.getStatus()).append("\",\"") + .append(StringEscapeUtils.escapeJavaScript( + StringEscapeUtils.escapeHtml(info.getStatus()))).append("\",\"") .append(info.getState()).append("\",\"") .append(info.getStartTime()).append("\",\"") Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestBlocks.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestBlocks.java?rev=1616589&r1=1616588&r2=1616589&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestBlocks.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestBlocks.java Thu Aug 7 20:26:52 2014 @@ -106,6 +106,7 @@ public class TestBlocks { when(report.getTaskState()).thenReturn(TaskState.SUCCEEDED); when(report.getStartTime()).thenReturn(100001L); when(report.getFinishTime()).thenReturn(100011L); + when(report.getStatus()).thenReturn("Dummy Status \n*"); when(task.getReport()).thenReturn(report); @@ -134,6 +135,8 @@ public class TestBlocks { assertTrue(data.toString().contains("SUCCEEDED")); assertTrue(data.toString().contains("100001")); assertTrue(data.toString().contains("100011")); + assertFalse(data.toString().contains("Dummy Status \n*")); + assertTrue(data.toString().contains("Dummy Status \\n*")); } Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java?rev=1616589&r1=1616588&r2=1616589&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java Thu Aug 7 20:26:52 2014 @@ -149,7 +149,8 @@ public class HsTaskPage extends HsView { attemptsTableData.append("[\"") .append(sortId + " ").append(taid).append("\",\"") .append(ta.getState()).append("\",\"") - .append(ta.getStatus()).append("\",\"") + .append(StringEscapeUtils.escapeJavaScript( + StringEscapeUtils.escapeHtml(ta.getStatus()))).append("\",\"") .append("<a class='nodelink' href='" + MRWebAppUtil.getYARNWebappScheme() + nodeHttpAddr + "'>") .append(nodeRackName + "/" + nodeHttpAddr + "</a>\",\"") Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java?rev=1616589&r1=1616588&r2=1616589&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java Thu Aug 7 20:26:52 2014 @@ -159,7 +159,7 @@ public class TestBlocks { when(taReport.getSortFinishTime()).thenReturn(taSortFinishTime); when(taReport.getContainerId()).thenReturn(containerId); when(taReport.getProgress()).thenReturn(1.0f); - when(taReport.getStateString()).thenReturn("Processed 128/128 records"); + when(taReport.getStateString()).thenReturn("Processed 128/128 records <p> \n"); when(taReport.getTaskAttemptState()).thenReturn(taState); when(taReport.getDiagnosticInfo()).thenReturn(""); @@ -184,6 +184,8 @@ public class TestBlocks { // should be printed information about attempts assertTrue(data.toString().contains("0 attempt_0_0001_r_000000_0")); assertTrue(data.toString().contains("SUCCEEDED")); + assertFalse(data.toString().contains("Processed 128/128 records <p> \n")); + assertTrue(data.toString().contains("Processed 128\\/128 records <p> \\n")); assertTrue(data.toString().contains( "_0005_01_000001:attempt_0_0001_r_000000_0:User:")); assertTrue(data.toString().contains("100002")); Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1616589&r1=1616588&r2=1616589&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Thu Aug 7 20:26:52 2014 @@ -447,6 +447,15 @@ public class YARNRunner implements Clien Map<String, String> environment = new HashMap<String, String>(); MRApps.setClasspath(environment, conf); + // Shell + environment.put(Environment.SHELL.name(), + conf.get(MRJobConfig.MAPRED_ADMIN_USER_SHELL, + MRJobConfig.DEFAULT_SHELL)); + + // Add the container working directory at the front of LD_LIBRARY_PATH + MRApps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(), + MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf); + // Setup the environment variables for Admin first MRApps.setEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV), conf); Modified: hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java?rev=1616589&r1=1616588&r2=1616589&view=diff ============================================================================== --- hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java (original) +++ hadoop/common/branches/fs-encryption/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java Thu Aug 7 20:26:52 2014 @@ -29,6 +29,7 @@ import static org.mockito.Mockito.when; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -36,6 +37,7 @@ import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.List; +import java.util.Map; import junit.framework.TestCase; @@ -44,22 +46,27 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.JobPriority; import org.apache.hadoop.mapreduce.JobStatus.State; +import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; import org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse; +import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; @@ -94,6 +101,7 @@ import org.apache.log4j.Layout; import org.apache.log4j.Logger; import org.apache.log4j.SimpleLayout; import org.apache.log4j.WriterAppender; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; @@ -146,8 +154,12 @@ public class TestYARNRunner extends Test FileContext.getLocalFSFileContext().delete(new Path(testWorkDir.toString()), true); } testWorkDir.mkdirs(); - } + } + @After + public void cleanup() { + FileUtil.fullyDelete(testWorkDir); + } @Test(timeout=20000) public void testJobKill() throws Exception { @@ -397,21 +409,8 @@ public class TestYARNRunner extends Test YARNRunner yarnRunner = new YARNRunner(jobConf); - File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE); - OutputStream out = new FileOutputStream(jobxml); - conf.writeXml(out); - out.close(); - - File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT); - out = new FileOutputStream(jobsplit); - out.close(); - - File jobsplitmetainfo = new File(testWorkDir, MRJobConfig.JOB_SPLIT_METAINFO); - out = new FileOutputStream(jobsplitmetainfo); - out.close(); - - ApplicationSubmissionContext submissionContext = - yarnRunner.createApplicationSubmissionContext(jobConf, testWorkDir.toString(), new Credentials()); + ApplicationSubmissionContext submissionContext = + buildSubmitContext(yarnRunner, jobConf); ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec(); List<String> commands = containerSpec.getCommands(); @@ -463,22 +462,9 @@ public class TestYARNRunner extends Test YARNRunner yarnRunner = new YARNRunner(jobConf); - File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE); - OutputStream out = new FileOutputStream(jobxml); - conf.writeXml(out); - out.close(); - - File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT); - out = new FileOutputStream(jobsplit); - out.close(); - - File jobsplitmetainfo = new File(testWorkDir, MRJobConfig.JOB_SPLIT_METAINFO); - out = new FileOutputStream(jobsplitmetainfo); - out.close(); - @SuppressWarnings("unused") - ApplicationSubmissionContext submissionContext = - yarnRunner.createApplicationSubmissionContext(jobConf, testWorkDir.toString(), new Credentials()); + ApplicationSubmissionContext submissionContext = + buildSubmitContext(yarnRunner, jobConf); String logMsg = bout.toString(); assertTrue(logMsg.contains("WARN - Usage of -Djava.library.path in " + @@ -492,4 +478,60 @@ public class TestYARNRunner extends Test "be set as part of the LD_LIBRARY_PATH in the app master JVM env " + "using yarn.app.mapreduce.am.env config settings.")); } + + @Test + public void testAMStandardEnv() throws Exception { + final String ADMIN_LIB_PATH = "foo"; + final String USER_LIB_PATH = "bar"; + final String USER_SHELL = "shell"; + JobConf jobConf = new JobConf(); + + jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" + + ADMIN_LIB_PATH); + jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH=" + + USER_LIB_PATH); + jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL); + + YARNRunner yarnRunner = new YARNRunner(jobConf); + ApplicationSubmissionContext appSubCtx = + buildSubmitContext(yarnRunner, jobConf); + + // make sure PWD is first in the lib path + ContainerLaunchContext clc = appSubCtx.getAMContainerSpec(); + Map<String, String> env = clc.getEnvironment(); + String libPath = env.get(Environment.LD_LIBRARY_PATH.name()); + assertNotNull("LD_LIBRARY_PATH not set", libPath); + String cps = jobConf.getBoolean( + MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, + MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM) + ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator; + assertEquals("Bad AM LD_LIBRARY_PATH setting", + MRApps.crossPlatformifyMREnv(conf, Environment.PWD) + + cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath); + + // make sure SHELL is set + String shell = env.get(Environment.SHELL.name()); + assertNotNull("SHELL not set", shell); + assertEquals("Bad SHELL setting", USER_SHELL, shell); + } + + private ApplicationSubmissionContext buildSubmitContext( + YARNRunner yarnRunner, JobConf jobConf) throws IOException { + File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE); + OutputStream out = new FileOutputStream(jobxml); + conf.writeXml(out); + out.close(); + + File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT); + out = new FileOutputStream(jobsplit); + out.close(); + + File jobsplitmetainfo = new File(testWorkDir, + MRJobConfig.JOB_SPLIT_METAINFO); + out = new FileOutputStream(jobsplitmetainfo); + out.close(); + + return yarnRunner.createApplicationSubmissionContext(jobConf, + testWorkDir.toString(), new Credentials()); + } }