Author: jing9
Date: Fri Aug 8 22:21:35 2014
New Revision: 1616897
URL: http://svn.apache.org/r1616897
Log:
Merging r1616428 through r1616893 from trunk.
Modified:
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
Propchange: hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1616428-1616893
Modified: hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/CHANGES.txt?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/CHANGES.txt Fri
Aug 8 22:21:35 2014
@@ -184,6 +184,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6014. New task status field in task attempts table can lead to
an empty web page (Mit Desai via jlowe)
+ MAPREDUCE-6021. MR AM should have working directory in LD_LIBRARY_PATH
+ (jlowe)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1616428-1616893
Modified:
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Fri Aug 8 22:21:35 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapreduce.v2.app.rm;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -38,6 +39,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.JobCounter;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
@@ -58,6 +60,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent;
import org.apache.hadoop.mapreduce.v2.app.rm.preemption.AMPreemptionPolicy;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.Container;
@@ -70,9 +73,11 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.client.api.NMTokenCache;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.RackResolver;
@@ -674,7 +679,12 @@ public class RMContainerAllocator extend
nmToken.getToken());
}
}
-
+
+ // Setting AMRMToken
+ if (response.getAMRMToken() != null) {
+ updateAMRMToken(response.getAMRMToken());
+ }
+
List<ContainerStatus> finishedContainers =
response.getCompletedContainersStatuses();
// propagate preemption requests
@@ -726,7 +736,19 @@ public class RMContainerAllocator extend
}
return newContainers;
}
-
+
+ private void updateAMRMToken(Token token) throws IOException {
+ org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken =
+ new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token
+ .getIdentifier().array(), token.getPassword().array(), new Text(
+ token.getKind()), new Text(token.getService()));
+ UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
+ if (UserGroupInformation.isSecurityEnabled()) {
+ currentUGI = UserGroupInformation.getLoginUser();
+ }
+ currentUGI.addToken(amrmToken);
+ }
+
@VisibleForTesting
public TaskAttemptEvent createContainerFinishedEvent(ContainerStatus cont,
TaskAttemptId attemptID) {
Modified:
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Fri Aug 8 22:21:35 2014
@@ -447,6 +447,15 @@ public class YARNRunner implements Clien
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, conf);
+ // Shell
+ environment.put(Environment.SHELL.name(),
+ conf.get(MRJobConfig.MAPRED_ADMIN_USER_SHELL,
+ MRJobConfig.DEFAULT_SHELL));
+
+ // Add the container working directory at the front of LD_LIBRARY_PATH
+ MRApps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
+ MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
+
// Setup the environment variables for Admin first
MRApps.setEnvFromInputString(environment,
conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV), conf);
Modified:
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java?rev=1616897&r1=1616896&r2=1616897&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
Fri Aug 8 22:21:35 2014
@@ -29,6 +29,7 @@ import static org.mockito.Mockito.when;
import java.io.ByteArrayOutputStream;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@@ -36,6 +37,7 @@ import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
import java.util.List;
+import java.util.Map;
import junit.framework.TestCase;
@@ -44,22 +46,27 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.JobPriority;
import org.apache.hadoop.mapreduce.JobStatus.State;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
import org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier;
import
org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest;
import
org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
@@ -94,6 +101,7 @@ import org.apache.log4j.Layout;
import org.apache.log4j.Logger;
import org.apache.log4j.SimpleLayout;
import org.apache.log4j.WriterAppender;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
@@ -146,8 +154,12 @@ public class TestYARNRunner extends Test
FileContext.getLocalFSFileContext().delete(new
Path(testWorkDir.toString()), true);
}
testWorkDir.mkdirs();
- }
+ }
+ @After
+ public void cleanup() {
+ FileUtil.fullyDelete(testWorkDir);
+ }
@Test(timeout=20000)
public void testJobKill() throws Exception {
@@ -397,21 +409,8 @@ public class TestYARNRunner extends Test
YARNRunner yarnRunner = new YARNRunner(jobConf);
- File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE);
- OutputStream out = new FileOutputStream(jobxml);
- conf.writeXml(out);
- out.close();
-
- File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT);
- out = new FileOutputStream(jobsplit);
- out.close();
-
- File jobsplitmetainfo = new File(testWorkDir,
MRJobConfig.JOB_SPLIT_METAINFO);
- out = new FileOutputStream(jobsplitmetainfo);
- out.close();
-
- ApplicationSubmissionContext submissionContext =
- yarnRunner.createApplicationSubmissionContext(jobConf,
testWorkDir.toString(), new Credentials());
+ ApplicationSubmissionContext submissionContext =
+ buildSubmitContext(yarnRunner, jobConf);
ContainerLaunchContext containerSpec =
submissionContext.getAMContainerSpec();
List<String> commands = containerSpec.getCommands();
@@ -463,22 +462,9 @@ public class TestYARNRunner extends Test
YARNRunner yarnRunner = new YARNRunner(jobConf);
- File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE);
- OutputStream out = new FileOutputStream(jobxml);
- conf.writeXml(out);
- out.close();
-
- File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT);
- out = new FileOutputStream(jobsplit);
- out.close();
-
- File jobsplitmetainfo = new File(testWorkDir,
MRJobConfig.JOB_SPLIT_METAINFO);
- out = new FileOutputStream(jobsplitmetainfo);
- out.close();
-
@SuppressWarnings("unused")
- ApplicationSubmissionContext submissionContext =
- yarnRunner.createApplicationSubmissionContext(jobConf,
testWorkDir.toString(), new Credentials());
+ ApplicationSubmissionContext submissionContext =
+ buildSubmitContext(yarnRunner, jobConf);
String logMsg = bout.toString();
assertTrue(logMsg.contains("WARN - Usage of -Djava.library.path in " +
@@ -492,4 +478,60 @@ public class TestYARNRunner extends Test
"be set as part of the LD_LIBRARY_PATH in the app master JVM env " +
"using yarn.app.mapreduce.am.env config settings."));
}
+
+ @Test
+ public void testAMStandardEnv() throws Exception {
+ final String ADMIN_LIB_PATH = "foo";
+ final String USER_LIB_PATH = "bar";
+ final String USER_SHELL = "shell";
+ JobConf jobConf = new JobConf();
+
+ jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" +
+ ADMIN_LIB_PATH);
+ jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH="
+ + USER_LIB_PATH);
+ jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL);
+
+ YARNRunner yarnRunner = new YARNRunner(jobConf);
+ ApplicationSubmissionContext appSubCtx =
+ buildSubmitContext(yarnRunner, jobConf);
+
+ // make sure PWD is first in the lib path
+ ContainerLaunchContext clc = appSubCtx.getAMContainerSpec();
+ Map<String, String> env = clc.getEnvironment();
+ String libPath = env.get(Environment.LD_LIBRARY_PATH.name());
+ assertNotNull("LD_LIBRARY_PATH not set", libPath);
+ String cps = jobConf.getBoolean(
+ MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
+ MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
+ ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
+ assertEquals("Bad AM LD_LIBRARY_PATH setting",
+ MRApps.crossPlatformifyMREnv(conf, Environment.PWD)
+ + cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath);
+
+ // make sure SHELL is set
+ String shell = env.get(Environment.SHELL.name());
+ assertNotNull("SHELL not set", shell);
+ assertEquals("Bad SHELL setting", USER_SHELL, shell);
+ }
+
+ private ApplicationSubmissionContext buildSubmitContext(
+ YARNRunner yarnRunner, JobConf jobConf) throws IOException {
+ File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE);
+ OutputStream out = new FileOutputStream(jobxml);
+ conf.writeXml(out);
+ out.close();
+
+ File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT);
+ out = new FileOutputStream(jobsplit);
+ out.close();
+
+ File jobsplitmetainfo = new File(testWorkDir,
+ MRJobConfig.JOB_SPLIT_METAINFO);
+ out = new FileOutputStream(jobsplitmetainfo);
+ out.close();
+
+ return yarnRunner.createApplicationSubmissionContext(jobConf,
+ testWorkDir.toString(), new Credentials());
+ }
}
