Author: bobby
Date: Thu Aug 23 18:10:28 2012
New Revision: 1376624
URL: http://svn.apache.org/viewvc?rev=1376624&view=rev
Log:
svn merge -c 1376618 FIXES: HADOOP-8225. DistCp fails when invoked by Oozie
(daryn via bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1376624&r1=1376623&r2=1376624&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
Thu Aug 23 18:10:28 2012
@@ -39,7 +39,6 @@ import org.apache.hadoop.fs.LocalDirAllo
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -55,7 +54,6 @@ import org.apache.hadoop.security.Creden
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
@@ -92,11 +90,15 @@ class YarnChild {
DefaultMetricsSystem.initialize(
StringUtils.camelize(firstTaskid.getTaskType().name()) +"Task");
- Token<JobTokenIdentifier> jt = loadCredentials(defaultConf, address);
-
+ // Security framework already loaded the tokens into current ugi
+ Credentials credentials =
+ UserGroupInformation.getCurrentUser().getCredentials();
+
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner =
UserGroupInformation.createRemoteUser(firstTaskid.getJobID().toString());
+ Token<JobTokenIdentifier> jt = TokenCache.getJobToken(credentials);
+ SecurityUtil.setTokenService(jt, address);
taskOwner.addToken(jt);
final TaskUmbilicalProtocol umbilical =
taskOwner.doAs(new PrivilegedExceptionAction<TaskUmbilicalProtocol>() {
@@ -132,17 +134,14 @@ class YarnChild {
YarnChild.taskid = task.getTaskID();
// Create the job-conf and set credentials
- final JobConf job =
- configureTask(task, defaultConf.getCredentials(), jt);
+ final JobConf job = configureTask(task, credentials, jt);
// Initiate Java VM metrics
JvmMetrics.initSingleton(jvmId.toString(), job.getSessionId());
childUGI = UserGroupInformation.createRemoteUser(System
.getenv(ApplicationConstants.Environment.USER.toString()));
// Add tokens to new user so that it may execute its task correctly.
- for(Token<?> token : UserGroupInformation.getCurrentUser().getTokens()) {
- childUGI.addToken(token);
- }
+ job.getCredentials().addTokensToUGI(childUGI);
// Create a final reference to the task for the doAs block
final Task taskFinal = task;
@@ -206,30 +205,6 @@ class YarnChild {
}
}
- private static Token<JobTokenIdentifier> loadCredentials(JobConf conf,
- InetSocketAddress address) throws IOException {
- //load token cache storage
- String tokenFileLocation =
- System.getenv(ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME);
- String jobTokenFile =
- new Path(tokenFileLocation).makeQualified(FileSystem.getLocal(conf))
- .toUri().getPath();
- Credentials credentials =
- TokenCache.loadTokens(jobTokenFile, conf);
- LOG.debug("loading token. # keys =" +credentials.numberOfSecretKeys() +
- "; from file=" + jobTokenFile);
- Token<JobTokenIdentifier> jt = TokenCache.getJobToken(credentials);
- SecurityUtil.setTokenService(jt, address);
- UserGroupInformation current = UserGroupInformation.getCurrentUser();
- current.addToken(jt);
- for (Token<? extends TokenIdentifier> tok : credentials.getAllTokens()) {
- current.addToken(tok);
- }
- // Set the credentials
- conf.setCredentials(credentials);
- return jt;
- }
-
/**
* Configure mapred-local dirs. This config is used by the task for finding
* out an output directory.
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1376624&r1=1376623&r2=1376624&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
Thu Aug 23 18:10:28 2012
@@ -87,8 +87,6 @@ import org.apache.hadoop.mapreduce.v2.ut
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.yarn.Clock;
@@ -489,15 +487,7 @@ public class MRAppMaster extends Composi
fsTokens.addAll(Credentials.readTokenStorageFile(jobTokenFile, conf));
LOG.info("jobSubmitDir=" + jobSubmitDir + " jobTokenFile="
+ jobTokenFile);
-
- for (Token<? extends TokenIdentifier> tk : fsTokens.getAllTokens()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Token of kind " + tk.getKind()
- + "in current ugi in the AppMaster for service "
- + tk.getService());
- }
- currentUser.addToken(tk); // For use by AppMaster itself.
- }
+ fsTokens.addTokensToUGI(currentUser); // For use by AppMaster itself.
}
} catch (IOException e) {
throw new YarnException(e);
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java?rev=1376624&r1=1376623&r2=1376624&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
Thu Aug 23 18:10:28 2012
@@ -131,6 +131,8 @@ public class Job extends JobContextImpl
Job(JobConf conf) throws IOException {
super(conf, null);
+ // propagate existing user credentials to job
+ this.credentials.mergeAll(this.ugi.getCredentials());
this.cluster = null;
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java?rev=1376624&r1=1376623&r2=1376624&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
Thu Aug 23 18:10:28 2012
@@ -18,14 +18,17 @@
package org.apache.hadoop.mapreduce;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
import java.io.IOException;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.JobStatus.State;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.junit.Assert;
import org.junit.Test;
@@ -50,4 +53,16 @@ public class TestJob {
Assert.assertNotNull(job.toString());
}
+ @Test
+ public void testUGICredentialsPropogation() throws Exception {
+ Token<?> token = mock(Token.class);
+ Text service = new Text("service");
+
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ ugi.addToken(service, token);
+
+ JobConf jobConf = new JobConf();
+ Job job = new Job(jobConf);
+ assertSame(token, job.getCredentials().getToken(service));
+ }
}
