Author: bobby
Date: Tue Jan 15 15:46:20 2013
New Revision: 1433469
URL: http://svn.apache.org/viewvc?rev=1433469&view=rev
Log:
MAPREDUCE-4921. JobClient should acquire HS token with RM principal (Daryn
Sharp via bobby)
Added:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
- copied, changed from r1433463,
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java
Removed:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1433469&r1=1433468&r2=1433469&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue
Jan 15 15:46:20 2013
@@ -86,6 +86,9 @@ Release 0.23.6 - UNRELEASED
MAPREDUCE-4925. The pentomino option parser may be buggy.
(Karthik Kambatla via harsh)
+ MAPREDUCE-4921. JobClient should acquire HS token with RM principal
+ (Daryn Sharp via bobby)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java?rev=1433469&r1=1433468&r2=1433469&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
Tue Jan 15 15:46:20 2013
@@ -144,13 +144,9 @@ public class JobClient extends CLI {
* we have to add this hack.
*/
private boolean getDelegationTokenCalled = false;
- /* notes the renewer that will renew the delegation token */
- private String dtRenewer = null;
/* do we need a HS delegation token for this client */
static final String HS_DELEGATION_TOKEN_REQUIRED
= "mapreduce.history.server.delegationtoken.required";
- static final String HS_DELEGATION_TOKEN_RENEWER
- = "mapreduce.history.server.delegationtoken.renewer";
static{
ConfigUtil.loadResources();
@@ -568,8 +564,6 @@ public class JobClient extends CLI {
if (getDelegationTokenCalled) {
conf.setBoolean(HS_DELEGATION_TOKEN_REQUIRED,
getDelegationTokenCalled);
getDelegationTokenCalled = false;
- conf.set(HS_DELEGATION_TOKEN_RENEWER, dtRenewer);
- dtRenewer = null;
}
Job job = clientUgi.doAs(new PrivilegedExceptionAction<Job> () {
@Override
@@ -1170,7 +1164,6 @@ public class JobClient extends CLI {
public Token<DelegationTokenIdentifier>
getDelegationToken(final Text renewer) throws IOException,
InterruptedException {
getDelegationTokenCalled = true;
- dtRenewer = renewer.toString();
return clientUgi.doAs(new
PrivilegedExceptionAction<Token<DelegationTokenIdentifier>>() {
public Token<DelegationTokenIdentifier> run() throws IOException,
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1433469&r1=1433468&r2=1433469&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Tue Jan 15 15:46:20 2013
@@ -85,6 +85,7 @@ import org.apache.hadoop.yarn.util.Build
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.ProtoUtils;
+import com.google.common.annotations.VisibleForTesting;
/**
* This class enables the current JobClient (0.22 hadoop) to run on YARN.
@@ -184,12 +185,12 @@ public class YARNRunner implements Clien
return resMgrDelegate.getClusterMetrics();
}
- private Token<?> getDelegationTokenFromHS(
- MRClientProtocol hsProxy, Text renewer) throws IOException,
- InterruptedException {
+ @VisibleForTesting
+ Token<?> getDelegationTokenFromHS(MRClientProtocol hsProxy)
+ throws IOException, InterruptedException {
GetDelegationTokenRequest request = recordFactory
.newRecordInstance(GetDelegationTokenRequest.class);
- request.setRenewer(renewer.toString());
+ request.setRenewer(Master.getMasterPrincipal(conf));
DelegationToken mrDelegationToken = hsProxy.getDelegationToken(request)
.getDelegationToken();
return ProtoUtils.convertFromProtoFormat(mrDelegationToken,
@@ -269,8 +270,7 @@ public class YARNRunner implements Clien
// the delegation tokens for the HistoryServer also.
if (conf.getBoolean(JobClient.HS_DELEGATION_TOKEN_REQUIRED,
DEFAULT_HS_DELEGATION_TOKEN_REQUIRED)) {
- Token hsDT = getDelegationTokenFromHS(hsProxy, new Text(
- conf.get(JobClient.HS_DELEGATION_TOKEN_RENEWER)));
+ Token hsDT = getDelegationTokenFromHS(hsProxy);
ts.addToken(hsDT.getService(), hsDT);
}
}
Copied:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
(from r1433463,
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java)
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java?p2=hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java&p1=hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java&r1=1433463&r2=1433469&rev=1433469&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
Tue Jan 15 15:46:20 2013
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.mapreduce.v2;
+package org.apache.hadoop.mapred;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doAnswer;
@@ -29,6 +29,8 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.security.PrivilegedExceptionAction;
import java.util.List;
import junit.framework.TestCase;
@@ -48,7 +50,11 @@ import org.apache.hadoop.mapreduce.JobPr
import org.apache.hadoop.mapreduce.JobStatus.State;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
+import
org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest;
+import
org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ClientRMProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
@@ -69,6 +75,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.DelegationToken;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -239,6 +246,46 @@ public class TestYARNRunner extends Test
delegate.getQueueAclsForCurrentUser();
verify(clientRMProtocol).getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class));
}
+
+ @Test
+ public void testHistoryServerToken() throws Exception {
+ final String masterPrincipal = Master.getMasterPrincipal(conf);
+
+ final MRClientProtocol hsProxy = mock(MRClientProtocol.class);
+
when(hsProxy.getDelegationToken(any(GetDelegationTokenRequest.class))).thenAnswer(
+ new Answer<GetDelegationTokenResponse>() {
+ public GetDelegationTokenResponse answer(InvocationOnMock
invocation) {
+ GetDelegationTokenRequest request =
+ (GetDelegationTokenRequest)invocation.getArguments()[0];
+ // check that the renewer matches the cluster's RM principal
+ assertEquals(request.getRenewer(), masterPrincipal);
+
+ DelegationToken token =
+ recordFactory.newRecordInstance(DelegationToken.class);
+ // none of these fields matter for the sake of the test
+ token.setKind("");
+ token.setService("");
+ token.setIdentifier(ByteBuffer.allocate(0));
+ token.setPassword(ByteBuffer.allocate(0));
+ GetDelegationTokenResponse tokenResponse =
+
recordFactory.newRecordInstance(GetDelegationTokenResponse.class);
+ tokenResponse.setDelegationToken(token);
+ return tokenResponse;
+ }
+ });
+
+ UserGroupInformation.createRemoteUser("someone").doAs(
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ yarnRunner = new YARNRunner(conf, null, null);
+ yarnRunner.getDelegationTokenFromHS(hsProxy);
+ verify(hsProxy).
+ getDelegationToken(any(GetDelegationTokenRequest.class));
+ return null;
+ }
+ });
+ }
@Test
public void testAMAdminCommandOpts() throws Exception {
