Author: bikas
Date: Fri Sep 13 23:57:56 2013
New Revision: 1523149
URL: http://svn.apache.org/r1523149
Log:
Merge r1523146 from trunk to branch-2 YARN-1116. Populate AMRMTokens back to
AMRMTokenSecretManager after RM restarts (Jian He via bikas)
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1523149&r1=1523148&r2=1523149&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Fri Sep 13
23:57:56 2013
@@ -179,6 +179,9 @@ Release 2.1.1-beta - UNRELEASED
YARN-1194. TestContainerLogsPage fails with native builds (Roman Shaposhnik
via jlowe)
+ YARN-1116. Populate AMRMTokens back to AMRMTokenSecretManager after RM
+ restarts (Jian He via bikas)
+
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1523149&r1=1523148&r2=1523149&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
Fri Sep 13 23:57:56 2013
@@ -569,7 +569,7 @@ public class RMAppImpl implements RMApp,
}
@Override
- public void recover(RMState state) {
+ public void recover(RMState state) throws Exception{
ApplicationState appState =
state.getApplicationState().get(getApplicationId());
LOG.info("Recovering app: " + getApplicationId() + " with " +
+ appState.getAttemptCount() + " attempts");
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1523149&r1=1523148&r2=1523149&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
Fri Sep 13 23:57:56 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.re
import static org.apache.hadoop.yarn.util.StringHelper.pjoin;
+import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
@@ -675,7 +676,7 @@ public class RMAppAttemptImpl implements
}
@Override
- public void recover(RMState state) {
+ public void recover(RMState state) throws Exception{
ApplicationState appState =
state.getApplicationState().get(getAppAttemptId().getApplicationId());
ApplicationAttemptState attemptState =
appState.getAttempt(getAppAttemptId());
@@ -690,7 +691,8 @@ public class RMAppAttemptImpl implements
RMAppAttemptEventType.RECOVER));
}
- private void recoverAppAttemptCredentials(Credentials appAttemptTokens) {
+ private void recoverAppAttemptCredentials(Credentials appAttemptTokens)
+ throws IOException {
if (appAttemptTokens == null) {
return;
}
@@ -707,11 +709,7 @@ public class RMAppAttemptImpl implements
this.amrmToken =
(Token<AMRMTokenIdentifier>) appAttemptTokens
.getToken(RMStateStore.AM_RM_TOKEN_SERVICE);
-
- // For now, no need to populate tokens back to AMRMTokenSecretManager,
- // because running attempts are rebooted. Later in work-preserve restart,
- // we'll create NEW->RUNNING transition in which the restored tokens will
be
- // added to the secret manager
+ rmContext.getAMRMTokenSecretManager().addPersistedPassword(this.amrmToken);
}
private static class BaseTransition implements
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java?rev=1523149&r1=1523148&r2=1523149&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
Fri Sep 13 23:57:56 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.security;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
@@ -30,6 +31,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -124,6 +126,19 @@ public class AMRMTokenSecretManager exte
}
/**
+ * Populate persisted password of AMRMToken back to AMRMTokenSecretManager.
+ */
+ public synchronized void
+ addPersistedPassword(Token<AMRMTokenIdentifier> token) throws
IOException {
+ AMRMTokenIdentifier identifier = token.decodeIdentifier();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding password for " + identifier.getApplicationAttemptId());
+ }
+ this.passwords.put(identifier.getApplicationAttemptId(),
+ token.getPassword());
+ }
+
+ /**
* Retrieve the password for the given {@link AMRMTokenIdentifier}.
* Used by RPC layer to validate a remote {@link AMRMTokenIdentifier}.
*/
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1523149&r1=1523148&r2=1523149&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
Fri Sep 13 23:57:56 2013
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
@@ -577,14 +578,16 @@ public class TestRMRestart {
attempt1.getClientTokenMasterKey(),
loadedAttempt1.getClientTokenMasterKey());
- // assert secret manager also knows about the key
+ // assert ClientTokenSecretManager also knows about the key
Assert.assertArrayEquals(clientTokenMasterKey,
rm2.getClientToAMTokenSecretManager().getMasterKey(attemptId1)
.getEncoded());
- // Not testing ApplicationTokenSecretManager has the password populated
back,
- // that is needed in work-preserving restart
-
+ // assert AMRMTokenSecretManager also knows about the AMRMToken password
+ Token<AMRMTokenIdentifier> amrmToken = loadedAttempt1.getAMRMToken();
+ Assert.assertArrayEquals(amrmToken.getPassword(),
+ rm2.getAMRMTokenSecretManager().retrievePassword(
+ amrmToken.decodeIdentifier()));
rm1.stop();
rm2.stop();
}
