Repository: hadoop Updated Branches: refs/heads/branch-2.6 843dac535 -> 70289432f
YARN-3857: Memory leak in ResourceManager with SIMPLE mode. Contributed by mujunchao. (cherry picked from commit 3a76a010b85176f2bcb85ed6f74c25dcb8acfe4d) Conflicts: hadoop-yarn-project/CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/70289432 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/70289432 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/70289432 Branch: refs/heads/branch-2.6 Commit: 70289432f7aee2793ad75f1b018a1681b20be8ee Parents: 843dac5 Author: Zhihai Xu <z...@apache.org> Authored: Tue Aug 18 10:36:40 2015 -0700 Committer: Zhihai Xu <z...@apache.org> Committed: Mon Dec 14 22:17:23 2015 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 2 ++ .../rmapp/attempt/RMAppAttemptImpl.java | 8 +++-- .../ClientToAMTokenSecretManagerInRM.java | 7 +++++ .../attempt/TestRMAppAttemptTransitions.java | 32 ++++++++++++++++++++ 4 files changed, 46 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/70289432/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 19fa0c2..7200e20 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -14,6 +14,8 @@ Release 2.6.4 - UNRELEASED YARN-2975. FSLeafQueue app lists are accessed without required locks. (kasha) + YARN-3857. Memory leak in ResourceManager with SIMPLE mode. (mujunchao via zxu) + Release 2.6.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/70289432/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 13e620a..a19f5ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1262,9 +1262,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { // register the ClientTokenMasterKey after it is saved in the store, // otherwise client may hold an invalid ClientToken after RM restarts. - appAttempt.rmContext.getClientToAMTokenSecretManager() - .registerApplication(appAttempt.getAppAttemptId(), - appAttempt.getClientTokenMasterKey()); + if (UserGroupInformation.isSecurityEnabled()) { + appAttempt.rmContext.getClientToAMTokenSecretManager() + .registerApplication(appAttempt.getAppAttemptId(), + appAttempt.getClientTokenMasterKey()); + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/70289432/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java index 4fbe2ce..4047bd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; import javax.crypto.SecretKey; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager; @@ -61,4 +62,10 @@ public class ClientToAMTokenSecretManagerInRM extends ApplicationAttemptId applicationAttemptID) { return this.masterKeys.get(applicationAttemptID); } + + @VisibleForTesting + public synchronized boolean hasMasterKey( + ApplicationAttemptId applicationAttemptID) { + return this.masterKeys.containsKey(applicationAttemptID); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/70289432/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 8018f41..ccd2613 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -1363,6 +1363,38 @@ public class TestRMAppAttemptTransitions { Assert.assertNull(token); } + // this is to test master key is saved in the secret manager only after + // attempt is launched and in secure-mode + @Test + public void testApplicationAttemptMasterKey() throws Exception { + Container amContainer = allocateApplicationAttempt(); + ApplicationAttemptId appid = applicationAttempt.getAppAttemptId(); + boolean isMasterKeyExisted = false; + + // before attempt is launched, can not get MasterKey + isMasterKeyExisted = clientToAMTokenManager.hasMasterKey(appid); + Assert.assertFalse(isMasterKeyExisted); + + launchApplicationAttempt(amContainer); + // after attempt is launched and in secure mode, can get MasterKey + isMasterKeyExisted = clientToAMTokenManager.hasMasterKey(appid); + if (isSecurityEnabled) { + Assert.assertTrue(isMasterKeyExisted); + Assert.assertNotNull(clientToAMTokenManager.getMasterKey(appid)); + } else { + Assert.assertFalse(isMasterKeyExisted); + } + + applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt + .getAppAttemptId(), RMAppAttemptEventType.KILL)); + assertEquals(YarnApplicationAttemptState.LAUNCHED, + applicationAttempt.createApplicationAttemptState()); + sendAttemptUpdateSavedEvent(applicationAttempt); + // after attempt is killed, can not get MasterKey + isMasterKeyExisted = clientToAMTokenManager.hasMasterKey(appid); + Assert.assertFalse(isMasterKeyExisted); + } + @Test public void testFailedToFailed() { // create a failed attempt.