YARN-3336. FileSystem memory leak in DelegationTokenRenewer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ca1f120 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ca1f120 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ca1f120 Branch: refs/heads/YARN-2139 Commit: 6ca1f12024fd7cec7b01df0f039ca59f3f365dc1 Parents: 7e6f384 Author: cnauroth <cnaur...@apache.org> Authored: Mon Mar 23 10:45:50 2015 -0700 Committer: cnauroth <cnaur...@apache.org> Committed: Mon Mar 23 10:45:50 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../security/DelegationTokenRenewer.java | 13 +++++++-- .../security/TestDelegationTokenRenewer.java | 30 ++++++++++++++++++-- 3 files changed, 41 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ca1f120/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e04624e..b90109c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -819,6 +819,9 @@ Release 2.7.0 - UNRELEASED YARN-3384. TestLogAggregationService.verifyContainerLogs fails after YARN-2777. (Naganarasimha G R via ozawa) + YARN-3336. FileSystem memory leak in DelegationTokenRenewer. + (Zhihai Xu via cnauroth) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ca1f120/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index cb456d8..2619971 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -605,6 +605,7 @@ public class DelegationTokenRenewer extends AbstractService { rmContext.getSystemCredentialsForApps().put(applicationId, byteBuffer); } + @VisibleForTesting protected Token<?>[] obtainSystemTokensForUser(String user, final Credentials credentials) throws IOException, InterruptedException { // Get new hdfs tokens on behalf of this user @@ -615,8 +616,16 @@ public class DelegationTokenRenewer extends AbstractService { proxyUser.doAs(new PrivilegedExceptionAction<Token<?>[]>() { @Override public Token<?>[] run() throws Exception { - return FileSystem.get(getConfig()).addDelegationTokens( - UserGroupInformation.getLoginUser().getUserName(), credentials); + FileSystem fs = FileSystem.get(getConfig()); + try { + return fs.addDelegationTokens( + UserGroupInformation.getLoginUser().getUserName(), + credentials); + } finally { + // Close the FileSystem created by the new proxy user, + // So that we don't leave an entry in the FileSystem cache + fs.close(); + } } }); return newTokens; http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ca1f120/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 5d31404..99a506a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -287,9 +287,16 @@ public class TestDelegationTokenRenewer { * exception */ static class MyFS extends DistributedFileSystem { - - public MyFS() {} - public void close() {} + private static AtomicInteger instanceCounter = new AtomicInteger(); + public MyFS() { + instanceCounter.incrementAndGet(); + } + public void close() { + instanceCounter.decrementAndGet(); + } + public static int getInstanceCounter() { + return instanceCounter.get(); + } @Override public void initialize(URI uri, Configuration conf) throws IOException {} @@ -299,6 +306,11 @@ public class TestDelegationTokenRenewer { LOG.info("Called MYDFS.getdelegationtoken " + result); return result; } + + public Token<?>[] addDelegationTokens( + final String renewer, Credentials credentials) throws IOException { + return new Token<?>[0]; + } } /** @@ -1022,4 +1034,16 @@ public class TestDelegationTokenRenewer { // app2 completes, app1 is still running, check the token is not cancelled Assert.assertFalse(Renewer.cancelled); } + + // Test FileSystem memory leak in obtainSystemTokensForUser. + @Test + public void testFSLeakInObtainSystemTokensForUser() throws Exception{ + Credentials credentials = new Credentials(); + String user = "test"; + int oldCounter = MyFS.getInstanceCounter(); + delegationTokenRenewer.obtainSystemTokensForUser(user, credentials); + delegationTokenRenewer.obtainSystemTokensForUser(user, credentials); + delegationTokenRenewer.obtainSystemTokensForUser(user, credentials); + Assert.assertEquals(oldCounter, MyFS.getInstanceCounter()); + } }