Repository: hadoop Updated Branches: refs/heads/branch-2.8 dcf2c8b3c -> 7bb48ed16
MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread. Contributed by Xuan Gong (cherry picked from commit 59a212b6e1265adfa9b55c71b65a22157dfccf77) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7bb48ed1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7bb48ed1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7bb48ed1 Branch: refs/heads/branch-2.8 Commit: 7bb48ed162b5216caff643df4586c01230c43c53 Parents: dcf2c8b Author: Jason Lowe <jl...@apache.org> Authored: Mon Feb 1 16:05:06 2016 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Mon Feb 1 16:11:20 2016 +0000 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 6 +++++ .../org/apache/hadoop/mapred/ClientCache.java | 24 +++++++++++++++++++- .../hadoop/mapred/ClientServiceDelegate.java | 16 +++++++++++++ .../org/apache/hadoop/mapred/YARNRunner.java | 11 +++++++++ .../mapred/YarnClientProtocolProvider.java | 5 ++-- .../TestYarnClientProtocolProvider.java | 6 +++-- 6 files changed, 63 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bb48ed1/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a82f872..a745bed 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -433,6 +433,9 @@ Release 2.7.3 - UNRELEASED MAPREDUCE-6554. MRAppMaster servicestart failing with NPE in MRAppMaster#parsePreviousJobHistory (Bibin A Chundatt via jlowe) + MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread. + (Xuan Gong via jlowe) + Release 2.7.2 - 2016-01-25 INCOMPATIBLE CHANGES @@ -735,6 +738,9 @@ Release 2.6.4 - UNRELEASED MAPREDUCE-6554. MRAppMaster servicestart failing with NPE in MRAppMaster#parsePreviousJobHistory (Bibin A Chundatt via jlowe) + MAPREDUCE-6618. YarnClientProtocolProvider leaking the YarnClient thread. + (Xuan Gong via jlowe) + Release 2.6.3 - 2015-12-17 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bb48ed1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java index 4335c82..93ea5c4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java @@ -22,11 +22,11 @@ import java.io.IOException; import java.security.PrivilegedAction; import java.util.HashMap; import java.util.Map; - import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; @@ -97,4 +97,26 @@ public class ClientCache { } }); } + + public void close() throws IOException { + if (rm != null) { + rm.close(); + } + + if (hsProxy != null) { + RPC.stopProxy(hsProxy); + hsProxy = null; + } + + if (cache != null && !cache.isEmpty()) { + for (ClientServiceDelegate delegate : cache.values()) { + if (delegate != null) { + delegate.close(); + delegate = null; + } + } + cache.clear(); + cache = null; + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bb48ed1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java index 8517c19..eac8dbc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java @@ -33,6 +33,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -532,4 +533,19 @@ public class ClientServiceDelegate { throw new IOException("Cannot get log path for a in-progress job"); } } + + public void close() throws IOException { + if (rm != null) { + rm.close(); + } + + if (historyServerProxy != null) { + RPC.stopProxy(historyServerProxy); + } + + if (realProxy != null) { + RPC.stopProxy(realProxy); + realProxy = null; + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bb48ed1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index fe631de..1342282 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -756,4 +756,15 @@ public class YARNRunner implements ClientProtocol { envConf + " config settings."); } } + + public void close() throws IOException { + if (resMgrDelegate != null) { + resMgrDelegate.close(); + resMgrDelegate = null; + } + if (clientCache != null) { + clientCache.close(); + clientCache = null; + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bb48ed1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java index 29d3f66..6583fa2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YarnClientProtocolProvider.java @@ -44,7 +44,8 @@ public class YarnClientProtocolProvider extends ClientProtocolProvider { @Override public void close(ClientProtocol clientProtocol) throws IOException { - // nothing to do + if (clientProtocol instanceof YARNRunner) { + ((YARNRunner)clientProtocol).close(); + } } - } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bb48ed1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java index aeb20cd..4d84fa9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java @@ -21,10 +21,11 @@ package org.apache.hadoop.mapreduce; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.doNothing; import java.io.IOException; import java.nio.ByteBuffer; - import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; @@ -113,6 +114,8 @@ public class TestYarnClientProtocolProvider extends TestCase { @Override protected void serviceStart() throws Exception { assertTrue(this.client instanceof YarnClientImpl); + this.client = spy(this.client); + doNothing().when(this.client).close(); ((YarnClientImpl) this.client).setRMClient(cRMProtocol); } }; @@ -126,5 +129,4 @@ public class TestYarnClientProtocolProvider extends TestCase { } } } - }
