Repository: hive Updated Branches: refs/heads/master 5bf52be60 -> a4a0ae1ff
HIVE-16225: Memory leak in webhcat service (FileSystem CACHE entries) (Daniel Dai, reviewec by Thejas Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a4a0ae1f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a4a0ae1f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a4a0ae1f Branch: refs/heads/master Commit: a4a0ae1ffd3e7d12e3ab4d94c7cf2af0b22c02bd Parents: 5bf52be Author: Daniel Dai <da...@hortonworks.com> Authored: Mon Apr 3 12:01:26 2017 -0700 Committer: Daniel Dai <da...@hortonworks.com> Committed: Mon Apr 3 12:01:26 2017 -0700 ---------------------------------------------------------------------- .../hcatalog/templeton/DeleteDelegator.java | 6 ++- .../hcatalog/templeton/LauncherDelegator.java | 47 +++++++++++++------- .../hive/hcatalog/templeton/ListDelegator.java | 6 ++- .../hcatalog/templeton/SecureProxySupport.java | 3 ++ .../hcatalog/templeton/StatusDelegator.java | 6 ++- .../hcatalog/templeton/tool/TempletonUtils.java | 1 + 6 files changed, 51 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a4a0ae1f/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java index 4b2dfec..622f92d 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java @@ -24,6 +24,7 @@ import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.security.UserGroupInformation; @@ -41,10 +42,11 @@ public class DeleteDelegator extends TempletonDelegator { public QueueStatusBean run(String user, String id) throws NotAuthorizedException, BadParam, IOException, InterruptedException { - UserGroupInformation ugi = UgiFactory.getUgi(user); + UserGroupInformation ugi = null; WebHCatJTShim tracker = null; JobState state = null; try { + ugi = UgiFactory.getUgi(user); tracker = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi); JobID jobid = StatusDelegator.StringToJobID(id); if (jobid == null) @@ -69,6 +71,8 @@ public class DeleteDelegator extends TempletonDelegator { tracker.close(); if (state != null) state.close(); + if (ugi != null) + FileSystem.closeAllForUGI(ugi); } } } http://git-wip-us.apache.org/repos/asf/hive/blob/a4a0ae1f/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java index 1455316..9bea897 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java @@ -28,6 +28,7 @@ import java.util.concurrent.TimeoutException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.shims.HadoopShimsSecure; @@ -178,8 +179,9 @@ public class LauncherDelegator extends TempletonDelegator { List<String> args, TempletonControllerJob controllerJob) throws NotAuthorizedException, BusyException, IOException, QueueException { + UserGroupInformation ugi = null; try { - UserGroupInformation ugi = UgiFactory.getUgi(user); + ugi = UgiFactory.getUgi(user); final long startTime = System.nanoTime(); @@ -197,6 +199,10 @@ public class LauncherDelegator extends TempletonDelegator { return new EnqueueBean(id); } catch (InterruptedException e) { throw new QueueException("Unable to launch job " + e); + } finally { + if (ugi != null) { + FileSystem.closeAllForUGI(ugi); + } } } @@ -344,24 +350,35 @@ public class LauncherDelegator extends TempletonDelegator { */ private String getShimLibjars() { WebHCatJTShim shim = null; + UserGroupInformation ugi = null; try { - shim = ShimLoader.getHadoopShims().getWebHCatShim(appConf, UserGroupInformation.getCurrentUser()); + ugi = UserGroupInformation.getCurrentUser(); + shim = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi); + + // Besides the HiveShims jar which is Hadoop version dependent we also + // always need to include hive shims common jars. + Path shimCommonJar = new Path( + TempletonUtils.findContainingJar(ShimLoader.class, HIVE_SHIMS_FILENAME_PATTERN)); + Path shimCommonSecureJar = new Path( + TempletonUtils.findContainingJar(HadoopShimsSecure.class, HIVE_SHIMS_FILENAME_PATTERN)); + Path shimJar = new Path( + TempletonUtils.findContainingJar(shim.getClass(), HIVE_SHIMS_FILENAME_PATTERN)); + + return String.format( + "%s,%s,%s", + shimCommonJar.toString(), shimCommonSecureJar.toString(), shimJar.toString()); } catch (IOException e) { - throw new RuntimeException("Failed to get WebHCatShim", e); + throw new RuntimeException("Failed to get shimLibJars", e); + } finally { + try { + if (ugi != null) { + FileSystem.closeAllForUGI(ugi); + } + } catch (IOException e) { + throw new RuntimeException("Failed to closeAllForUGI", e); + } } - // Besides the HiveShims jar which is Hadoop version dependent we also - // always need to include hive shims common jars. - Path shimCommonJar = new Path( - TempletonUtils.findContainingJar(ShimLoader.class, HIVE_SHIMS_FILENAME_PATTERN)); - Path shimCommonSecureJar = new Path( - TempletonUtils.findContainingJar(HadoopShimsSecure.class, HIVE_SHIMS_FILENAME_PATTERN)); - Path shimJar = new Path( - TempletonUtils.findContainingJar(shim.getClass(), HIVE_SHIMS_FILENAME_PATTERN)); - - return String.format( - "%s,%s,%s", - shimCommonJar.toString(), shimCommonSecureJar.toString(), shimJar.toString()); } // Storage vars http://git-wip-us.apache.org/repos/asf/hive/blob/a4a0ae1f/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java index ed4cea9..dfa59f8 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java @@ -27,6 +27,7 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.JobStatus; @@ -104,11 +105,12 @@ public class ListDelegator extends TempletonDelegator { int numRecords, boolean showDetails) throws NotAuthorizedException, BadParam, IOException, InterruptedException { - UserGroupInformation ugi = UgiFactory.getUgi(user); + UserGroupInformation ugi = null; WebHCatJTShim tracker = null; ArrayList<String> ids = new ArrayList<String>(); try { + ugi = UgiFactory.getUgi(user); tracker = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi); JobStatus[] jobs = tracker.getAllJobs(); @@ -125,6 +127,8 @@ public class ListDelegator extends TempletonDelegator { } finally { if (tracker != null) tracker.close(); + if (ugi != null) + FileSystem.closeAllForUGI(ugi); } return getJobStatus(ids, user, showall, jobId, numRecords, showDetails); http://git-wip-us.apache.org/repos/asf/hive/blob/a4a0ae1f/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java index 5aed3b3..793881b 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java @@ -170,6 +170,7 @@ public class SecureProxySupport { return null; } }); + FileSystem.closeAllForUGI(ugi); return twrapper.tokens; } private static void collectTokens(FileSystem fs, TokenWrapper twrapper, Credentials creds, String userName) throws IOException { @@ -204,6 +205,7 @@ public class SecureProxySupport { return null; } }); + FileSystem.closeAllForUGI(ugi); } @@ -220,6 +222,7 @@ public class SecureProxySupport { return client.getDelegationToken(c.getUser(), u); } }); + FileSystem.closeAllForUGI(ugi); return s; } } http://git-wip-us.apache.org/repos/asf/hive/blob/a4a0ae1f/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java index 4112eef..c042ae8 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java @@ -25,6 +25,7 @@ import java.util.concurrent.TimeoutException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.mapred.JobID; @@ -112,8 +113,9 @@ public class StatusDelegator extends TempletonDelegator { { WebHCatJTShim tracker = null; JobState state = null; + UserGroupInformation ugi = null; try { - UserGroupInformation ugi = UgiFactory.getUgi(user); + ugi = UgiFactory.getUgi(user); tracker = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi); JobID jobid = StatusDelegator.StringToJobID(id); if (jobid == null) @@ -127,6 +129,8 @@ public class StatusDelegator extends TempletonDelegator { tracker.close(); if (state != null) state.close(); + if (ugi != null) + FileSystem.closeAllForUGI(ugi); } } http://git-wip-us.apache.org/repos/asf/hive/blob/a4a0ae1f/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java index 07b005b..e0ccc70 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java @@ -362,6 +362,7 @@ public class TempletonUtils { if (hadoopFsIsMissing(defaultFs, p)) throw new FileNotFoundException("File " + fname + " does not exist."); + FileSystem.closeAllForUGI(ugi); return p; }
