HIVE-10453 : HS2 leaking open file descriptors when using UDFs (Yongzhi Chen via Szehon)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/42f88ca9 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/42f88ca9 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/42f88ca9 Branch: refs/heads/beeline-cli Commit: 42f88ca9a105956a2085b75dae681d62dd784cef Parents: 6176333 Author: Szehon Ho <sze...@cloudera.com> Authored: Thu May 7 11:20:53 2015 -0700 Committer: Szehon Ho <sze...@cloudera.com> Committed: Thu May 7 11:20:53 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/exec/Registry.java | 29 ++++++++++++++++++-- .../apache/hadoop/hive/ql/exec/Utilities.java | 5 ++++ .../hadoop/hive/ql/session/SessionState.java | 4 +-- 3 files changed, 34 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42f88ca9/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java index 2ba91d0..a5d59ae 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java @@ -20,8 +20,10 @@ package org.apache.hadoop.hive.ql.exec; import com.google.common.base.Splitter; import com.google.common.collect.Sets; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.ql.exec.FunctionInfo.FunctionResource; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -43,9 +45,12 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.util.ReflectionUtils; +import java.io.IOException; +import java.net.URLClassLoader; import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -66,6 +71,7 @@ public class Registry { */ private final Map<String, FunctionInfo> mFunctions = new LinkedHashMap<String, FunctionInfo>(); private final Set<Class<?>> builtIns = Collections.synchronizedSet(new HashSet<Class<?>>()); + private final Set<ClassLoader> mSessionUDFLoaders = new LinkedHashSet<ClassLoader>(); private final boolean isNative; @@ -443,7 +449,6 @@ public class Registry { // should be called after session registry is checked private FunctionInfo registerToSessionRegistry(String qualifiedName, FunctionInfo function) { FunctionInfo ret = null; - ClassLoader prev = Utilities.getSessionSpecifiedClassLoader(); try { // Found UDF in metastore - now add it to the function registry @@ -455,7 +460,6 @@ public class Registry { LOG.error("Unable to load resources for " + qualifiedName + ":" + e, e); return null; } - ClassLoader loader = Utilities.getSessionSpecifiedClassLoader(); Class<?> udfClass = Class.forName(function.getClassName(), true, loader); @@ -463,6 +467,9 @@ public class Registry { if (ret == null) { LOG.error(function.getClassName() + " is not a valid UDF class and was not registered."); } + if (SessionState.get().isHiveServerQuery()) { + SessionState.getRegistryForWrite().addToUDFLoaders(loader); + } } catch (ClassNotFoundException e) { // Lookup of UDf class failed LOG.error("Unable to load UDF class: " + e); @@ -489,6 +496,24 @@ public class Registry { builtIns.clear(); } + public synchronized void closeCUDFLoaders() { + try { + for(ClassLoader loader: mSessionUDFLoaders) { + JavaUtils.closeClassLoader(loader); + } + } catch (IOException ie) { + LOG.error("Error in close loader: " + ie); + } + mSessionUDFLoaders.clear(); + } + + public synchronized void addToUDFLoaders(ClassLoader loader) { + mSessionUDFLoaders.add(loader); + } + public synchronized void removeFromUDFLoaders(ClassLoader loader) { + mSessionUDFLoaders.remove(loader); + } + /** * Setup blocked flag for all builtin UDFs as per udf whitelist and blacklist * @param whiteListStr http://git-wip-us.apache.org/repos/asf/hive/blob/42f88ca9/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index ad5c8f8..7b48b8b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -2250,6 +2250,11 @@ public final class Utilities { } } JavaUtils.closeClassLoader(loader); +//this loader is closed, remove it from cached registry loaders to avoid remove it again. + Registry reg = SessionState.getRegistry(); + if(reg != null) { + reg.removeFromUDFLoaders(loader); + } loader = new URLClassLoader(newPath.toArray(new URL[0])); curThread.setContextClassLoader(loader); http://git-wip-us.apache.org/repos/asf/hive/blob/42f88ca9/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 8db78e5..37b6d6f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -1458,7 +1458,7 @@ public class SessionState { } public void close() throws IOException { - registry.clear();; + registry.clear(); if (txnMgr != null) txnMgr.closeTxnManager(); JavaUtils.closeClassLoadersTo(conf.getClassLoader(), parentLoader); File resourceDir = @@ -1493,7 +1493,7 @@ public class SessionState { sparkSession = null; } } - + registry.closeCUDFLoaders(); dropSessionPaths(conf); }
