Repository: hive Updated Branches: refs/heads/master 2ca70b919 -> cc8ac97bc
HIVE-19860 : HiveServer2 ObjectInspectorFactory memory leak with cachedUnionStructObjectInspector (Rajkumar Singh) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cc8ac97b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cc8ac97b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cc8ac97b Branch: refs/heads/master Commit: cc8ac97bcadd20a645e113f3193fc6b2d9db087d Parents: 2ca70b9 Author: sergey <ser...@apache.org> Authored: Tue Jul 3 12:08:54 2018 -0700 Committer: sergey <ser...@apache.org> Committed: Tue Jul 3 12:08:54 2018 -0700 ---------------------------------------------------------------------- .../objectinspector/ObjectInspectorFactory.java | 24 +++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/cc8ac97b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java index c0f9d08..b45a9de 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.serde2.objectinspector; + + import java.lang.reflect.GenericArrayType; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; @@ -29,11 +31,13 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import com.google.common.cache.CacheBuilder; import org.apache.hadoop.hive.common.StringInternUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.thrift.TUnion; - +import java.util.concurrent.TimeUnit; +import com.google.common.cache.Cache; /** * ObjectInspectorFactory is the primary way to create new ObjectInspector * instances. @@ -47,7 +51,6 @@ import org.apache.thrift.TUnion; * ObjectInspector. */ public final class ObjectInspectorFactory { - /** * ObjectInspectorOptions describes what ObjectInspector to use. JAVA is to * use pure JAVA reflection. THRIFT is to use JAVA reflection and filter out @@ -342,20 +345,19 @@ public final class ObjectInspectorFactory { return new StandardConstantStructObjectInspector(structFieldNames, structFieldObjectInspectors, value); } - static ConcurrentHashMap<List<StructObjectInspector>, UnionStructObjectInspector> cachedUnionStructObjectInspector = - new ConcurrentHashMap<List<StructObjectInspector>, UnionStructObjectInspector>(); + static Cache<List<StructObjectInspector>, UnionStructObjectInspector> cachedUnionStructObjectInspector = CacheBuilder.newBuilder() + .initialCapacity(1024) + .concurrencyLevel(Runtime.getRuntime().availableProcessors()) + .expireAfterAccess(5,TimeUnit.MINUTES) + .softValues() + .build(); public static UnionStructObjectInspector getUnionStructObjectInspector( List<StructObjectInspector> structObjectInspectors) { - UnionStructObjectInspector result = cachedUnionStructObjectInspector - .get(structObjectInspectors); + UnionStructObjectInspector result = cachedUnionStructObjectInspector.getIfPresent(structObjectInspectors); if (result == null) { result = new UnionStructObjectInspector(structObjectInspectors); - UnionStructObjectInspector prev = - cachedUnionStructObjectInspector.putIfAbsent(structObjectInspectors, result); - if (prev != null) { - result = prev; - } + cachedUnionStructObjectInspector.put(structObjectInspectors, result); } return result; }
