HIVE-20192: HS2 with embedded metastore is leaking JDOPersistenceManager objects (Sankar Hariappan, reviewed by Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6b158162 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6b158162 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6b158162 Branch: refs/heads/master-txnstats Commit: 6b158162376916433e3431f7e227771efe68b204 Parents: 170a012 Author: Sankar Hariappan <sank...@apache.org> Authored: Mon Jul 23 14:18:50 2018 +0530 Committer: Sankar Hariappan <sank...@apache.org> Committed: Mon Jul 23 14:18:50 2018 +0530 ---------------------------------------------------------------------- .../apache/hive/service/server/ThreadWithGarbageCleanup.java | 6 +++++- .../java/org/apache/hadoop/hive/metastore/HiveMetaStore.java | 8 +++++++- .../java/org/apache/hadoop/hive/metastore/ObjectStore.java | 7 ++++--- 3 files changed, 16 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6b158162/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java b/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java index 1ec8097..a8ed93e 100644 --- a/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java +++ b/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java @@ -68,7 +68,11 @@ public class ThreadWithGarbageCleanup extends Thread { public void cacheThreadLocalRawStore() { Long threadId = this.getId(); RawStore threadLocalRawStore = HiveMetaStore.HMSHandler.getRawStore(); - if (threadLocalRawStore != null && !threadRawStoreMap.containsKey(threadId)) { + if (threadLocalRawStore == null) { + LOG.debug("Thread Local RawStore is null, for the thread: " + + this.getName() + " and so removing entry from threadRawStoreMap."); + threadRawStoreMap.remove(threadId); + } else { LOG.debug("Adding RawStore: " + threadLocalRawStore + ", for the thread: " + this.getName() + " to threadRawStoreMap for future cleanup."); threadRawStoreMap.put(threadId, threadLocalRawStore); http://git-wip-us.apache.org/repos/asf/hive/blob/6b158162/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 47f819b..7fe5777 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -698,9 +698,15 @@ public class HiveMetaStore extends ThriftHiveMetastore { RawStore ms = threadLocalMS.get(); if (ms == null) { ms = newRawStoreForConf(conf); - ms.verifySchema(); + try { + ms.verifySchema(); + } catch (MetaException e) { + ms.shutdown(); + throw e; + } threadLocalMS.set(ms); ms = threadLocalMS.get(); + LOG.info("Created RawStore: " + ms + " from thread id: " + Thread.currentThread().getId()); } return ms; } http://git-wip-us.apache.org/repos/asf/hive/blob/6b158162/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index bdcbf41..6f8276f 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -425,6 +425,7 @@ public class ObjectStore implements RawStore, Configurable { initializeHelper(dsProps); return; // If we reach here, we succeed. } catch (Exception e){ + shutdown(); numTries--; boolean retriable = isRetriableException(e); if ((numTries > 0) && retriable){ @@ -486,6 +487,8 @@ public class ObjectStore implements RawStore, Configurable { LOG.debug("ObjectStore, initialize called"); prop = dsProps; pm = getPersistenceManager(); + LOG.info("RawStore: {}, with PersistenceManager: {}" + + " created in the thread with id: {}", this, pm, Thread.currentThread().getId()); try { String productName = MetaStoreDirectSql.getProductName(pm); sqlGenerator = new SQLGenerator(DatabaseProduct.determineDatabaseProduct(productName), conf); @@ -503,8 +506,6 @@ public class ObjectStore implements RawStore, Configurable { directSql = new MetaStoreDirectSql(pm, conf, schema); } } - LOG.debug("RawStore: {}, with PersistenceManager: {}" + - " created in the thread with id: {}", this, pm, Thread.currentThread().getId()); } private DatabaseProduct determineDatabaseProduct() { @@ -703,7 +704,7 @@ public class ObjectStore implements RawStore, Configurable { @Override public void shutdown() { - LOG.debug("RawStore: {}, with PersistenceManager: {} will be shutdown", this, pm); + LOG.info("RawStore: {}, with PersistenceManager: {} will be shutdown", this, pm); if (pm != null) { pm.close(); pm = null;
