This is an automated email from the ASF dual-hosted git repository. daijy pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new b58d50c HIVE-20615: CachedStore: Background refresh thread bug fixes (Vaibhav Gumashta, reviewed by Daniel Dai) b58d50c is described below commit b58d50cb73a1f79a5d079e0a2c5ac33d2efc33a0 Author: Daniel Dai <dai...@gmail.com> AuthorDate: Fri Apr 19 10:23:01 2019 -0700 HIVE-20615: CachedStore: Background refresh thread bug fixes (Vaibhav Gumashta, reviewed by Daniel Dai) --- .../hadoop/hive/metastore/cache/CacheUtils.java | 4 +++ .../hadoop/hive/metastore/cache/CachedStore.java | 4 ++- .../hadoop/hive/metastore/cache/SharedCache.java | 29 ++++++++++++++-------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java index 944c813..d50fa13 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java @@ -40,6 +40,10 @@ public class CacheUtils { return buildKey(catName.toLowerCase(), dbName.toLowerCase()); } + public static String buildDbKeyWithDelimiterSuffix(String catName, String dbName) { + return buildKey(catName.toLowerCase(), dbName.toLowerCase()) + delimit; + } + /** * Builds a key for the partition cache which is concatenation of partition values, each value * separated by a delimiter diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index e366ebd..6ef9a19 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -724,6 +724,7 @@ public class CachedStore implements RawStore, Configurable { } else { try { triggerPreWarm(rawStore); + shouldRunPrewarm = false; } catch (Exception e) { LOG.error("Prewarm failure", e); return; @@ -815,7 +816,6 @@ public class CachedStore implements RawStore, Configurable { if (table != null && !table.isSetPartitionKeys()) { List<String> colNames = MetaStoreUtils.getColumnNamesForTable(table); Deadline.startTimer("getTableColumnStatistics"); - ColumnStatistics tableColStats = rawStore.getTableColumnStatistics(catName, dbName, tblName, colNames); Deadline.stopTimer(); @@ -865,7 +865,9 @@ public class CachedStore implements RawStore, Configurable { rawStore.getPartitionColumnStatistics(catName, dbName, tblName, partNames, colNames); Deadline.stopTimer(); sharedCache.refreshPartitionColStatsInCache(catName, dbName, tblName, partitionColStats); + Deadline.startTimer("getPartitionsByNames"); List<Partition> parts = rawStore.getPartitionsByNames(catName, dbName, tblName, partNames); + Deadline.stopTimer(); // Also save partitions for consistency as they have the stats state. for (Partition part : parts) { sharedCache.alterPartitionInCache(catName, dbName, tblName, part.getValues(), part); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java index 1c23022..60862d4 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java @@ -1317,11 +1317,13 @@ public class SharedCache { //in case of retry, ignore second try. return; } - byte[] sdHash = tblWrapper.getSdHash(); - if (sdHash != null) { - decrSd(sdHash); + if (tblWrapper != null) { + byte[] sdHash = tblWrapper.getSdHash(); + if (sdHash != null) { + decrSd(sdHash); + } + isTableCacheDirty.set(true); } - isTableCacheDirty.set(true); } finally { cacheLock.writeLock().unlock(); } @@ -1438,25 +1440,30 @@ public class SharedCache { public void refreshTablesInCache(String catName, String dbName, List<Table> tables) { try { - cacheLock.writeLock().lock(); if (isTableCacheDirty.compareAndSet(true, false)) { LOG.debug("Skipping table cache update; the table list we have is dirty."); return; } - Map<String, TableWrapper> newTableCache = new HashMap<>(); + Map<String, TableWrapper> newCacheForDB = new TreeMap<>(); for (Table tbl : tables) { String tblName = StringUtils.normalizeIdentifier(tbl.getTableName()); - TableWrapper tblWrapper = - tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.updateTableObj(tbl, this); } else { tblWrapper = createTableWrapper(catName, dbName, tblName, tbl); } - newTableCache.put(CacheUtils.buildTableKey(catName, dbName, tblName), tblWrapper); + newCacheForDB.put(CacheUtils.buildTableKey(catName, dbName, tblName), tblWrapper); + } + cacheLock.writeLock().lock(); + Iterator<Entry<String, TableWrapper>> entryIterator = tableCache.entrySet().iterator(); + while (entryIterator.hasNext()) { + String key = entryIterator.next().getKey(); + if (key.startsWith(CacheUtils.buildDbKeyWithDelimiterSuffix(catName, dbName))) { + entryIterator.remove(); + } } - tableCache.clear(); - tableCache = newTableCache; + tableCache.putAll(newCacheForDB); } finally { cacheLock.writeLock().unlock(); }