HIVE-19446: QueryCache: Transaction lists needed for pending cache entries (Jason Dere, reviewed by GopalV)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d5d89820 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d5d89820 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d5d89820 Branch: refs/heads/branch-3 Commit: d5d8982097e98e69f08e5398e6d6cd6c5d216867 Parents: 71be5ac Author: Jason Dere <jd...@hortonworks.com> Authored: Tue May 8 11:52:01 2018 -0700 Committer: Jason Dere <jd...@hortonworks.com> Committed: Tue May 8 11:52:52 2018 -0700 ---------------------------------------------------------------------- ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 9 ++++++--- .../hive/ql/cache/results/QueryResultsCache.java | 14 +++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/d5d89820/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 3987b5f..08f9a67 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -1978,10 +1978,14 @@ public class Driver implements IDriver { if (cacheUsage != null) { if (cacheUsage.getStatus() == CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS && plan.getFetchTask() != null) { + ValidTxnWriteIdList txnWriteIdList = null; + if (plan.hasAcidResourcesInQuery()) { + txnWriteIdList = AcidUtils.getValidTxnWriteIdList(conf); + } // The results of this query execution might be cacheable. // Add a placeholder entry in the cache so other queries know this result is pending. CacheEntry pendingCacheEntry = - QueryResultsCache.getInstance().addToCache(cacheUsage.getQueryInfo()); + QueryResultsCache.getInstance().addToCache(cacheUsage.getQueryInfo(), txnWriteIdList); if (pendingCacheEntry != null) { // Update cacheUsage to reference the pending entry. this.cacheUsage.setCacheEntry(pendingCacheEntry); @@ -2012,8 +2016,7 @@ public class Driver implements IDriver { } boolean savedToCache = QueryResultsCache.getInstance().setEntryValid( cacheUsage.getCacheEntry(), - plan.getFetchTask().getWork(), - txnWriteIdList); + plan.getFetchTask().getWork()); LOG.info("savedToCache: {}", savedToCache); if (savedToCache) { useFetchFromCache(cacheUsage.getCacheEntry()); http://git-wip-us.apache.org/repos/asf/hive/blob/d5d89820/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java index 6734ac5..11b9c27 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java @@ -488,11 +488,12 @@ public final class QueryResultsCache { * @param queryInfo * @return */ - public CacheEntry addToCache(QueryInfo queryInfo) { + public CacheEntry addToCache(QueryInfo queryInfo, ValidTxnWriteIdList txnWriteIdList) { // Create placeholder entry with PENDING state. String queryText = queryInfo.getLookupInfo().getQueryText(); CacheEntry addedEntry = new CacheEntry(); addedEntry.queryInfo = queryInfo; + addedEntry.txnWriteIdList = txnWriteIdList; Lock writeLock = rwLock.writeLock(); try { @@ -522,7 +523,7 @@ public final class QueryResultsCache { * @param fetchWork * @return */ - public boolean setEntryValid(CacheEntry cacheEntry, FetchWork fetchWork, ValidTxnWriteIdList txnWriteIdList) { + public boolean setEntryValid(CacheEntry cacheEntry, FetchWork fetchWork) { String queryText = cacheEntry.getQueryText(); boolean dataDirMoved = false; Path queryResultsPath = null; @@ -571,7 +572,6 @@ public final class QueryResultsCache { cacheEntry.cachedResultsPath = cachedResultsPath; cacheEntry.size = resultSize; this.cacheSize += resultSize; - cacheEntry.txnWriteIdList = txnWriteIdList; cacheEntry.setStatus(CacheEntryStatus.VALID); // Mark this entry as being in use. Caller will need to release later. @@ -700,6 +700,14 @@ public final class QueryResultsCache { boolean writeIdCheckPassed = false; String tableName = tableUsed.getFullyQualifiedName(); ValidTxnWriteIdList currentTxnWriteIdList = lookupInfo.txnWriteIdListProvider.get(); + if (currentTxnWriteIdList == null) { + LOG.warn("Current query's txnWriteIdList is null!"); + return false; + } + if (entry.txnWriteIdList == null) { + LOG.warn("Cache entry's txnWriteIdList is null!"); + return false; + } ValidWriteIdList currentWriteIdForTable = currentTxnWriteIdList.getTableValidWriteIdList(tableName); ValidWriteIdList cachedWriteIdForTable = entry.txnWriteIdList.getTableValidWriteIdList(tableName);
