[06/50] [abbrv] hive git commit: HIVE-19446: QueryCache: Transaction lists needed for pending cache entries (Jason Dere, reviewed by GopalV)
HIVE-19446: QueryCache: Transaction lists needed for pending cache entries
(Jason Dere, reviewed by GopalV)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d5d89820
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d5d89820
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d5d89820
Branch: refs/heads/branch-3
Commit: d5d8982097e98e69f08e5398e6d6cd6c5d216867
Parents: 71be5ac
Author: Jason Dere
Authored: Tue May 8 11:52:01 2018 -0700
Committer: Jason Dere
Committed: Tue May 8 11:52:52 2018 -0700
--
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 9 ++---
.../hive/ql/cache/results/QueryResultsCache.java | 14 +++---
2 files changed, 17 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/d5d89820/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 3987b5f..08f9a67 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -1978,10 +1978,14 @@ public class Driver implements IDriver {
if (cacheUsage != null) {
if (cacheUsage.getStatus() ==
CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS &&
plan.getFetchTask() != null) {
+ValidTxnWriteIdList txnWriteIdList = null;
+if (plan.hasAcidResourcesInQuery()) {
+ txnWriteIdList = AcidUtils.getValidTxnWriteIdList(conf);
+}
// The results of this query execution might be cacheable.
// Add a placeholder entry in the cache so other queries know this
result is pending.
CacheEntry pendingCacheEntry =
-
QueryResultsCache.getInstance().addToCache(cacheUsage.getQueryInfo());
+
QueryResultsCache.getInstance().addToCache(cacheUsage.getQueryInfo(),
txnWriteIdList);
if (pendingCacheEntry != null) {
// Update cacheUsage to reference the pending entry.
this.cacheUsage.setCacheEntry(pendingCacheEntry);
@@ -2012,8 +2016,7 @@ public class Driver implements IDriver {
}
boolean savedToCache = QueryResultsCache.getInstance().setEntryValid(
cacheUsage.getCacheEntry(),
-plan.getFetchTask().getWork(),
-txnWriteIdList);
+plan.getFetchTask().getWork());
LOG.info("savedToCache: {}", savedToCache);
if (savedToCache) {
useFetchFromCache(cacheUsage.getCacheEntry());
http://git-wip-us.apache.org/repos/asf/hive/blob/d5d89820/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
index 6734ac5..11b9c27 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
@@ -488,11 +488,12 @@ public final class QueryResultsCache {
* @param queryInfo
* @return
*/
- public CacheEntry addToCache(QueryInfo queryInfo) {
+ public CacheEntry addToCache(QueryInfo queryInfo, ValidTxnWriteIdList
txnWriteIdList) {
// Create placeholder entry with PENDING state.
String queryText = queryInfo.getLookupInfo().getQueryText();
CacheEntry addedEntry = new CacheEntry();
addedEntry.queryInfo = queryInfo;
+addedEntry.txnWriteIdList = txnWriteIdList;
Lock writeLock = rwLock.writeLock();
try {
@@ -522,7 +523,7 @@ public final class QueryResultsCache {
* @param fetchWork
* @return
*/
- public boolean setEntryValid(CacheEntry cacheEntry, FetchWork fetchWork,
ValidTxnWriteIdList txnWriteIdList) {
+ public boolean setEntryValid(CacheEntry cacheEntry, FetchWork fetchWork) {
String queryText = cacheEntry.getQueryText();
boolean dataDirMoved = false;
Path queryResultsPath = null;
@@ -571,7 +572,6 @@ public final class QueryResultsCache {
cacheEntry.cachedResultsPath = cachedResultsPath;
cacheEntry.size = resultSize;
this.cacheSize += resultSize;
-cacheEntry.txnWriteIdList = txnWriteIdList;
cacheEntry.setStatus(CacheEntryStatus.VALID);
// Mark this entry as being in use. Caller will need to release later.
@@ -700,6 +700,14 @@ public final class QueryResultsCache {
boolean writeIdCheckPassed = false;
String tableName = tableUsed.getFullyQualifiedName();
ValidTxnWriteIdList currentTxnWriteIdList =
lookupInfo.txnWriteIdListProvider.get();
+ if (currentTxn
[06/50] [abbrv] hive git commit: HIVE-19446: QueryCache: Transaction lists needed for pending cache entries (Jason Dere, reviewed by GopalV)
HIVE-19446: QueryCache: Transaction lists needed for pending cache entries
(Jason Dere, reviewed by GopalV)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d5d89820
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d5d89820
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d5d89820
Branch: refs/heads/branch-3.0.0
Commit: d5d8982097e98e69f08e5398e6d6cd6c5d216867
Parents: 71be5ac
Author: Jason Dere
Authored: Tue May 8 11:52:01 2018 -0700
Committer: Jason Dere
Committed: Tue May 8 11:52:52 2018 -0700
--
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 9 ++---
.../hive/ql/cache/results/QueryResultsCache.java | 14 +++---
2 files changed, 17 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/d5d89820/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 3987b5f..08f9a67 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -1978,10 +1978,14 @@ public class Driver implements IDriver {
if (cacheUsage != null) {
if (cacheUsage.getStatus() ==
CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS &&
plan.getFetchTask() != null) {
+ValidTxnWriteIdList txnWriteIdList = null;
+if (plan.hasAcidResourcesInQuery()) {
+ txnWriteIdList = AcidUtils.getValidTxnWriteIdList(conf);
+}
// The results of this query execution might be cacheable.
// Add a placeholder entry in the cache so other queries know this
result is pending.
CacheEntry pendingCacheEntry =
-
QueryResultsCache.getInstance().addToCache(cacheUsage.getQueryInfo());
+
QueryResultsCache.getInstance().addToCache(cacheUsage.getQueryInfo(),
txnWriteIdList);
if (pendingCacheEntry != null) {
// Update cacheUsage to reference the pending entry.
this.cacheUsage.setCacheEntry(pendingCacheEntry);
@@ -2012,8 +2016,7 @@ public class Driver implements IDriver {
}
boolean savedToCache = QueryResultsCache.getInstance().setEntryValid(
cacheUsage.getCacheEntry(),
-plan.getFetchTask().getWork(),
-txnWriteIdList);
+plan.getFetchTask().getWork());
LOG.info("savedToCache: {}", savedToCache);
if (savedToCache) {
useFetchFromCache(cacheUsage.getCacheEntry());
http://git-wip-us.apache.org/repos/asf/hive/blob/d5d89820/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
index 6734ac5..11b9c27 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
@@ -488,11 +488,12 @@ public final class QueryResultsCache {
* @param queryInfo
* @return
*/
- public CacheEntry addToCache(QueryInfo queryInfo) {
+ public CacheEntry addToCache(QueryInfo queryInfo, ValidTxnWriteIdList
txnWriteIdList) {
// Create placeholder entry with PENDING state.
String queryText = queryInfo.getLookupInfo().getQueryText();
CacheEntry addedEntry = new CacheEntry();
addedEntry.queryInfo = queryInfo;
+addedEntry.txnWriteIdList = txnWriteIdList;
Lock writeLock = rwLock.writeLock();
try {
@@ -522,7 +523,7 @@ public final class QueryResultsCache {
* @param fetchWork
* @return
*/
- public boolean setEntryValid(CacheEntry cacheEntry, FetchWork fetchWork,
ValidTxnWriteIdList txnWriteIdList) {
+ public boolean setEntryValid(CacheEntry cacheEntry, FetchWork fetchWork) {
String queryText = cacheEntry.getQueryText();
boolean dataDirMoved = false;
Path queryResultsPath = null;
@@ -571,7 +572,6 @@ public final class QueryResultsCache {
cacheEntry.cachedResultsPath = cachedResultsPath;
cacheEntry.size = resultSize;
this.cacheSize += resultSize;
-cacheEntry.txnWriteIdList = txnWriteIdList;
cacheEntry.setStatus(CacheEntryStatus.VALID);
// Mark this entry as being in use. Caller will need to release later.
@@ -700,6 +700,14 @@ public final class QueryResultsCache {
boolean writeIdCheckPassed = false;
String tableName = tableUsed.getFullyQualifiedName();
ValidTxnWriteIdList currentTxnWriteIdList =
lookupInfo.txnWriteIdListProvider.get();
+ if (curren
