Repository: hive Updated Branches: refs/heads/hive-14535 21c209e32 -> 52e0f8f34
HIVE-16746: Reduce number of index lookups for same table in IndexWhereTaskDispatcher (Rajesh Balamohan, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3fe65a3c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3fe65a3c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3fe65a3c Branch: refs/heads/hive-14535 Commit: 3fe65a3c5b0f3bb60da0a866b9c00ec7a6448055 Parents: a74c1e7 Author: Rajesh Balamohan <rbalamo...@apache.org> Authored: Thu May 25 04:03:54 2017 +0530 Committer: Rajesh Balamohan <rbalamo...@apache.org> Committed: Thu May 25 04:03:54 2017 +0530 ---------------------------------------------------------------------- .../index/IndexWhereTaskDispatcher.java | 34 ++++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/3fe65a3c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java index ae96def..7e3fb1a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java @@ -20,14 +20,16 @@ package org.apache.hadoop.hive.ql.optimizer.physical.index; import java.io.Serializable; import java.util.ArrayList; -import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Stack; +import com.google.common.collect.Maps; import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.metastore.cache.CacheUtils; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; @@ -49,6 +51,7 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hive.common.util.HiveStringUtils; /** * @@ -60,10 +63,17 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; public class IndexWhereTaskDispatcher implements Dispatcher { private final PhysicalContext physicalContext; + // To store table to index mapping + private final Map<String, List<Index>> indexMap; + private final List<String> supportedIndexes; public IndexWhereTaskDispatcher(PhysicalContext context) { super(); physicalContext = context; + indexMap = Maps.newHashMap(); + supportedIndexes = new ArrayList<String>(); + supportedIndexes.add(CompactIndexHandler.class.getName()); + supportedIndexes.add(BitmapIndexHandler.class.getName()); } @Override @@ -104,6 +114,21 @@ public class IndexWhereTaskDispatcher implements Dispatcher { return null; } + private List<Index> getIndex(Table table) throws SemanticException { + String indexCacheKey = CacheUtils.buildKey( + HiveStringUtils.normalizeIdentifier(table.getDbName()), + HiveStringUtils.normalizeIdentifier(table.getTableName())); + List<Index>indexList = indexMap.get(indexCacheKey); + if (indexList == null) { + indexList = IndexUtils.getIndexes(table, supportedIndexes); + if (indexList == null) { + indexList = Collections.emptyList(); + } + indexMap.put(indexCacheKey, indexList); + } + return indexList; + } + /** * Create a set of rules that only matches WHERE predicates on columns we have * an index on. @@ -112,16 +137,11 @@ public class IndexWhereTaskDispatcher implements Dispatcher { private Map<Rule, NodeProcessor> createOperatorRules(ParseContext pctx) throws SemanticException { Map<Rule, NodeProcessor> operatorRules = new LinkedHashMap<Rule, NodeProcessor>(); - List<String> supportedIndexes = new ArrayList<String>(); - supportedIndexes.add(CompactIndexHandler.class.getName()); - supportedIndexes.add(BitmapIndexHandler.class.getName()); - // query the metastore to know what columns we have indexed Map<TableScanOperator, List<Index>> indexes = new HashMap<TableScanOperator, List<Index>>(); for (Operator<? extends OperatorDesc> op : pctx.getTopOps().values()) { if (op instanceof TableScanOperator) { - List<Index> tblIndexes = IndexUtils.getIndexes(((TableScanOperator) op).getConf() - .getTableMetadata(), supportedIndexes); + List<Index> tblIndexes = getIndex(((TableScanOperator) op).getConf().getTableMetadata()); if (tblIndexes.size() > 0) { indexes.put((TableScanOperator) op, tblIndexes); }
