http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java deleted file mode 100644 index 658422c..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.optimizer.index; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; - -import org.apache.hadoop.hive.ql.optimizer.FieldNode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.exec.ColumnInfo; -import org.apache.hadoop.hive.ql.exec.FunctionRegistry; -import org.apache.hadoop.hive.ql.exec.GroupByOperator; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.OperatorUtils; -import org.apache.hadoop.hive.ql.exec.RowSchema; -import org.apache.hadoop.hive.ql.exec.SelectOperator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.metadata.Hive; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.optimizer.ColumnPrunerProcFactory; -import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AggregationDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.GroupByDesc; -import org.apache.hadoop.hive.ql.plan.TableScanDesc; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; - -/** - * RewriteQueryUsingAggregateIndexCtx class stores the - * context for the {@link RewriteQueryUsingAggregateIndex} - * used to rewrite operator plan with index table instead of base table. - */ - -public final class RewriteQueryUsingAggregateIndexCtx implements NodeProcessorCtx { - private static final Logger LOG = LoggerFactory.getLogger(RewriteQueryUsingAggregateIndexCtx.class.getName()); - private RewriteQueryUsingAggregateIndexCtx(ParseContext parseContext, Hive hiveDb, - RewriteCanApplyCtx canApplyCtx) { - this.parseContext = parseContext; - this.hiveDb = hiveDb; - this.canApplyCtx = canApplyCtx; - this.indexTableName = canApplyCtx.getIndexTableName(); - this.alias = canApplyCtx.getAlias(); - this.aggregateFunction = canApplyCtx.getAggFunction(); - this.indexKey = canApplyCtx.getIndexKey(); - } - - public static RewriteQueryUsingAggregateIndexCtx getInstance(ParseContext parseContext, - Hive hiveDb, RewriteCanApplyCtx canApplyCtx) { - return new RewriteQueryUsingAggregateIndexCtx( - parseContext, hiveDb, canApplyCtx); - } - - // Assumes one instance of this + single-threaded compilation for each query. - private final Hive hiveDb; - private final ParseContext parseContext; - private final RewriteCanApplyCtx canApplyCtx; - //We need the GenericUDAFEvaluator for GenericUDAF function "sum" - private GenericUDAFEvaluator eval = null; - private final String indexTableName; - private final String alias; - private final String aggregateFunction; - private ExprNodeColumnDesc aggrExprNode = null; - private String indexKey; - - public ParseContext getParseContext() { - return parseContext; - } - - public Hive getHiveDb() { - return hiveDb; - } - - public String getIndexName() { - return indexTableName; - } - - public GenericUDAFEvaluator getEval() { - return eval; - } - - public void setEval(GenericUDAFEvaluator eval) { - this.eval = eval; - } - - public void setAggrExprNode(ExprNodeColumnDesc aggrExprNode) { - this.aggrExprNode = aggrExprNode; - } - - public ExprNodeColumnDesc getAggrExprNode() { - return aggrExprNode; - } - - public String getAlias() { - return alias; - } - - public String getAggregateFunction() { - return aggregateFunction; - } - - public String getIndexKey() { - return indexKey; - } - - public void setIndexKey(String indexKey) { - this.indexKey = indexKey; - } - - public void invokeRewriteQueryProc() throws SemanticException { - this.replaceTableScanProcess(canApplyCtx.getTableScanOperator()); - //We need aggrExprNode. Thus, replaceGroupByOperatorProcess should come before replaceSelectOperatorProcess - for (int index = 0; index < canApplyCtx.getGroupByOperators().size(); index++) { - this.replaceGroupByOperatorProcess(canApplyCtx.getGroupByOperators().get(index), index); - } - for (SelectOperator selectperator : canApplyCtx.getSelectOperators()) { - this.replaceSelectOperatorProcess(selectperator); - } - } - - /** - * This method replaces the original TableScanOperator with the new - * TableScanOperator and metadata that scans over the index table rather than - * scanning over the original table. - * - */ - private void replaceTableScanProcess(TableScanOperator scanOperator) throws SemanticException { - RewriteQueryUsingAggregateIndexCtx rewriteQueryCtx = this; - String alias = rewriteQueryCtx.getAlias(); - - // Need to remove the original TableScanOperators from these data structures - // and add new ones - HashMap<String, TableScanOperator> topOps = rewriteQueryCtx.getParseContext() - .getTopOps(); - - // remove original TableScanOperator - topOps.remove(alias); - - String indexTableName = rewriteQueryCtx.getIndexName(); - Table indexTableHandle = null; - try { - indexTableHandle = rewriteQueryCtx.getHiveDb().getTable(indexTableName); - } catch (HiveException e) { - LOG.error("Error while getting the table handle for index table."); - LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); - throw new SemanticException(e.getMessage(), e); - } - - // construct a new descriptor for the index table scan - TableScanDesc indexTableScanDesc = new TableScanDesc(indexTableHandle); - indexTableScanDesc.setGatherStats(false); - - String k = org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(indexTableName) + Path.SEPARATOR; - indexTableScanDesc.setStatsAggPrefix(k); - scanOperator.setConf(indexTableScanDesc); - - // Construct the new RowResolver for the new TableScanOperator - ArrayList<ColumnInfo> sigRS = new ArrayList<ColumnInfo>(); - try { - StructObjectInspector rowObjectInspector = (StructObjectInspector) indexTableHandle - .getDeserializer().getObjectInspector(); - StructField field = rowObjectInspector.getStructFieldRef(rewriteQueryCtx.getIndexKey()); - sigRS.add(new ColumnInfo(field.getFieldName(), TypeInfoUtils.getTypeInfoFromObjectInspector( - field.getFieldObjectInspector()), indexTableName, false)); - } catch (SerDeException e) { - LOG.error("Error while creating the RowResolver for new TableScanOperator."); - LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); - throw new SemanticException(e.getMessage(), e); - } - RowSchema rs = new RowSchema(sigRS); - - // Set row resolver for new table - String newAlias = indexTableName; - int index = alias.lastIndexOf(":"); - if (index >= 0) { - newAlias = alias.substring(0, index) + ":" + indexTableName; - } - - // Scan operator now points to other table - scanOperator.getConf().setAlias(newAlias); - scanOperator.setAlias(indexTableName); - topOps.put(newAlias, scanOperator); - rewriteQueryCtx.getParseContext().setTopOps(topOps); - - ColumnPrunerProcFactory.setupNeededColumns(scanOperator, rs, - Arrays.asList(new FieldNode(rewriteQueryCtx.getIndexKey()))); - } - - /** - * This method replaces the original SelectOperator with the new - * SelectOperator with a new column indexed_key_column. - */ - private void replaceSelectOperatorProcess(SelectOperator operator) throws SemanticException { - RewriteQueryUsingAggregateIndexCtx rewriteQueryCtx = this; - // we need to set the colList, outputColumnNames, colExprMap, - // rowSchema for only that SelectOperator which precedes the GroupByOperator - // count(indexed_key_column) needs to be replaced by - // sum(`_count_of_indexed_key_column`) - List<ExprNodeDesc> selColList = operator.getConf().getColList(); - selColList.add(rewriteQueryCtx.getAggrExprNode()); - - List<String> selOutputColNames = operator.getConf().getOutputColumnNames(); - selOutputColNames.add(rewriteQueryCtx.getAggrExprNode().getColumn()); - - operator.getColumnExprMap().put(rewriteQueryCtx.getAggrExprNode().getColumn(), - rewriteQueryCtx.getAggrExprNode()); - - RowSchema selRS = operator.getSchema(); - List<ColumnInfo> selRSSignature = selRS.getSignature(); - // Need to create a new type for Column[_count_of_indexed_key_column] node - PrimitiveTypeInfo pti = TypeInfoFactory.getPrimitiveTypeInfo("bigint"); - pti.setTypeName("bigint"); - ColumnInfo newCI = new ColumnInfo(rewriteQueryCtx.getAggregateFunction(), pti, "", false); - selRSSignature.add(newCI); - selRS.setSignature((ArrayList<ColumnInfo>) selRSSignature); - operator.setSchema(selRS); - } - - /** - * We need to replace the count(indexed_column_key) GenericUDAF aggregation - * function for group-by construct to "sum" GenericUDAF. This method creates a - * new operator tree for a sample query that creates a GroupByOperator with - * sum aggregation function and uses that GroupByOperator information to - * replace the original GroupByOperator aggregation information. It replaces - * the AggregationDesc (aggregation descriptor) of the old GroupByOperator - * with the new Aggregation Desc of the new GroupByOperator. - * @return - */ - private void replaceGroupByOperatorProcess(GroupByOperator operator, int index) - throws SemanticException { - RewriteQueryUsingAggregateIndexCtx rewriteQueryCtx = this; - - // We need to replace the GroupByOperator which is before RS - if (index == 0) { - // the query contains the sum aggregation GenericUDAF - String selReplacementCommand = "select sum(`" + rewriteQueryCtx.getAggregateFunction() + "`)" - + " from `" + rewriteQueryCtx.getIndexName() + "` group by " - + rewriteQueryCtx.getIndexKey() + " "; - // retrieve the operator tree for the query, and the required GroupByOperator from it - Operator<?> newOperatorTree = RewriteParseContextGenerator.generateOperatorTree( - rewriteQueryCtx.getParseContext().getQueryState(), - selReplacementCommand); - - // we get our new GroupByOperator here - GroupByOperator newGbyOperator = OperatorUtils.findLastOperatorUpstream( - newOperatorTree, GroupByOperator.class); - if (newGbyOperator == null) { - throw new SemanticException("Error replacing GroupBy operator."); - } - - // we need this information to set the correct colList, outputColumnNames - // in SelectOperator - ExprNodeColumnDesc aggrExprNode = null; - - // Construct the new AggregationDesc to get rid of the current - // internal names and replace them with new internal names - // as required by the operator tree - GroupByDesc newConf = newGbyOperator.getConf(); - List<AggregationDesc> newAggrList = newConf.getAggregators(); - if (newAggrList != null && newAggrList.size() > 0) { - for (AggregationDesc aggregationDesc : newAggrList) { - rewriteQueryCtx.setEval(aggregationDesc.getGenericUDAFEvaluator()); - aggrExprNode = (ExprNodeColumnDesc) aggregationDesc.getParameters().get(0); - rewriteQueryCtx.setAggrExprNode(aggrExprNode); - } - } - - // Now the GroupByOperator has the new AggregationList; - // sum(`_count_of_indexed_key`) - // instead of count(indexed_key) - GroupByDesc oldConf = operator.getConf(); - oldConf.setAggregators((ArrayList<AggregationDesc>) newAggrList); - operator.setConf(oldConf); - - } else { - // we just need to reset the GenericUDAFEvaluator and its name for this - // GroupByOperator whose parent is the ReduceSinkOperator - GroupByDesc childConf = operator.getConf(); - List<AggregationDesc> childAggrList = childConf.getAggregators(); - if (childAggrList != null && childAggrList.size() > 0) { - for (AggregationDesc aggregationDesc : childAggrList) { - List<ExprNodeDesc> paraList = aggregationDesc.getParameters(); - List<ObjectInspector> parametersOIList = new ArrayList<ObjectInspector>(); - for (ExprNodeDesc expr : paraList) { - parametersOIList.add(expr.getWritableObjectInspector()); - } - GenericUDAFEvaluator evaluator = FunctionRegistry.getGenericUDAFEvaluator("sum", - parametersOIList, false, false); - aggregationDesc.setGenericUDAFEvaluator(evaluator); - aggregationDesc.setGenericUDAFName("sum"); - } - } - } - } -}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java deleted file mode 100644 index d204fe8..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.optimizer.physical; - -import java.util.ArrayList; - -import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; -import org.apache.hadoop.hive.ql.lib.Dispatcher; -import org.apache.hadoop.hive.ql.lib.GraphWalker; -import org.apache.hadoop.hive.ql.lib.Node; -import org.apache.hadoop.hive.ql.optimizer.physical.index.IndexWhereTaskDispatcher; -import org.apache.hadoop.hive.ql.parse.SemanticException; - -public class IndexWhereResolver implements PhysicalPlanResolver { - - @Override - public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticException { - Dispatcher dispatcher = new IndexWhereTaskDispatcher(physicalContext); - GraphWalker opGraphWalker = new DefaultGraphWalker(dispatcher); - ArrayList<Node> topNodes = new ArrayList<Node>(); - topNodes.addAll(physicalContext.getRootTasks()); - opGraphWalker.startWalking(topNodes, null); - - return physicalContext; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java index a64a498..0f3c5f2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java @@ -60,9 +60,6 @@ public class PhysicalOptimizer { } } - if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVEOPTINDEXFILTER)) { - resolvers.add(new IndexWhereResolver()); - } resolvers.add(new MapJoinResolver()); if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVEMETADATAONLYQUERIES)) { resolvers.add(new MetadataOnlyOptimizer()); http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcCtx.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcCtx.java deleted file mode 100644 index 179d4c2..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcCtx.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.optimizer.physical.index; - -import java.io.Serializable; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.parse.ParseContext; - -public class IndexWhereProcCtx implements NodeProcessorCtx { - - private static final Logger LOG = LoggerFactory.getLogger(IndexWhereProcCtx.class.getName()); - - private final Task<? extends Serializable> currentTask; - private final ParseContext parseCtx; - - public IndexWhereProcCtx(Task<? extends Serializable> task, ParseContext parseCtx) { - this.currentTask = task; - this.parseCtx = parseCtx; - } - - public ParseContext getParseContext() { - return parseCtx; - } - - public Task<? extends Serializable> getCurrentTask() { - return currentTask; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java deleted file mode 100644 index b284afa..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.optimizer.physical.index; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Stack; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.fs.ContentSummary; -import org.apache.hadoop.hive.metastore.api.Index; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.exec.mr.MapRedTask; -import org.apache.hadoop.hive.ql.hooks.ReadEntity; -import org.apache.hadoop.hive.ql.index.HiveIndexHandler; -import org.apache.hadoop.hive.ql.index.HiveIndexQueryContext; -import org.apache.hadoop.hive.ql.lib.Node; -import org.apache.hadoop.hive.ql.lib.NodeProcessor; -import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.HiveUtils; -import org.apache.hadoop.hive.ql.metadata.Partition; -import org.apache.hadoop.hive.ql.optimizer.IndexUtils; -import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.MapWork; -import org.apache.hadoop.hive.ql.plan.MapredWork; -import org.apache.hadoop.hive.ql.plan.TableDesc; -import org.apache.hadoop.hive.ql.plan.TableScanDesc; - -/** -* -* IndexWhereProcessor. -* Processes Operator Nodes to look for WHERE queries with a predicate column -* on which we have an index. Creates an index subquery Task for these -* WHERE queries to use the index automatically. -*/ -public class IndexWhereProcessor implements NodeProcessor { - - private static final Logger LOG = LoggerFactory.getLogger(IndexWhereProcessor.class.getName()); - private final Map<TableScanOperator, List<Index>> tsToIndices; - - public IndexWhereProcessor(Map<TableScanOperator, List<Index>> tsToIndices) { - super(); - this.tsToIndices = tsToIndices; - } - - @Override - /** - * Process a node of the operator tree. This matches on the rule in IndexWhereTaskDispatcher - */ - public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - - TableScanOperator operator = (TableScanOperator) nd; - List<Node> opChildren = operator.getChildren(); - TableScanDesc operatorDesc = operator.getConf(); - if (operatorDesc == null || !tsToIndices.containsKey(operator)) { - return null; - } - List<Index> indexes = tsToIndices.get(operator); - - ExprNodeDesc predicate = operatorDesc.getFilterExpr(); - - IndexWhereProcCtx context = (IndexWhereProcCtx) procCtx; - ParseContext pctx = context.getParseContext(); - LOG.info("Processing predicate for index optimization"); - - if (predicate == null) { - LOG.info("null predicate pushed down"); - return null; - } - LOG.info(predicate.getExprString()); - - // check if we have tsToIndices on all partitions in this table scan - Set<Partition> queryPartitions; - try { - queryPartitions = IndexUtils.checkPartitionsCoveredByIndex(operator, pctx, indexes); - if (queryPartitions == null) { // partitions not covered - return null; - } - } catch (HiveException e) { - LOG.error("Fatal Error: problem accessing metastore", e); - throw new SemanticException(e); - } - - // we can only process MapReduce tasks to check input size - if (!context.getCurrentTask().isMapRedTask()) { - return null; - } - MapRedTask currentTask = (MapRedTask) context.getCurrentTask(); - - // get potential reentrant index queries from each index - Map<Index, HiveIndexQueryContext> queryContexts = new HashMap<Index, HiveIndexQueryContext>(); - // make sure we have an index on the table being scanned - TableDesc tblDesc = operator.getTableDescSkewJoin(); - - Map<String, List<Index>> indexesByType = new HashMap<String, List<Index>>(); - for (Index indexOnTable : indexes) { - if (indexesByType.get(indexOnTable.getIndexHandlerClass()) == null) { - List<Index> newType = new ArrayList<Index>(); - newType.add(indexOnTable); - indexesByType.put(indexOnTable.getIndexHandlerClass(), newType); - } else { - indexesByType.get(indexOnTable.getIndexHandlerClass()).add(indexOnTable); - } - } - - // choose index type with most tsToIndices of the same type on the table - // TODO HIVE-2130 This would be a good place for some sort of cost based choice? - List<Index> bestIndexes = indexesByType.values().iterator().next(); - for (List<Index> indexTypes : indexesByType.values()) { - if (bestIndexes.size() < indexTypes.size()) { - bestIndexes = indexTypes; - } - } - - // rewrite index queries for the chosen index type - HiveIndexQueryContext tmpQueryContext = new HiveIndexQueryContext(); - tmpQueryContext.setQueryPartitions(queryPartitions); - rewriteForIndexes(predicate, bestIndexes, pctx, currentTask, tmpQueryContext); - List<Task<?>> indexTasks = tmpQueryContext.getQueryTasks(); - - if (indexTasks != null && indexTasks.size() > 0) { - queryContexts.put(bestIndexes.get(0), tmpQueryContext); - } - // choose an index rewrite to use - if (queryContexts.size() > 0) { - // TODO HIVE-2130 This would be a good place for some sort of cost based choice? - Index chosenIndex = queryContexts.keySet().iterator().next(); - - // modify the parse context to use indexing - // we need to delay this until we choose one index so that we don't attempt to modify pctx multiple times - HiveIndexQueryContext queryContext = queryContexts.get(chosenIndex); - - // prepare the map reduce job to use indexing - MapWork work = currentTask.getWork().getMapWork(); - work.setInputformat(queryContext.getIndexInputFormat()); - work.addIndexIntermediateFile(queryContext.getIndexIntermediateFile()); - // modify inputs based on index query - Set<ReadEntity> inputs = pctx.getSemanticInputs(); - inputs.addAll(queryContext.getAdditionalSemanticInputs()); - List<Task<?>> chosenRewrite = queryContext.getQueryTasks(); - - // add dependencies so index query runs first - insertIndexQuery(pctx, context, chosenRewrite); - } - - return null; - } - - /** - * Get a list of Tasks to activate use of tsToIndices. - * Generate the tasks for the index query (where we store results of - * querying the index in a tmp file) inside the IndexHandler - * @param predicate Predicate of query to rewrite - * @param index Index to use for rewrite - * @param pctx - * @param task original task before rewrite - * @param queryContext stores return values - */ - private void rewriteForIndexes(ExprNodeDesc predicate, List<Index> indexes, - ParseContext pctx, Task<MapredWork> task, - HiveIndexQueryContext queryContext) - throws SemanticException { - HiveIndexHandler indexHandler; - // All tsToIndices in the list are of the same type, and therefore can use the - // same handler to generate the index query tasks - Index index = indexes.get(0); - try { - indexHandler = HiveUtils.getIndexHandler(pctx.getConf(), index.getIndexHandlerClass()); - } catch (HiveException e) { - LOG.error("Exception while loading IndexHandler: " + index.getIndexHandlerClass(), e); - throw new SemanticException("Failed to load indexHandler: " + index.getIndexHandlerClass(), e); - } - - // check the size - try { - ContentSummary inputSummary = Utilities.getInputSummary(pctx.getContext(), task.getWork().getMapWork(), null); - long inputSize = inputSummary.getLength(); - if (!indexHandler.checkQuerySize(inputSize, pctx.getConf())) { - queryContext.setQueryTasks(null); - return; - } - } catch (IOException e) { - throw new SemanticException("Failed to get task size", e); - } - - // use the IndexHandler to generate the index query - indexHandler.generateIndexQuery(indexes, predicate, pctx, queryContext); - // TODO HIVE-2115 use queryContext.residualPredicate to process residual predicate - - return; - } - - - /** - * Insert the rewrite tasks at the head of the pctx task tree - * @param pctx - * @param context - * @param chosenRewrite - */ - private void insertIndexQuery(ParseContext pctx, IndexWhereProcCtx context, List<Task<?>> chosenRewrite) { - Task<?> wholeTableScan = context.getCurrentTask(); - LinkedHashSet<Task<?>> rewriteLeaves = new LinkedHashSet<Task<?>>(); - findLeaves(chosenRewrite, rewriteLeaves); - - for (Task<?> leaf : rewriteLeaves) { - leaf.addDependentTask(wholeTableScan); // add full scan task as child for every index query task - } - - // replace the original with the index sub-query as a root task - pctx.replaceRootTask(wholeTableScan, chosenRewrite); - } - - /** - * Find the leaves of the task tree - */ - private void findLeaves(List<Task<?>> tasks, Set<Task<?>> leaves) { - for (Task<?> t : tasks) { - if (t.getDependentTasks() == null) { - leaves.add(t); - } else { - findLeaves(t.getDependentTasks(), leaves); - } - } - } - -} - http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java deleted file mode 100644 index c9dae8f..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.optimizer.physical.index; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Stack; - -import com.google.common.collect.Maps; -import org.apache.hadoop.hive.metastore.api.Index; -import org.apache.hadoop.hive.metastore.cache.CacheUtils; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler; -import org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler; -import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; -import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; -import org.apache.hadoop.hive.ql.lib.Dispatcher; -import org.apache.hadoop.hive.ql.lib.GraphWalker; -import org.apache.hadoop.hive.ql.lib.Node; -import org.apache.hadoop.hive.ql.lib.NodeProcessor; -import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.lib.Rule; -import org.apache.hadoop.hive.ql.lib.RuleRegExp; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.optimizer.IndexUtils; -import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext; -import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.MapredWork; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hive.common.util.HiveStringUtils; - -/** - * - * IndexWhereTaskDispatcher. Walks a Task tree, and for the right kind of Task, - * walks the operator tree to create an index subquery. Then attaches the - * subquery task to the task tree. - * - */ -public class IndexWhereTaskDispatcher implements Dispatcher { - - private final PhysicalContext physicalContext; - // To store table to index mapping - private final Map<String, List<Index>> indexMap; - private final List<String> supportedIndexes; - - public IndexWhereTaskDispatcher(PhysicalContext context) { - super(); - physicalContext = context; - indexMap = Maps.newHashMap(); - supportedIndexes = new ArrayList<String>(); - supportedIndexes.add(CompactIndexHandler.class.getName()); - supportedIndexes.add(BitmapIndexHandler.class.getName()); - } - - @Override - public Object dispatch(Node nd, Stack<Node> stack, Object... nodeOutputs) - throws SemanticException { - - Task<? extends Serializable> task = (Task<? extends Serializable>) nd; - - ParseContext pctx = physicalContext.getParseContext(); - - // create the regex's so the walker can recognize our WHERE queries - Map<Rule, NodeProcessor> operatorRules = createOperatorRules(pctx); - - // check for no indexes on any table - if (operatorRules == null) { - return null; - } - - // create context so the walker can carry the current task with it. - IndexWhereProcCtx indexWhereOptimizeCtx = new IndexWhereProcCtx(task, pctx); - - // create the dispatcher, which fires the processor according to the rule that - // best matches - Dispatcher dispatcher = new DefaultRuleDispatcher(getDefaultProcessor(), - operatorRules, - indexWhereOptimizeCtx); - - // walk the mapper operator(not task) tree for each specific task - GraphWalker ogw = new DefaultGraphWalker(dispatcher); - ArrayList<Node> topNodes = new ArrayList<Node>(); - if (task.getWork() instanceof MapredWork) { - topNodes.addAll(((MapredWork)task.getWork()).getMapWork().getAliasToWork().values()); - } else { - return null; - } - ogw.startWalking(topNodes, null); - - return null; - } - - private List<Index> getIndex(Table table) throws SemanticException { - String indexCacheKey = CacheUtils.buildKey( - HiveStringUtils.normalizeIdentifier(table.getDbName()), - HiveStringUtils.normalizeIdentifier(table.getTableName())); - List<Index>indexList = indexMap.get(indexCacheKey); - if (indexList == null) { - indexList = IndexUtils.getIndexes(table, supportedIndexes); - if (indexList == null) { - indexList = Collections.emptyList(); - } - indexMap.put(indexCacheKey, indexList); - } - return indexList; - } - - /** - * Create a set of rules that only matches WHERE predicates on columns we have - * an index on. - * @return - */ - private Map<Rule, NodeProcessor> createOperatorRules(ParseContext pctx) throws SemanticException { - Map<Rule, NodeProcessor> operatorRules = new LinkedHashMap<Rule, NodeProcessor>(); - - // query the metastore to know what columns we have indexed - Map<TableScanOperator, List<Index>> indexes = new HashMap<TableScanOperator, List<Index>>(); - for (Operator<? extends OperatorDesc> op : pctx.getTopOps().values()) { - if (op instanceof TableScanOperator) { - List<Index> tblIndexes = getIndex(((TableScanOperator) op).getConf().getTableMetadata()); - if (tblIndexes.size() > 0) { - indexes.put((TableScanOperator) op, tblIndexes); - } - } - } - - // quit if our tables don't have any indexes - if (indexes.size() == 0) { - return null; - } - - // We set the pushed predicate from the WHERE clause as the filter expr on - // all table scan operators, so we look for table scan operators(TS%) - operatorRules.put(new RuleRegExp("RULEWhere", TableScanOperator.getOperatorName() + "%"), - new IndexWhereProcessor(indexes)); - - return operatorRules; - } - - - private NodeProcessor getDefaultProcessor() { - return new NodeProcessor() { - @Override - public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - return null; - } - }; - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 4338fa6..41d878f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -43,7 +43,6 @@ import org.antlr.runtime.tree.CommonTree; import org.antlr.runtime.tree.Tree; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -52,9 +51,7 @@ import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; @@ -82,9 +79,6 @@ import org.apache.hadoop.hive.ql.hooks.Entity.Type; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity.WriteType; -import org.apache.hadoop.hive.ql.index.HiveIndex; -import org.apache.hadoop.hive.ql.index.HiveIndex.IndexType; -import org.apache.hadoop.hive.ql.index.HiveIndexHandler; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; @@ -106,8 +100,6 @@ import org.apache.hadoop.hive.ql.plan.AbortTxnsDesc; import org.apache.hadoop.hive.ql.plan.AddPartitionDesc; import org.apache.hadoop.hive.ql.plan.AddPartitionDesc.OnePartitionDesc; import org.apache.hadoop.hive.ql.plan.AlterDatabaseDesc; -import org.apache.hadoop.hive.ql.plan.AlterIndexDesc; -import org.apache.hadoop.hive.ql.plan.AlterIndexDesc.AlterIndexTypes; import org.apache.hadoop.hive.ql.plan.AlterMaterializedViewDesc; import org.apache.hadoop.hive.ql.plan.AlterMaterializedViewDesc.AlterMaterializedViewTypes; import org.apache.hadoop.hive.ql.plan.AlterResourcePlanDesc; @@ -121,7 +113,6 @@ import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.CacheMetadataDesc; import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork; import org.apache.hadoop.hive.ql.plan.CreateDatabaseDesc; -import org.apache.hadoop.hive.ql.plan.CreateIndexDesc; import org.apache.hadoop.hive.ql.plan.CreateOrAlterWMMappingDesc; import org.apache.hadoop.hive.ql.plan.CreateOrAlterWMPoolDesc; import org.apache.hadoop.hive.ql.plan.CreateOrDropTriggerToPoolMappingDesc; @@ -132,7 +123,6 @@ import org.apache.hadoop.hive.ql.plan.DescDatabaseDesc; import org.apache.hadoop.hive.ql.plan.DescFunctionDesc; import org.apache.hadoop.hive.ql.plan.DescTableDesc; import org.apache.hadoop.hive.ql.plan.DropDatabaseDesc; -import org.apache.hadoop.hive.ql.plan.DropIndexDesc; import org.apache.hadoop.hive.ql.plan.DropResourcePlanDesc; import org.apache.hadoop.hive.ql.plan.DropTableDesc; import org.apache.hadoop.hive.ql.plan.DropWMMappingDesc; @@ -162,7 +152,6 @@ import org.apache.hadoop.hive.ql.plan.ShowCreateTableDesc; import org.apache.hadoop.hive.ql.plan.ShowDatabasesDesc; import org.apache.hadoop.hive.ql.plan.ShowFunctionsDesc; import org.apache.hadoop.hive.ql.plan.ShowGrantDesc; -import org.apache.hadoop.hive.ql.plan.ShowIndexesDesc; import org.apache.hadoop.hive.ql.plan.ShowLocksDesc; import org.apache.hadoop.hive.ql.plan.ShowPartitionsDesc; import org.apache.hadoop.hive.ql.plan.ShowResourcePlanDesc; @@ -176,7 +165,6 @@ import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TruncateTableDesc; import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc; import org.apache.hadoop.hive.ql.plan.UnlockTableDesc; -import org.apache.hadoop.hive.ql.session.LineageState; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.serde.serdeConstants; @@ -373,12 +361,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { case HiveParser.TOK_TRUNCATETABLE: analyzeTruncateTable(ast); break; - case HiveParser.TOK_CREATEINDEX: - analyzeCreateIndex(ast); - break; - case HiveParser.TOK_DROPINDEX: - analyzeDropIndex(ast); - break; case HiveParser.TOK_DESCTABLE: ctx.setResFile(ctx.getLocalTmpPath()); analyzeDescribeTable(ast); @@ -485,12 +467,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { } break; } - case HiveParser.TOK_ALTERINDEX_REBUILD: - analyzeAlterIndexRebuild(ast); - break; - case HiveParser.TOK_ALTERINDEX_PROPERTIES: - analyzeAlterIndexProps(ast); - break; case HiveParser.TOK_SHOWPARTITIONS: ctx.setResFile(ctx.getLocalTmpPath()); analyzeShowPartitions(ast); @@ -503,10 +479,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { ctx.setResFile(ctx.getLocalTmpPath()); analyzeShowCreateTable(ast); break; - case HiveParser.TOK_SHOWINDEXES: - ctx.setResFile(ctx.getLocalTmpPath()); - analyzeShowIndexes(ast); - break; case HiveParser.TOK_LOCKTABLE: analyzeLockTable(ast); break; @@ -1486,11 +1458,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { try { columnNames = getColumnNames((ASTNode)ast.getChild(1)); - // Throw an error if the table is indexed - List<Index> indexes = db.getIndexes(table.getDbName(), tableName, (short)1); - if (indexes != null && indexes.size() > 0) { - throw new SemanticException(ErrorMsg.TRUNCATE_COLUMN_INDEXED_TABLE.getMsg()); - } // It would be possible to support this, but this is such a pointless command. if (AcidUtils.isInsertOnlyTable(table.getParameters())) { throw new SemanticException("Truncating MM table columns not presently supported"); @@ -1634,235 +1601,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { return true; } - private void analyzeCreateIndex(ASTNode ast) throws SemanticException { - String indexName = unescapeIdentifier(ast.getChild(0).getText()); - String typeName = unescapeSQLString(ast.getChild(1).getText()); - String[] qTabName = getQualifiedTableName((ASTNode) ast.getChild(2)); - List<String> indexedCols = getColumnNames((ASTNode) ast.getChild(3)); - - IndexType indexType = HiveIndex.getIndexType(typeName); - if (indexType != null) { - typeName = indexType.getHandlerClsName(); - } else { - try { - JavaUtils.loadClass(typeName); - } catch (Exception e) { - throw new SemanticException("class name provided for index handler not found.", e); - } - } - - String indexTableName = null; - boolean deferredRebuild = false; - String location = null; - Map<String, String> tblProps = null; - Map<String, String> idxProps = null; - String indexComment = null; - - RowFormatParams rowFormatParams = new RowFormatParams(); - StorageFormat storageFormat = new StorageFormat(conf); - - for (int idx = 4; idx < ast.getChildCount(); idx++) { - ASTNode child = (ASTNode) ast.getChild(idx); - if (storageFormat.fillStorageFormat(child)) { - continue; - } - switch (child.getToken().getType()) { - case HiveParser.TOK_TABLEROWFORMAT: - rowFormatParams.analyzeRowFormat(child); - break; - case HiveParser.TOK_CREATEINDEX_INDEXTBLNAME: - ASTNode ch = (ASTNode) child.getChild(0); - indexTableName = getUnescapedName(ch); - break; - case HiveParser.TOK_DEFERRED_REBUILDINDEX: - deferredRebuild = true; - break; - case HiveParser.TOK_TABLELOCATION: - location = unescapeSQLString(child.getChild(0).getText()); - addLocationToOutputs(location); - break; - case HiveParser.TOK_TABLEPROPERTIES: - tblProps = DDLSemanticAnalyzer.getProps((ASTNode) child.getChild(0)); - break; - case HiveParser.TOK_INDEXPROPERTIES: - idxProps = DDLSemanticAnalyzer.getProps((ASTNode) child.getChild(0)); - break; - case HiveParser.TOK_TABLESERIALIZER: - child = (ASTNode) child.getChild(0); - storageFormat.setSerde(unescapeSQLString(child.getChild(0).getText())); - if (child.getChildCount() == 2) { - readProps((ASTNode) (child.getChild(1).getChild(0)), - storageFormat.getSerdeProps()); - } - break; - case HiveParser.TOK_INDEXCOMMENT: - child = (ASTNode) child.getChild(0); - indexComment = unescapeSQLString(child.getText()); - } - } - - storageFormat.fillDefaultStorageFormat(false, false); - if (indexTableName == null) { - indexTableName = MetaStoreUtils.getIndexTableName(qTabName[0], qTabName[1], indexName); - indexTableName = qTabName[0] + "." + indexTableName; // on same database with base table - } else { - indexTableName = getDotName(Utilities.getDbTableName(indexTableName)); - } - inputs.add(new ReadEntity(getTable(qTabName))); - - CreateIndexDesc crtIndexDesc = new CreateIndexDesc(getDotName(qTabName), indexName, - indexedCols, indexTableName, deferredRebuild, storageFormat.getInputFormat(), - storageFormat.getOutputFormat(), - storageFormat.getStorageHandler(), typeName, location, idxProps, tblProps, - storageFormat.getSerde(), storageFormat.getSerdeProps(), rowFormatParams.collItemDelim, - rowFormatParams.fieldDelim, rowFormatParams.fieldEscape, - rowFormatParams.lineDelim, rowFormatParams.mapKeyDelim, indexComment); - Task<?> createIndex = - TaskFactory.get(new DDLWork(getInputs(), getOutputs(), crtIndexDesc), conf); - rootTasks.add(createIndex); - } - - private void analyzeDropIndex(ASTNode ast) throws SemanticException { - String indexName = unescapeIdentifier(ast.getChild(0).getText()); - String tableName = getUnescapedName((ASTNode) ast.getChild(1)); - boolean ifExists = (ast.getFirstChildWithType(HiveParser.TOK_IFEXISTS) != null); - // we want to signal an error if the index doesn't exist and we're - // configured not to ignore this - boolean throwException = - !ifExists && !HiveConf.getBoolVar(conf, ConfVars.DROPIGNORESNONEXISTENT); - Table tbl = getTable(tableName, false); - if (throwException && tbl == null) { - throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName)); - } - try { - Index idx = db.getIndex(tableName, indexName); - } catch (HiveException e) { - if (!(e.getCause() instanceof NoSuchObjectException)) { - throw new SemanticException(ErrorMsg.CANNOT_DROP_INDEX.getMsg("dropping index"), e); - } - if (throwException) { - throw new SemanticException(ErrorMsg.INVALID_INDEX.getMsg(indexName)); - } - } - if (tbl != null) { - inputs.add(new ReadEntity(tbl)); - } - - DropIndexDesc dropIdxDesc = new DropIndexDesc(indexName, tableName, throwException); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), - dropIdxDesc), conf)); - } - - private void analyzeAlterIndexRebuild(ASTNode ast) throws SemanticException { - String[] qualified = getQualifiedTableName((ASTNode) ast.getChild(0)); - String indexName = unescapeIdentifier(ast.getChild(1).getText()); - HashMap<String, String> partSpec = null; - Tree part = ast.getChild(2); - if (part != null) { - partSpec = getValidatedPartSpec(getTable(qualified), (ASTNode)part, conf, false); - } - List<Task<?>> indexBuilder = getIndexBuilderMapRed(qualified, indexName, partSpec); - rootTasks.addAll(indexBuilder); - - // Handle updating index timestamps - AlterIndexDesc alterIdxDesc = new AlterIndexDesc(AlterIndexTypes.UPDATETIMESTAMP); - alterIdxDesc.setIndexName(indexName); - alterIdxDesc.setBaseTableName(getDotName(qualified)); - alterIdxDesc.setSpec(partSpec); - - Task<?> tsTask = TaskFactory.get(new DDLWork(alterIdxDesc), conf); - for (Task<?> t : indexBuilder) { - t.addDependentTask(tsTask); - } - } - - private void analyzeAlterIndexProps(ASTNode ast) - throws SemanticException { - - String[] qualified = getQualifiedTableName((ASTNode) ast.getChild(0)); - String indexName = unescapeIdentifier(ast.getChild(1).getText()); - HashMap<String, String> mapProp = getProps((ASTNode) (ast.getChild(2)) - .getChild(0)); - - AlterIndexDesc alterIdxDesc = new AlterIndexDesc(AlterIndexTypes.ADDPROPS); - alterIdxDesc.setProps(mapProp); - alterIdxDesc.setIndexName(indexName); - alterIdxDesc.setBaseTableName(getDotName(qualified)); - - rootTasks.add(TaskFactory.get(new DDLWork(alterIdxDesc), conf)); - } - - private List<Task<?>> getIndexBuilderMapRed(String[] names, String indexName, - HashMap<String, String> partSpec) throws SemanticException { - try { - Index index = db.getIndex(names[0], names[1], indexName); - Table indexTbl = null; - String indexTableName = index.getIndexTableName(); - if (indexTableName != null) { - indexTbl = getTable(Utilities.getDbTableName(index.getDbName(), indexTableName)); - } - Table baseTbl = getTable(new String[] {index.getDbName(), index.getOrigTableName()}); - - String handlerCls = index.getIndexHandlerClass(); - HiveIndexHandler handler = HiveUtils.getIndexHandler(conf, handlerCls); - - List<Partition> indexTblPartitions = null; - List<Partition> baseTblPartitions = null; - if (indexTbl != null) { - indexTblPartitions = new ArrayList<Partition>(); - baseTblPartitions = preparePartitions(baseTbl, partSpec, - indexTbl, db, indexTblPartitions); - } - - LineageState lineageState = queryState.getLineageState(); - List<Task<?>> ret = handler.generateIndexBuildTaskList(baseTbl, - index, indexTblPartitions, baseTblPartitions, indexTbl, getInputs(), getOutputs(), - lineageState); - return ret; - } catch (Exception e) { - throw new SemanticException(e); - } - } - - private List<Partition> preparePartitions( - org.apache.hadoop.hive.ql.metadata.Table baseTbl, - HashMap<String, String> partSpec, - org.apache.hadoop.hive.ql.metadata.Table indexTbl, Hive db, - List<Partition> indexTblPartitions) - throws HiveException, MetaException { - List<Partition> baseTblPartitions = new ArrayList<Partition>(); - if (partSpec != null) { - // if partspec is specified, then only producing index for that - // partition - Partition part = db.getPartition(baseTbl, partSpec, false); - if (part == null) { - throw new HiveException("Partition " - + Warehouse.makePartName(partSpec, false) - + " does not exist in table " - + baseTbl.getTableName()); - } - baseTblPartitions.add(part); - Partition indexPart = db.getPartition(indexTbl, partSpec, false); - if (indexPart == null) { - indexPart = db.createPartition(indexTbl, partSpec); - } - indexTblPartitions.add(indexPart); - } else if (baseTbl.isPartitioned()) { - // if no partition is specified, create indexes for all partitions one - // by one. - baseTblPartitions = db.getPartitions(baseTbl); - for (Partition basePart : baseTblPartitions) { - HashMap<String, String> pSpec = basePart.getSpec(); - Partition indexPart = db.getPartition(indexTbl, pSpec, false); - if (indexPart == null) { - indexPart = db.createPartition(indexTbl, pSpec); - } - indexTblPartitions.add(indexPart); - } - } - return baseTblPartitions; - } - private void validateAlterTableType(Table tbl, AlterTableTypes op) throws SemanticException { validateAlterTableType(tbl, op, false); } @@ -2190,17 +1928,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { List<String> bucketCols = null; Class<? extends InputFormat> inputFormatClass = null; boolean isArchived = false; - boolean checkIndex = HiveConf.getBoolVar(conf, - HiveConf.ConfVars.HIVE_CONCATENATE_CHECK_INDEX); - if (checkIndex) { - List<Index> indexes = db.getIndexes(tblObj.getDbName(), tblObj.getTableName(), - Short.MAX_VALUE); - if (indexes != null && indexes.size() > 0) { - throw new SemanticException("can not do merge because source table " - + tableName + " is indexed."); - } - } - if (tblObj.isPartitioned()) { if (partSpec == null) { throw new SemanticException("source table " + tableName @@ -2762,6 +2489,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { setFetchTask(createFetchTask(showCreateDbDesc.getSchema())); } + private void analyzeShowCreateTable(ASTNode ast) throws SemanticException { ShowCreateTableDesc showCreateTblDesc; String tableName = getUnescapedName((ASTNode)ast.getChild(0)); @@ -2896,21 +2624,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { setFetchTask(createFetchTask(showTblPropertiesDesc.getSchema())); } - private void analyzeShowIndexes(ASTNode ast) throws SemanticException { - ShowIndexesDesc showIndexesDesc; - String tableName = getUnescapedName((ASTNode) ast.getChild(0)); - showIndexesDesc = new ShowIndexesDesc(tableName, ctx.getResFile()); - - if (ast.getChildCount() == 2) { - int descOptions = ast.getChild(1).getType(); - showIndexesDesc.setFormatted(descOptions == HiveParser.KW_FORMATTED); - } - - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), - showIndexesDesc), conf)); - setFetchTask(createFetchTask(showIndexesDesc.getSchema())); - } - /** * Add the task according to the parsed command tree. This is used for the CLI * command "SHOW FUNCTIONS;". http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 9073623..3e84fd6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -151,10 +151,6 @@ TOK_COLTYPELIST; TOK_CREATEDATABASE; TOK_CREATETABLE; TOK_TRUNCATETABLE; -TOK_CREATEINDEX; -TOK_CREATEINDEX_INDEXTBLNAME; -TOK_DEFERRED_REBUILDINDEX; -TOK_DROPINDEX; TOK_LIKETABLE; TOK_DESCTABLE; TOK_DESCFUNCTION; @@ -189,8 +185,6 @@ TOK_ALTERTABLE_CLUSTER_SORT; TOK_ALTERTABLE_COMPACT; TOK_ALTERTABLE_DROPCONSTRAINT; TOK_ALTERTABLE_ADDCONSTRAINT; -TOK_ALTERINDEX_REBUILD; -TOK_ALTERINDEX_PROPERTIES; TOK_MSCK; TOK_SHOWDATABASES; TOK_SHOWTABLES; @@ -271,8 +265,6 @@ TOK_EXPLAIN_SQ_REWRITE; TOK_TABLESERIALIZER; TOK_TABLEPROPERTIES; TOK_TABLEPROPLIST; -TOK_INDEXPROPERTIES; -TOK_INDEXPROPLIST; TOK_TABTYPE; TOK_LIMIT; TOK_OFFSET; @@ -310,7 +302,6 @@ TOK_PRIV_ALTER_METADATA; TOK_PRIV_ALTER_DATA; TOK_PRIV_DELETE; TOK_PRIV_DROP; -TOK_PRIV_INDEX; TOK_PRIV_INSERT; TOK_PRIV_LOCK; TOK_PRIV_SELECT; @@ -324,9 +315,7 @@ TOK_SHOW_ROLE_GRANT; TOK_SHOW_ROLES; TOK_SHOW_SET_ROLE; TOK_SHOW_ROLE_PRINCIPALS; -TOK_SHOWINDEXES; TOK_SHOWDBLOCKS; -TOK_INDEXCOMMENT; TOK_DESCDATABASE; TOK_DATABASEPROPERTIES; TOK_DATABASELOCATION; @@ -929,8 +918,6 @@ ddlStatement | dropMaterializedViewStatement | createFunctionStatement | createMacroStatement - | createIndexStatement - | dropIndexStatement | dropFunctionStatement | reloadFunctionStatement | dropMacroStatement @@ -1102,80 +1089,6 @@ truncateTableStatement @after { popMsg(state); } : KW_TRUNCATE KW_TABLE tablePartitionPrefix (KW_COLUMNS LPAREN columnNameList RPAREN)? -> ^(TOK_TRUNCATETABLE tablePartitionPrefix columnNameList?); -createIndexStatement -@init { pushMsg("create index statement", state);} -@after {popMsg(state);} - : KW_CREATE KW_INDEX indexName=identifier - KW_ON KW_TABLE tab=tableName LPAREN indexedCols=columnNameList RPAREN - KW_AS typeName=StringLiteral - autoRebuild? - indexPropertiesPrefixed? - indexTblName? - tableRowFormat? - tableFileFormat? - tableLocation? - tablePropertiesPrefixed? - indexComment? - ->^(TOK_CREATEINDEX $indexName $typeName $tab $indexedCols - autoRebuild? - indexPropertiesPrefixed? - indexTblName? - tableRowFormat? - tableFileFormat? - tableLocation? - tablePropertiesPrefixed? - indexComment?) - ; - -indexComment -@init { pushMsg("comment on an index", state);} -@after {popMsg(state);} - : - KW_COMMENT comment=StringLiteral -> ^(TOK_INDEXCOMMENT $comment) - ; - -autoRebuild -@init { pushMsg("auto rebuild index", state);} -@after {popMsg(state);} - : KW_WITH KW_DEFERRED KW_REBUILD - ->^(TOK_DEFERRED_REBUILDINDEX) - ; - -indexTblName -@init { pushMsg("index table name", state);} -@after {popMsg(state);} - : KW_IN KW_TABLE indexTbl=tableName - ->^(TOK_CREATEINDEX_INDEXTBLNAME $indexTbl) - ; - -indexPropertiesPrefixed -@init { pushMsg("table properties with prefix", state); } -@after { popMsg(state); } - : - KW_IDXPROPERTIES! indexProperties - ; - -indexProperties -@init { pushMsg("index properties", state); } -@after { popMsg(state); } - : - LPAREN indexPropertiesList RPAREN -> ^(TOK_INDEXPROPERTIES indexPropertiesList) - ; - -indexPropertiesList -@init { pushMsg("index properties list", state); } -@after { popMsg(state); } - : - keyValueProperty (COMMA keyValueProperty)* -> ^(TOK_INDEXPROPLIST keyValueProperty+) - ; - -dropIndexStatement -@init { pushMsg("drop index statement", state);} -@after {popMsg(state);} - : KW_DROP KW_INDEX ifExists? indexName=identifier KW_ON tab=tableName - ->^(TOK_DROPINDEX $indexName $tab ifExists?) - ; - dropTableStatement @init { pushMsg("drop statement", state); } @after { popMsg(state); } @@ -1190,7 +1103,6 @@ alterStatement | KW_ALTER KW_VIEW tableName KW_AS? alterViewStatementSuffix -> ^(TOK_ALTERVIEW tableName alterViewStatementSuffix) | KW_ALTER KW_MATERIALIZED KW_VIEW tableName alterMaterializedViewStatementSuffix -> ^(TOK_ALTER_MATERIALIZED_VIEW tableName alterMaterializedViewStatementSuffix) - | KW_ALTER KW_INDEX alterIndexStatementSuffix -> alterIndexStatementSuffix | KW_ALTER (KW_DATABASE|KW_SCHEMA) alterDatabaseStatementSuffix -> alterDatabaseStatementSuffix ; @@ -1254,20 +1166,6 @@ alterMaterializedViewStatementSuffix | alterMaterializedViewSuffixRebuild ; -alterIndexStatementSuffix -@init { pushMsg("alter index statement", state); } -@after { popMsg(state); } - : indexName=identifier KW_ON tableName partitionSpec? - ( - KW_REBUILD - ->^(TOK_ALTERINDEX_REBUILD tableName $indexName partitionSpec?) - | - KW_SET KW_IDXPROPERTIES - indexProperties - ->^(TOK_ALTERINDEX_PROPERTIES tableName $indexName indexProperties) - ) - ; - alterDatabaseStatementSuffix @init { pushMsg("alter database statement", state); } @after { popMsg(state); } @@ -1652,8 +1550,6 @@ showStatement | (parttype=partTypeExpr)? (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWLOCKS $parttype? $isExtended?) ) - | KW_SHOW (showOptions=KW_FORMATTED)? (KW_INDEX|KW_INDEXES) KW_ON showStmtIdentifier ((KW_FROM|KW_IN) db_name=identifier)? - -> ^(TOK_SHOWINDEXES showStmtIdentifier $showOptions? $db_name?) | KW_SHOW KW_COMPACTIONS -> ^(TOK_SHOW_COMPACTIONS) | KW_SHOW KW_TRANSACTIONS -> ^(TOK_SHOW_TRANSACTIONS) | KW_SHOW KW_CONF StringLiteral -> ^(TOK_SHOWCONF StringLiteral) @@ -1839,7 +1735,6 @@ privilegeType | KW_UPDATE -> ^(TOK_PRIV_ALTER_DATA) | KW_CREATE -> ^(TOK_PRIV_CREATE) | KW_DROP -> ^(TOK_PRIV_DROP) - | KW_INDEX -> ^(TOK_PRIV_INDEX) | KW_LOCK -> ^(TOK_PRIV_LOCK) | KW_SELECT -> ^(TOK_PRIV_SELECT) | KW_SHOW_DATABASE -> ^(TOK_PRIV_SHOW_DATABASE) http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java deleted file mode 100644 index 22b6697..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse; - - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.api.Index; -import org.apache.hadoop.hive.ql.Driver; -import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.hooks.ReadEntity; -import org.apache.hadoop.hive.ql.metadata.Hive; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Partition; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.optimizer.IndexUtils; -import org.apache.hadoop.hive.ql.plan.LoadTableDesc; -import org.apache.hadoop.hive.ql.plan.TableDesc; -import org.apache.hadoop.hive.ql.session.LineageState; - -import java.io.Serializable; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class IndexUpdater { - private List<LoadTableDesc> loadTableWork; - private HiveConf conf; - // Assumes one instance of this + single-threaded compilation for each query. - private Hive hive; - private List<Task<? extends Serializable>> tasks; - private Set<ReadEntity> inputs; - private LineageState lineageState; - - public IndexUpdater(List<LoadTableDesc> loadTableWork, Set<ReadEntity> inputs, Configuration conf, - LineageState lineageState) { - this.loadTableWork = loadTableWork; - this.inputs = inputs; - this.conf = new HiveConf(conf, IndexUpdater.class); - this.lineageState = lineageState; - this.tasks = new LinkedList<Task<? extends Serializable>>(); - } - - public IndexUpdater(LoadTableDesc loadTableWork, Set<ReadEntity> inputs, - Configuration conf) { - this.loadTableWork = new LinkedList<LoadTableDesc>(); - this.loadTableWork.add(loadTableWork); - this.conf = new HiveConf(conf, IndexUpdater.class); - this.tasks = new LinkedList<Task<? extends Serializable>>(); - this.inputs = inputs; - } - - public List<Task<? extends Serializable>> generateUpdateTasks() throws - HiveException { - hive = Hive.get(this.conf); - for (LoadTableDesc ltd : loadTableWork) { - TableDesc td = ltd.getTable(); - Table srcTable = hive.getTable(td.getTableName()); - List<Index> tblIndexes = IndexUtils.getAllIndexes(srcTable, (short)-1); - Map<String, String> partSpec = ltd.getPartitionSpec(); - if (partSpec == null || partSpec.size() == 0) { - //unpartitioned table, update whole index - doIndexUpdate(tblIndexes); - } else { - doIndexUpdate(tblIndexes, partSpec); - } - } - return tasks; - } - - private void doIndexUpdate(List<Index> tblIndexes) throws HiveException { - for (Index idx : tblIndexes) { - StringBuilder sb = new StringBuilder(); - sb.append("ALTER INDEX "); - sb.append(idx.getIndexName()); - sb.append(" ON "); - sb.append(idx.getDbName()).append('.'); - sb.append(idx.getOrigTableName()); - sb.append(" REBUILD"); - compileRebuild(sb.toString()); - } - } - - private void doIndexUpdate(List<Index> tblIndexes, Map<String, String> - partSpec) throws HiveException { - for (Index index : tblIndexes) { - if (containsPartition(index, partSpec)) { - doIndexUpdate(index, partSpec); - } - } - } - - private void doIndexUpdate(Index index, Map<String, String> partSpec) { - StringBuilder ps = new StringBuilder(); - boolean first = true; - ps.append("("); - for (String key : partSpec.keySet()) { - if (!first) { - ps.append(", "); - } else { - first = false; - } - ps.append(key); - ps.append("="); - ps.append(partSpec.get(key)); - } - ps.append(")"); - StringBuilder sb = new StringBuilder(); - sb.append("ALTER INDEX "); - sb.append(index.getIndexName()); - sb.append(" ON "); - sb.append(index.getDbName()).append('.'); - sb.append(index.getOrigTableName()); - sb.append(" PARTITION "); - sb.append(ps.toString()); - sb.append(" REBUILD"); - compileRebuild(sb.toString()); - } - - private void compileRebuild(String query) { - Driver driver = new Driver(this.conf, lineageState); - driver.compile(query, false); - tasks.addAll(driver.getPlan().getRootTasks()); - inputs.addAll(driver.getPlan().getInputs()); - } - - - private boolean containsPartition(Index index, - Map<String, String> partSpec) throws HiveException { - String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName()); - Table indexTable = hive.getTable(qualified[0], qualified[1]); - List<Partition> parts = hive.getPartitions(indexTable, partSpec); - return (parts == null || parts.size() == 0); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index 3619763..cc66936 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -360,25 +360,7 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer { statTask = TaskFactory.get(columnStatsWork, conf); } - // HIVE-3334 has been filed for load file with index auto update - if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEINDEXAUTOUPDATE)) { - IndexUpdater indexUpdater = new IndexUpdater(loadTableWork, getInputs(), conf); - try { - List<Task<? extends Serializable>> indexUpdateTasks = indexUpdater.generateUpdateTasks(); - - for (Task<? extends Serializable> updateTask : indexUpdateTasks) { - //LOAD DATA will either have a copy & move or just a move, - // we always want the update to be dependent on the move - childTask.addDependentTask(updateTask); - if (statTask != null) { - updateTask.addDependentTask(statTask); - } - } - } catch (HiveException e) { - console.printInfo("WARNING: could not auto-update stale indexes, indexes are not out of sync"); - } - } - else if (statTask != null) { + if (statTask != null) { childTask.addDependentTask(statTask); } } http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java index 2e1f50e..34963ff 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java @@ -74,7 +74,6 @@ public final class SemanticAnalyzerFactory { commandType.put(HiveParser.TOK_SHOW_CREATEDATABASE, HiveOperation.SHOW_CREATEDATABASE); commandType.put(HiveParser.TOK_SHOW_CREATETABLE, HiveOperation.SHOW_CREATETABLE); commandType.put(HiveParser.TOK_SHOWFUNCTIONS, HiveOperation.SHOWFUNCTIONS); - commandType.put(HiveParser.TOK_SHOWINDEXES, HiveOperation.SHOWINDEXES); commandType.put(HiveParser.TOK_SHOWPARTITIONS, HiveOperation.SHOWPARTITIONS); commandType.put(HiveParser.TOK_SHOWLOCKS, HiveOperation.SHOWLOCKS); commandType.put(HiveParser.TOK_SHOWDBLOCKS, HiveOperation.SHOWLOCKS); @@ -90,10 +89,6 @@ public final class SemanticAnalyzerFactory { commandType.put(HiveParser.TOK_CREATE_MATERIALIZED_VIEW, HiveOperation.CREATE_MATERIALIZED_VIEW); commandType.put(HiveParser.TOK_DROPVIEW, HiveOperation.DROPVIEW); commandType.put(HiveParser.TOK_DROP_MATERIALIZED_VIEW, HiveOperation.DROP_MATERIALIZED_VIEW); - commandType.put(HiveParser.TOK_CREATEINDEX, HiveOperation.CREATEINDEX); - commandType.put(HiveParser.TOK_DROPINDEX, HiveOperation.DROPINDEX); - commandType.put(HiveParser.TOK_ALTERINDEX_REBUILD, HiveOperation.ALTERINDEX_REBUILD); - commandType.put(HiveParser.TOK_ALTERINDEX_PROPERTIES, HiveOperation.ALTERINDEX_PROPS); commandType.put(HiveParser.TOK_ALTERVIEW_PROPERTIES, HiveOperation.ALTERVIEW_PROPERTIES); commandType.put(HiveParser.TOK_ALTERVIEW_DROPPROPERTIES, HiveOperation.ALTERVIEW_PROPERTIES); commandType.put(HiveParser.TOK_ALTERVIEW_ADDPARTS, HiveOperation.ALTERTABLE_ADDPARTS); @@ -299,8 +294,6 @@ public final class SemanticAnalyzerFactory { case HiveParser.TOK_DESCTABLE: case HiveParser.TOK_DESCFUNCTION: case HiveParser.TOK_MSCK: - case HiveParser.TOK_ALTERINDEX_REBUILD: - case HiveParser.TOK_ALTERINDEX_PROPERTIES: case HiveParser.TOK_SHOWDATABASES: case HiveParser.TOK_SHOWTABLES: case HiveParser.TOK_SHOWCOLUMNS: @@ -310,7 +303,6 @@ public final class SemanticAnalyzerFactory { case HiveParser.TOK_SHOW_CREATETABLE: case HiveParser.TOK_SHOWFUNCTIONS: case HiveParser.TOK_SHOWPARTITIONS: - case HiveParser.TOK_SHOWINDEXES: case HiveParser.TOK_SHOWLOCKS: case HiveParser.TOK_SHOWDBLOCKS: case HiveParser.TOK_SHOW_COMPACTIONS: @@ -319,8 +311,6 @@ public final class SemanticAnalyzerFactory { case HiveParser.TOK_SHOWCONF: case HiveParser.TOK_SHOWVIEWS: case HiveParser.TOK_SHOWMATERIALIZEDVIEWS: - case HiveParser.TOK_CREATEINDEX: - case HiveParser.TOK_DROPINDEX: case HiveParser.TOK_ALTERTABLE_CLUSTER_SORT: case HiveParser.TOK_LOCKTABLE: case HiveParser.TOK_UNLOCKTABLE: http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index 92d29e3..3122db8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -220,21 +220,6 @@ public abstract class TaskCompiler { .get(new MoveWork(null, null, ltd, null, false), conf); mvTask.add(tsk); - // Check to see if we are stale'ing any indexes and auto-update them if we want - if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEINDEXAUTOUPDATE)) { - IndexUpdater indexUpdater = new IndexUpdater(loadTableWork, inputs, conf, - queryState.getLineageState()); - try { - List<Task<? extends Serializable>> indexUpdateTasks = indexUpdater - .generateUpdateTasks(); - for (Task<? extends Serializable> updateTask : indexUpdateTasks) { - tsk.addDependentTask(updateTask); - } - } catch (HiveException e) { - console - .printInfo("WARNING: could not auto-update stale indexes, which are not in sync"); - } - } } boolean oneLoadFileForCtas = true; http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterIndexDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterIndexDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterIndexDesc.java deleted file mode 100644 index a335495..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterIndexDesc.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -import java.io.Serializable; -import java.util.Map; - -import org.apache.hadoop.hive.ql.plan.Explain.Level; - -/** - * AlterIndexDesc. - * - */ -@Explain(displayName = "Alter Index", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) -public class AlterIndexDesc extends DDLDesc implements Serializable { - private static final long serialVersionUID = 1L; - private String indexName; - private String baseTable; - private Map<String, String> partSpec; // partition specification of partitions touched - private Map<String, String> props; - - /** - * alterIndexTypes. - * - */ - public static enum AlterIndexTypes { - UPDATETIMESTAMP, - ADDPROPS}; - - AlterIndexTypes op; - - public AlterIndexDesc() { - } - - public AlterIndexDesc(AlterIndexTypes type) { - this.op = type; - } - - /** - * @return the name of the index - */ - @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) - public String getIndexName() { - return indexName; - } - - /** - * @param indexName - * the indexName to set - */ - public void setIndexName(String indexName) { - this.indexName = indexName; - } - - /** - * @return the baseTable - */ - @Explain(displayName = "new name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) - public String getBaseTableName() { - return baseTable; - } - - /** - * @param baseTable - * the baseTable to set - */ - public void setBaseTableName(String baseTable) { - this.baseTable = baseTable; - } - - /** - * @return the partition spec - */ - public Map<String, String> getSpec() { - return partSpec; - } - - /** - * @param partSpec - * the partition spec to set - */ - public void setSpec(Map<String, String> partSpec) { - this.partSpec = partSpec; - } - - /** - * @return the op - */ - public AlterIndexTypes getOp() { - return op; - } - - /** - * @param op - * the op to set - */ - public void setOp(AlterIndexTypes op) { - this.op = op; - } - - /** - * @return the props - */ - @Explain(displayName = "properties") - public Map<String, String> getProps() { - return props; - } - - /** - * @param props - * the props to set - */ - public void setProps(Map<String, String> props) { - this.props = props; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateIndexDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateIndexDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateIndexDesc.java deleted file mode 100644 index c003ee5..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateIndexDesc.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -/** - * create index descriptor - */ -public class CreateIndexDesc extends DDLDesc implements Serializable { - - private static final long serialVersionUID = 1L; - String tableName; - String indexName; - List<String> indexedCols; - String indexTableName; - boolean deferredRebuild; - String inputFormat; - String outputFormat; - String serde; - String storageHandler; - String indexTypeHandlerClass; - String location; - Map<String, String> idxProps; - Map<String, String> tblProps; - Map<String, String> serdeProps; - String collItemDelim; - String fieldDelim; - String fieldEscape; - String lineDelim; - String mapKeyDelim; - - String indexComment; - - public CreateIndexDesc() { - super(); - } - - public CreateIndexDesc(String tableName, String indexName, - List<String> indexedCols, String indexTableName, boolean deferredRebuild, - String inputFormat, String outputFormat, String storageHandler, - String typeName, String location, Map<String, String> idxProps, Map<String, String> tblProps, - String serde, Map<String, String> serdeProps, String collItemDelim, - String fieldDelim, String fieldEscape, String lineDelim, - String mapKeyDelim, String indexComment) { - super(); - this.tableName = tableName; - this.indexName = indexName; - this.indexedCols = indexedCols; - this.indexTableName = indexTableName; - this.deferredRebuild = deferredRebuild; - this.inputFormat = inputFormat; - this.outputFormat = outputFormat; - this.serde = serde; - this.storageHandler = storageHandler; - this.indexTypeHandlerClass = typeName; - this.location = location; - this.idxProps = idxProps; - this.tblProps = tblProps; - this.serde = serde; - this.serdeProps = serdeProps; - this.collItemDelim = collItemDelim; - this.fieldDelim = fieldDelim; - this.fieldEscape = fieldEscape; - this.lineDelim = lineDelim; - this.mapKeyDelim = mapKeyDelim; - this.indexComment = indexComment; - } - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public String getIndexName() { - return indexName; - } - - public void setIndexName(String indexName) { - this.indexName = indexName; - } - - public List<String> getIndexedCols() { - return indexedCols; - } - - public void setIndexedCols(List<String> indexedCols) { - this.indexedCols = indexedCols; - } - - public String getIndexTableName() { - return indexTableName; - } - - public void setIndexTableName(String indexTableName) { - this.indexTableName = indexTableName; - } - - public boolean isDeferredRebuild() { - return deferredRebuild; - } - - public boolean getDeferredRebuild() { - return deferredRebuild; - } - - public void setDeferredRebuild(boolean deferredRebuild) { - this.deferredRebuild = deferredRebuild; - } - - public String getInputFormat() { - return inputFormat; - } - - public void setInputFormat(String inputFormat) { - this.inputFormat = inputFormat; - } - - public String getOutputFormat() { - return outputFormat; - } - - public void setOutputFormat(String outputFormat) { - this.outputFormat = outputFormat; - } - - public String getSerde() { - return serde; - } - - public void setSerde(String serde) { - this.serde = serde; - } - - public String getStorageHandler() { - return storageHandler; - } - - public void setStorageHandler(String storageHandler) { - this.storageHandler = storageHandler; - } - - public String getLocation() { - return location; - } - - public void setLocation(String location) { - this.location = location; - } - - public Map<String, String> getIdxProps() { - return idxProps; - } - - public void setIdxProps(Map<String, String> idxProps) { - this.idxProps = idxProps; - } - - public Map<String, String> getTblProps() { - return tblProps; - } - - public void setTblProps(Map<String, String> tblProps) { - this.tblProps = tblProps; - } - - public Map<String, String> getSerdeProps() { - return serdeProps; - } - - public void setSerdeProps(Map<String, String> serdeProps) { - this.serdeProps = serdeProps; - } - - public String getCollItemDelim() { - return collItemDelim; - } - - public void setCollItemDelim(String collItemDelim) { - this.collItemDelim = collItemDelim; - } - - public String getFieldDelim() { - return fieldDelim; - } - - public void setFieldDelim(String fieldDelim) { - this.fieldDelim = fieldDelim; - } - - public String getFieldEscape() { - return fieldEscape; - } - - public void setFieldEscape(String fieldEscape) { - this.fieldEscape = fieldEscape; - } - - public String getLineDelim() { - return lineDelim; - } - - public void setLineDelim(String lineDelim) { - this.lineDelim = lineDelim; - } - - public String getMapKeyDelim() { - return mapKeyDelim; - } - - public void setMapKeyDelim(String mapKeyDelim) { - this.mapKeyDelim = mapKeyDelim; - } - - public String getIndexTypeHandlerClass() { - return indexTypeHandlerClass; - } - - public void setIndexTypeHandlerClass(String indexTypeHandlerClass) { - this.indexTypeHandlerClass = indexTypeHandlerClass; - } - - public String getIndexComment() { - return indexComment; - } - - public void setIndexComment(String indexComment) { - this.indexComment = indexComment; - } - -}