http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java index 1c7330a..6f99330 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java @@ -19,8 +19,10 @@ package org.apache.asterix.optimizer.rules.am; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -53,6 +55,7 @@ import org.apache.asterix.runtime.evaluators.functions.FullTextContainsDescripto import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.common.utils.Triple; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; @@ -70,7 +73,6 @@ import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; @@ -107,33 +109,34 @@ public class InvertedIndexAccessMethod implements IAccessMethod { DISJUNCTIVE } - private static List<FunctionIdentifier> funcIdents = new ArrayList<>(); - - static { - funcIdents.add(BuiltinFunctions.STRING_CONTAINS); - // For matching similarity-check functions. For example, similarity-jaccard-check returns a list of two items, - // and the select condition will get the first list-item and check whether it evaluates to true. - funcIdents.add(BuiltinFunctions.GET_ITEM); - // Full-text search function - funcIdents.add(BuiltinFunctions.FULLTEXT_CONTAINS); - funcIdents.add(BuiltinFunctions.FULLTEXT_CONTAINS_WO_OPTION); - } + // The second boolean value tells whether the given function generates false positive results. + // That is, this function can produce false positive results if it is set to true. + // In this case, an index-search alone cannot replace the given SELECT condition and + // that SELECT condition needs to be applied after the index-search to get the correct results. + // Currently, only full-text index search does not generate false positive results. + private static final List<Pair<FunctionIdentifier, Boolean>> FUNC_IDENTIFIERS = Collections.unmodifiableList( + Arrays.asList(new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.STRING_CONTAINS, true), + // For matching similarity-check functions. For example, similarity-jaccard-check returns + // a list of two items, and the select condition will get the first list-item and + // check whether it evaluates to true. + new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.GET_ITEM, true), + // Full-text search function + new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.FULLTEXT_CONTAINS, false), + new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.FULLTEXT_CONTAINS_WO_OPTION, false))); // These function identifiers are matched in this AM's analyzeFuncExprArgs(), // and are not visible to the outside driver. - private static HashSet<FunctionIdentifier> secondLevelFuncIdents = new HashSet<>(); - - static { - secondLevelFuncIdents.add(BuiltinFunctions.SIMILARITY_JACCARD_CHECK); - secondLevelFuncIdents.add(BuiltinFunctions.EDIT_DISTANCE_CHECK); - secondLevelFuncIdents.add(BuiltinFunctions.EDIT_DISTANCE_CONTAINS); - } + private static final List<Pair<FunctionIdentifier, Boolean>> SECOND_LEVEL_FUNC_IDENTIFIERS = + Collections.unmodifiableList(Arrays.asList( + new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.SIMILARITY_JACCARD_CHECK, true), + new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.EDIT_DISTANCE_CHECK, true), + new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.EDIT_DISTANCE_CONTAINS, true))); public static InvertedIndexAccessMethod INSTANCE = new InvertedIndexAccessMethod(); @Override - public List<FunctionIdentifier> getOptimizableFunctions() { - return funcIdents; + public List<Pair<FunctionIdentifier, Boolean>> getOptimizableFunctions() { + return FUNC_IDENTIFIERS; } @Override @@ -224,10 +227,21 @@ public class InvertedIndexAccessMethod implements IAccessMethod { } } } - // Check that the matched function is optimizable by this access method. - if (!secondLevelFuncIdents.contains(matchedFuncExpr.getFunctionIdentifier())) { + // Checks that the matched function is optimizable by this access method. + boolean found = false; + for (Iterator<Pair<FunctionIdentifier, Boolean>> iterator = SECOND_LEVEL_FUNC_IDENTIFIERS.iterator(); iterator + .hasNext();) { + FunctionIdentifier fID = iterator.next().first; + + if (fID != null && matchedFuncExpr != null && fID.equals(matchedFuncExpr.getFunctionIdentifier())) { + found = true; + break; + } + } + if (!found) { return false; } + boolean selectMatchFound = analyzeSelectSimilarityCheckFuncExprArgs(matchedFuncExpr, assignsAndUnnests, matchedAssignOrUnnestIndex, analysisCtx); boolean joinMatchFound = analyzeJoinSimilarityCheckFuncExprArgs(matchedFuncExpr, assignsAndUnnests, @@ -376,10 +390,18 @@ public class InvertedIndexAccessMethod implements IAccessMethod { } @Override - public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, - OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, - AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, - IOptimizationContext context) throws AlgebricksException { + public ILogicalOperator createIndexSearchPlan(List<Mutable<ILogicalOperator>> afterTopOpRefs, + Mutable<ILogicalOperator> topOpRef, Mutable<ILogicalExpression> conditionRef, + List<Mutable<ILogicalOperator>> assignBeforeTopOpRefs, OptimizableOperatorSubTree indexSubTree, + OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, + boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context, + LogicalVariable newNullPlaceHolderForLOJ) throws AlgebricksException { + // TODO: we currently do not support the index-only plan for the inverted index searches since + // there can be many <SK, PK> pairs for the same PK and we may see two different records with the same PK + // (e.g., the record is deleted and inserted with the same PK). The reason is that there are + // no locking processes during a secondary index DML operation. When a secondary index search can see + // the only one version of the record during the lifetime of a query, index-only plan can be applied. + boolean generateInstantTrylockResultFromIndexSearch = false; IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx); Dataset dataset = indexSubTree.getDataset(); @@ -412,6 +434,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod { // Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input). inputOp.getInputs().add(new MutableObject<>( OperatorManipulationUtil.deepCopy(dataSourceScan.getInputs().get(0).getValue()))); + context.computeAndSetTypeEnvironmentForOperator(inputOp); inputOp.setExecutionMode(dataSourceScan.getExecutionMode()); } else { // We are optimizing a join. Add the input variable to the secondaryIndexFuncArgs. @@ -420,13 +443,17 @@ public class InvertedIndexAccessMethod implements IAccessMethod { inputOp = (AbstractLogicalOperator) probeSubTree.getRoot(); } jobGenParams.setKeyVarList(keyVarList); + // By default, we don't generate SK output for an inverted index + // since it doesn't contain a field value, only part of it. ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType, - metaRecordType, chosenIndex, inputOp, jobGenParams, context, true, retainInput, retainNull); + metaRecordType, chosenIndex, inputOp, jobGenParams, context, retainInput, retainNull, + generateInstantTrylockResultFromIndexSearch); - // Generate the rest of the upstream plan which feeds the search results into the primary index. - AbstractUnnestMapOperator primaryIndexUnnestOp = - AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceScan, dataset, recordType, metaRecordType, - secondaryIndexUnnestOp, context, true, retainInput, retainNull, false); + // Generates the rest of the upstream plan which feeds the search results into the primary index. + ILogicalOperator primaryIndexUnnestOp = AccessMethodUtils.createRestOfIndexSearchPlan(afterTopOpRefs, topOpRef, + conditionRef, assignBeforeTopOpRefs, dataSourceScan, dataset, recordType, metaRecordType, + secondaryIndexUnnestOp, context, true, retainInput, retainNull, false, chosenIndex, analysisCtx, + indexSubTree, newNullPlaceHolderForLOJ); return primaryIndexUnnestOp; } @@ -451,23 +478,24 @@ public class InvertedIndexAccessMethod implements IAccessMethod { public boolean applySelectPlanTransformation(List<Mutable<ILogicalOperator>> afterSelectRefs, Mutable<ILogicalOperator> selectRef, OptimizableOperatorSubTree subTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException { - ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, subTree, null, chosenIndex, analysisCtx, - AccessMethodUtils.retainInputs(subTree.getDataSourceVariables(), subTree.getDataSourceRef().getValue(), - afterSelectRefs), - false, subTree.getDataSourceRef().getValue().getInputs().get(0).getValue() - .getExecutionMode() == ExecutionMode.UNPARTITIONED, - context); + SelectOperator selectOp = (SelectOperator) selectRef.getValue(); + ILogicalOperator indexPlanRootOp = + createIndexSearchPlan(afterSelectRefs, selectRef, selectOp.getCondition(), + subTree.getAssignsAndUnnestsRefs(), + subTree, null, chosenIndex, analysisCtx, false, false, subTree.getDataSourceRef().getValue() + .getInputs().get(0).getValue().getExecutionMode() == ExecutionMode.UNPARTITIONED, + context, null); + // Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap. subTree.getDataSourceRef().setValue(indexPlanRootOp); return true; } @Override - public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef, - OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex, - AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin, - boolean hasGroupBy) throws AlgebricksException { - // Figure out if the index is applicable on the left or right side (if both, we arbitrarily prefer the left side). + public boolean applyJoinPlanTransformation(List<Mutable<ILogicalOperator>> afterJoinRefs, + Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree, + OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, + IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException { Dataset dataset = analysisCtx.getDatasetFromIndexDatasetMap(chosenIndex); OptimizableOperatorSubTree indexSubTree; OptimizableOperatorSubTree probeSubTree; @@ -485,7 +513,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod { IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx); // The arguments of edit-distance-contains() function are asymmetrical, we can only use index - // if the dataset of index subtree and the dataset of first argument's subtree is the same + // if the dataset of index subtree and the dataset of first argument's subtree is the same. if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == BuiltinFunctions.EDIT_DISTANCE_CONTAINS && optFuncExpr.getOperatorSubTree(0).getDataset() != null && !optFuncExpr.getOperatorSubTree(0) .getDataset().getDatasetName().equals(indexSubTree.getDataset().getDatasetName())) { @@ -500,7 +528,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod { newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0); //reset the null place holder variable - AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context); + AccessMethodUtils.resetLOJMissingPlaceholderVarInGroupByOp(analysisCtx, newNullPlaceHolderVar, context); } AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) joinRef.getValue(); @@ -535,8 +563,9 @@ public class InvertedIndexAccessMethod implements IAccessMethod { probeSubTree.setRoot(newProbeRootRef.getValue()); } // Create regular indexed-nested loop join path. - ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, indexSubTree, probeSubTree, chosenIndex, - analysisCtx, true, isLeftOuterJoin, true, context); + ILogicalOperator indexPlanRootOp = createIndexSearchPlan(afterJoinRefs, joinRef, + new MutableObject<ILogicalExpression>(joinCond), indexSubTree.getAssignsAndUnnestsRefs(), indexSubTree, + probeSubTree, chosenIndex, analysisCtx, true, isLeftOuterJoin, true, context, newNullPlaceHolderVar); indexSubTree.getDataSourceRef().setValue(indexPlanRootOp); // Change join into a select with the same condition.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java index 2534680..7c2edb9 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java @@ -19,7 +19,9 @@ package org.apache.asterix.optimizer.rules.am; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; @@ -61,6 +63,7 @@ public class OptimizableOperatorSubTree { EXTERNAL_SCAN, PRIMARY_INDEX_LOOKUP, COLLECTION_SCAN, + INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP, NO_DATASOURCE } @@ -75,6 +78,9 @@ public class OptimizableOperatorSubTree { private Dataset dataset = null; private ARecordType recordType = null; private ARecordType metaRecordType = null; + // Contains the field names for all assign operations in this sub-tree. + // This will be used for the index-only plan check. + private Map<LogicalVariable, List<String>> varsToFieldNameMap = new HashMap<>(); // Additional datasources can exist if IntroduceJoinAccessMethodRule has been applied. // (E.g. There are index-nested-loop-joins in the plan.) @@ -83,6 +89,9 @@ public class OptimizableOperatorSubTree { private List<Dataset> ixJoinOuterAdditionalDatasets = null; private List<ARecordType> ixJoinOuterAdditionalRecordTypes = null; + /** + * Identifies the root of the subtree and initializes the data-source, assign, and unnest information. + */ public boolean initFromSubTree(Mutable<ILogicalOperator> subTreeOpRef) throws AlgebricksException { reset(); rootRef = subTreeOpRef; @@ -160,15 +169,11 @@ public class OptimizableOperatorSubTree { AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams(); jobGenParams.readFromFuncArgs(f.getArguments()); if (jobGenParams.isPrimaryIndex()) { - if (getDataSourceRef() == null) { - setDataSourceRef(subTreeOpRef); - setDataSourceType(DataSourceType.PRIMARY_INDEX_LOOKUP); - } else { - // One datasource already exists. This is an additional datasource. - initializeIxJoinOuterAddtionalDataSourcesIfEmpty(); - getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.PRIMARY_INDEX_LOOKUP); - getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef); - } + intializeDataSourceRefAndType(DataSourceType.PRIMARY_INDEX_LOOKUP, subTreeOpRef); + dataSourceFound = true; + } else if (unnestMapOp.getGenerateCallBackProceedResultVar()) { + intializeDataSourceRefAndType(DataSourceType.INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP, + subTreeOpRef); dataSourceFound = true; } } else if (f.getFunctionIdentifier().equals(BuiltinFunctions.EXTERNAL_LOOKUP)) { @@ -213,6 +218,18 @@ public class OptimizableOperatorSubTree { return false; } + private void intializeDataSourceRefAndType(DataSourceType dsType, Mutable<ILogicalOperator> opRef) { + if (getDataSourceRef() == null) { + setDataSourceRef(opRef); + setDataSourceType(dsType); + } else { + // One datasource already exists. This is an additional datasource. + initializeIxJoinOuterAddtionalDataSourcesIfEmpty(); + getIxJoinOuterAdditionalDataSourceTypes().add(dsType); + getIxJoinOuterAdditionalDataSourceRefs().add(opRef); + } + } + /** * Find the dataset corresponding to the datasource scan in the metadata. * Also sets recordType to be the type of that dataset. @@ -254,6 +271,7 @@ public class OptimizableOperatorSubTree { datasetName = datasetInfo.second; break; case PRIMARY_INDEX_LOOKUP: + case INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP: AbstractUnnestOperator unnestMapOp = (AbstractUnnestOperator) sourceOpRefs.get(i).getValue(); ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue(); AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr; @@ -369,7 +387,7 @@ public class OptimizableOperatorSubTree { } /** - * Get primary key variables from the given data-source. + * Gets the primary key variables from the given data-source. */ public void getPrimaryKeyVars(Mutable<ILogicalOperator> dataSourceRefToFetch, List<LogicalVariable> target) throws AlgebricksException { @@ -389,12 +407,26 @@ public class OptimizableOperatorSubTree { primaryKeys = AccessMethodUtils.getPrimaryKeyVarsFromPrimaryUnnestMap(dataset, unnestMapOp); target.addAll(primaryKeys); break; + case INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP: + AbstractUnnestMapOperator idxOnlyPlanUnnestMapOp = + (AbstractUnnestMapOperator) dataSourceRefToFetchKey.getValue(); + List<LogicalVariable> idxOnlyPlanKeyVars = idxOnlyPlanUnnestMapOp.getVariables(); + int indexOnlyPlanNumPrimaryKeys = dataset.getPrimaryKeys().size(); + // The order of variables: SK, PK, the result of instantTryLock on PK. + // The last variable keeps the result of instantTryLock on PK. + // Thus, we deduct 1 to only count key variables. + int start = idxOnlyPlanKeyVars.size() - 1 - indexOnlyPlanNumPrimaryKeys; + int end = start + indexOnlyPlanNumPrimaryKeys; + + for (int i = start; i < end; i++) { + target.add(idxOnlyPlanKeyVars.get(i)); + } + break; case EXTERNAL_SCAN: break; case NO_DATASOURCE: default: throw CompilationException.create(ErrorCode.SUBTREE_HAS_NO_DATA_SOURCE); - } } @@ -405,6 +437,11 @@ public class OptimizableOperatorSubTree { case PRIMARY_INDEX_LOOKUP: AbstractScanOperator scanOp = (AbstractScanOperator) getDataSourceRef().getValue(); return scanOp.getVariables(); + case INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP: + // This data-source doesn't have record variables. + List<LogicalVariable> pkVars = new ArrayList<>(); + getPrimaryKeyVars(dataSourceRef, pkVars); + return pkVars; case COLLECTION_SCAN: return new ArrayList<>(); case NO_DATASOURCE: @@ -422,6 +459,10 @@ public class OptimizableOperatorSubTree { AbstractScanOperator scanOp = (AbstractScanOperator) getIxJoinOuterAdditionalDataSourceRefs().get(idx).getValue(); return scanOp.getVariables(); + case INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP: + List<LogicalVariable> PKVars = new ArrayList<>(); + getPrimaryKeyVars(ixJoinOuterAdditionalDataSourceRefs.get(idx), PKVars); + return PKVars; case COLLECTION_SCAN: return new ArrayList<>(); case NO_DATASOURCE: @@ -539,4 +580,8 @@ public class OptimizableOperatorSubTree { this.ixJoinOuterAdditionalRecordTypes = ixJoinOuterAdditionalRecordTypes; } + public Map<LogicalVariable, List<String>> getVarsToFieldNameMap() { + return varsToFieldNameMap; + } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java index fd46194..f431603 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java @@ -19,6 +19,8 @@ package org.apache.asterix.optimizer.rules.am; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.apache.asterix.common.annotations.SkipSecondaryIndexSearchExpressionAnnotation; @@ -31,15 +33,19 @@ import org.apache.asterix.om.base.AInt32; import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.om.types.ARecordType; +import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; +import org.apache.hyracks.algebricks.common.utils.Quadruple; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; +import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; +import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression; @@ -49,27 +55,31 @@ import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator; import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil; +import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil; /** * Class for helping rewrite rules to choose and apply RTree indexes. */ public class RTreeAccessMethod implements IAccessMethod { - private static List<FunctionIdentifier> funcIdents = new ArrayList<>(); - - static { - funcIdents.add(BuiltinFunctions.SPATIAL_INTERSECT); - } + // The second boolean value tells whether the given function generates false positive results. + // That is, this function can produce false positive results if it is set to true. + // In this case, an index-search alone cannot replace the given SELECT condition and + // that SELECT condition needs to be applied after the index-search to get the final results. + // In R-Tree case, depending on the parameters of the SPATIAL_INTERSECT function, it may/may not produce + // false positive results. Thus, we need to have one more step to check whether the SPATIAL_INTERSECT generates + // false positive results or not. + private static final List<Pair<FunctionIdentifier, Boolean>> FUNC_IDENTIFIERS = Collections.unmodifiableList( + Arrays.asList(new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.SPATIAL_INTERSECT, true))); public static final RTreeAccessMethod INSTANCE = new RTreeAccessMethod(); @Override - public List<FunctionIdentifier> getOptimizableFunctions() { - return funcIdents; + public List<Pair<FunctionIdentifier, Boolean>> getOptimizableFunctions() { + return FUNC_IDENTIFIERS; } @Override @@ -98,87 +108,110 @@ public class RTreeAccessMethod implements IAccessMethod { public boolean applySelectPlanTransformation(List<Mutable<ILogicalOperator>> afterSelectRefs, Mutable<ILogicalOperator> selectRef, OptimizableOperatorSubTree subTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException { + SelectOperator selectOp = (SelectOperator) selectRef.getValue(); + Mutable<ILogicalExpression> conditionRef = selectOp.getCondition(); + AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) conditionRef.getValue(); + ARecordType recordType = subTree.getRecordType(); + // TODO: We can probably do something smarter here based on selectivity or MBR area. - ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(subTree, null, chosenIndex, analysisCtx, - AccessMethodUtils.retainInputs(subTree.getDataSourceVariables(), subTree.getDataSourceRef().getValue(), - afterSelectRefs), - false, false, context); - if (primaryIndexUnnestOp == null) { + IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx); + + int optFieldIdx = AccessMethodUtils.chooseFirstOptFuncVar(chosenIndex, analysisCtx); + Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(optFuncExpr.getFieldType(optFieldIdx), + optFuncExpr.getFieldName(optFieldIdx), recordType); + if (keyPairType == null) { return false; } - // Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap. - subTree.getDataSourceRef().setValue(primaryIndexUnnestOp); - return true; - } - @Override - public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef, - OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, - AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, - IOptimizationContext context) throws AlgebricksException { - return createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex, analysisCtx, retainInput, - retainNull, requiresBroadcast, context); - } + // To check whether the given plan is an index-only plan: + // index-only plan possible? + boolean isIndexOnlyPlan = false; - @Override - public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef, - OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex, - AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin, - boolean hasGroupBy) throws AlgebricksException { - // Determine if the index is applicable on the left or right side (if both, we arbitrarily prefer the left - // side). - Dataset dataset = analysisCtx.getDatasetFromIndexDatasetMap(chosenIndex); - OptimizableOperatorSubTree indexSubTree; - OptimizableOperatorSubTree probeSubTree; + // secondary key field usage after the select operator + boolean secondaryKeyFieldUsedAfterSelectOp = false; - // We assume that the left subtree is the outer branch and the right subtree is the inner branch. - // This assumption holds true since we only use an index from the right subtree. - // The following is just a sanity check. - if (rightSubTree.hasDataSourceScan() - && dataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) { - indexSubTree = rightSubTree; - probeSubTree = leftSubTree; - } else { + // Whether a verification is required after the secondary index search + // In other words, can the chosen method generate any false positive results? + boolean requireVerificationAfterSIdxSearch = false; + Pair<Boolean, Boolean> functionFalsePositiveCheck = + AccessMethodUtils.canFunctionGenerateFalsePositiveResultsUsingIndex(funcExpr, FUNC_IDENTIFIERS); + + if (!functionFalsePositiveCheck.first) { return false; } - LogicalVariable newNullPlaceHolderVar = null; - if (isLeftOuterJoin) { - // get a new null place holder variable that is the first field variable of the primary key - // from the indexSubTree's datasourceScanOp - newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0); + // Does the given index can cover all search predicates? + boolean doesSIdxSearchCoverAllPredicates = false; + + // Preliminary check for the index-only plan for R-Tree: + // If the given index is not built on a POINT or a RECTANGLE field, + // the query result can include false positives. And the result from secondary index search is an MBR, + // thus we can't construct original secondary field value to remove any false positive results. + if (keyPairType.first.getTypeTag() == BuiltinType.APOINT.getTypeTag() + || keyPairType.first.getTypeTag() == BuiltinType.ARECTANGLE.getTypeTag()) { + isIndexOnlyPlan = true; + // The following variable can be changed if a query shape is not a POINT or rectangle. + requireVerificationAfterSIdxSearch = false; + } else { + isIndexOnlyPlan = false; + requireVerificationAfterSIdxSearch = true; } - // TODO: We can probably do something smarter here based on selectivity or MBR area. - ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex, - analysisCtx, true, isLeftOuterJoin, true, context); + Quadruple<Boolean, Boolean, Boolean, Boolean> indexOnlyPlanInfo = + new Quadruple<>(isIndexOnlyPlan, secondaryKeyFieldUsedAfterSelectOp, requireVerificationAfterSIdxSearch, + doesSIdxSearchCoverAllPredicates); + + Dataset dataset = subTree.getDataset(); + + // Is this plan an index-only plan? + if (isIndexOnlyPlan) { + if (dataset.getDatasetType() == DatasetType.INTERNAL) { + AccessMethodUtils.indexOnlyPlanCheck(afterSelectRefs, selectRef, subTree, null, chosenIndex, + analysisCtx, context, indexOnlyPlanInfo); + isIndexOnlyPlan = indexOnlyPlanInfo.getFirst(); + } else { + // An index on an external dataset can't be optimized for the index-only plan. + isIndexOnlyPlan = false; + indexOnlyPlanInfo.setFirst(isIndexOnlyPlan); + } + } + + analysisCtx.setIndexOnlyPlanInfo(indexOnlyPlanInfo); + + ILogicalOperator primaryIndexUnnestOp = createIndexSearchPlan(afterSelectRefs, selectRef, + selectOp.getCondition(), subTree.getAssignsAndUnnestsRefs(), subTree, null, chosenIndex, analysisCtx, + AccessMethodUtils.retainInputs(subTree.getDataSourceVariables(), subTree.getDataSourceRef().getValue(), + afterSelectRefs), + false, false, context, null); + if (primaryIndexUnnestOp == null) { return false; } - if (isLeftOuterJoin && hasGroupBy) { - // reset the null place holder variable - AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context); + // Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap. + if (!isIndexOnlyPlan || dataset.getDatasetType() == DatasetType.EXTERNAL) { + subTree.getDataSourceRef().setValue(primaryIndexUnnestOp); + } else { + // If this is an index-only plan, the topmost operator returned is UNIONALL operator. + if (primaryIndexUnnestOp.getOperatorTag() == LogicalOperatorTag.UNIONALL) { + selectRef.setValue(primaryIndexUnnestOp); + } else { + subTree.getDataSourceRef().setValue(primaryIndexUnnestOp); + } } - - indexSubTree.getDataSourceRef().setValue(primaryIndexUnnestOp); - // Change join into a select with the same condition. - AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue(); - SelectOperator topSelect = new SelectOperator(joinOp.getCondition(), isLeftOuterJoin, newNullPlaceHolderVar); - topSelect.getInputs().add(indexSubTree.getRootRef()); - topSelect.setExecutionMode(ExecutionMode.LOCAL); - context.computeAndSetTypeEnvironmentForOperator(topSelect); - // Replace the original join with the new subtree rooted at the select op. - joinRef.setValue(topSelect); return true; } - private ILogicalOperator createSecondaryToPrimaryPlan(OptimizableOperatorSubTree indexSubTree, + @Override + public ILogicalOperator createIndexSearchPlan(List<Mutable<ILogicalOperator>> afterTopRefs, + Mutable<ILogicalOperator> topRef, Mutable<ILogicalExpression> conditionRef, + List<Mutable<ILogicalOperator>> assignBeforeTopRefs, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, - boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) - throws AlgebricksException { - + boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context, + LogicalVariable newNullPlaceHolderForLOJ) throws AlgebricksException { + // TODO: We can probably do something smarter here based on selectivity or MBR area. IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx); + Dataset dataset = indexSubTree.getDataset(); ARecordType recordType = indexSubTree.getRecordType(); ARecordType metaRecordType = indexSubTree.getMetaRecordType(); @@ -195,43 +228,48 @@ public class RTreeAccessMethod implements IAccessMethod { IAType spatialType = keyPairType.first; int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag()); int numSecondaryKeys = numDimensions * 2; - // we made sure indexSubTree has datasource scan + + Quadruple<Boolean, Boolean, Boolean, Boolean> indexOnlyPlanInfo = analysisCtx.getIndexOnlyPlanInfo(); + boolean isIndexOnlyPlan = indexOnlyPlanInfo.getFirst(); + // We apply index-only plan for an internal dataset. + boolean generateInstantTrylockResultFromIndexSearch = + dataset.getDatasetType() == DatasetType.INTERNAL && isIndexOnlyPlan ? true : false; + + // We made sure that the indexSubTree has a datasource scan. AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.getDataSourceRef().getValue(); RTreeJobGenParams jobGenParams = new RTreeJobGenParams(chosenIndex.getIndexName(), IndexType.RTREE, dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast); // A spatial object is serialized in the constant of the func expr we are optimizing. // The R-Tree expects as input an MBR represented with 1 field per dimension. - // Here we generate vars and funcs for extracting MBR fields from the constant into fields of a tuple (as the - // R-Tree expects them). - // List of variables for the assign. + // Here we generate vars and funcs for extracting MBR fields from the constant into fields of a tuple + // (as the R-Tree expects them). List of variables for the assign. ArrayList<LogicalVariable> keyVarList = new ArrayList<>(); // List of expressions for the assign. ArrayList<Mutable<ILogicalExpression>> keyExprList = new ArrayList<>(); - Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr = AccessMethodUtils.createSearchKeyExpr(chosenIndex, - optFuncExpr, optFieldType, indexSubTree, probeSubTree); - ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first; + ILogicalExpression returnedSearchKeyExpr = + AccessMethodUtils.createSearchKeyExpr(chosenIndex, optFuncExpr, optFieldType, probeSubTree).first; for (int i = 0; i < numSecondaryKeys; i++) { // The create MBR function "extracts" one field of an MBR around the given spatial object. AbstractFunctionCallExpression createMBR = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CREATE_MBR)); // Spatial object is the constant from the func expr we are optimizing. - createMBR.getArguments().add(new MutableObject<>(searchKeyExpr)); - // The number of dimensions. + createMBR.getArguments().add(new MutableObject<>(returnedSearchKeyExpr)); + // The number of dimensions createMBR.getArguments().add(new MutableObject<ILogicalExpression>( new ConstantExpression(new AsterixConstantValue(new AInt32(numDimensions))))); - // Which part of the MBR to extract. + // Which part of the MBR to extract? createMBR.getArguments().add(new MutableObject<ILogicalExpression>( new ConstantExpression(new AsterixConstantValue(new AInt32(i))))); - // Add a variable and its expr to the lists which will be passed into an assign op. + // Adds a variable and its expr to the lists which will be passed into an assign op. LogicalVariable keyVar = context.newVar(); keyVarList.add(keyVar); keyExprList.add(new MutableObject<ILogicalExpression>(createMBR)); } jobGenParams.setKeyVarList(keyVarList); - // Assign operator that "extracts" the MBR fields from the func-expr constant into a tuple. + // Assigns an operator that "extracts" the MBR fields from the func-expr constant into a tuple. AssignOperator assignSearchKeys = new AssignOperator(keyVarList, keyExprList); if (probeSubTree == null) { // We are optimizing a selection query. @@ -242,17 +280,77 @@ public class RTreeAccessMethod implements IAccessMethod { } else { // We are optimizing a join, place the assign op top of the probe subtree. assignSearchKeys.getInputs().add(probeSubTree.getRootRef()); + assignSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode()); + OperatorPropertiesUtil.typeOpRec(probeSubTree.getRootRef(), context); } + context.computeAndSetTypeEnvironmentForOperator(assignSearchKeys); ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType, - metaRecordType, chosenIndex, assignSearchKeys, jobGenParams, context, false, retainInput, retainNull); + metaRecordType, chosenIndex, assignSearchKeys, jobGenParams, context, retainInput, retainNull, + generateInstantTrylockResultFromIndexSearch); - // Generate the rest of the upstream plan which feeds the search results into the primary index. + // Generates the rest of the upstream plan which feeds the search results into the primary index. return dataset.getDatasetType() == DatasetType.EXTERNAL - ? AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp, dataset, recordType, - secondaryIndexUnnestOp, context, retainInput, retainNull) - : AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType, metaRecordType, - secondaryIndexUnnestOp, context, true, retainInput, false, false); + ? AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp, dataset, recordType, metaRecordType, + secondaryIndexUnnestOp, context, chosenIndex, retainInput, retainNull) + : AccessMethodUtils.createRestOfIndexSearchPlan(afterTopRefs, topRef, conditionRef, assignBeforeTopRefs, + dataSourceOp, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, + retainInput, retainNull, false, chosenIndex, analysisCtx, indexSubTree, + newNullPlaceHolderForLOJ); + } + + @Override + public boolean applyJoinPlanTransformation(List<Mutable<ILogicalOperator>> afterJoinRefs, + Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree, + OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, + IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException { + AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue(); + Mutable<ILogicalExpression> conditionRef = joinOp.getCondition(); + + AbstractFunctionCallExpression funcExpr = null; + if (conditionRef.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { + funcExpr = (AbstractFunctionCallExpression) conditionRef.getValue(); + } + + Dataset dataset = analysisCtx.getIndexDatasetMap().get(chosenIndex); + + // Determine if the index is applicable on the right (inner) side. + OptimizableOperatorSubTree indexSubTree = null; + OptimizableOperatorSubTree probeSubTree = null; + // We assume that the left subtree is the outer branch and the right subtree is the inner branch. + // This assumption holds true since we only use an index from the right subtree. + // The following is just a sanity check. + if (rightSubTree.hasDataSourceScan() + && dataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) { + indexSubTree = rightSubTree; + probeSubTree = leftSubTree; + } else { + return false; + } + + LogicalVariable newNullPlaceHolderVar = null; + if (isLeftOuterJoin) { + // Gets a new null place holder variable that is the first field variable of the primary key + // from the indexSubTree's datasourceScanOp. + newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0); + } + + boolean canContinue = AccessMethodUtils.setIndexOnlyPlanInfo(afterJoinRefs, joinRef, probeSubTree, indexSubTree, + chosenIndex, analysisCtx, context, funcExpr, FUNC_IDENTIFIERS); + if (!canContinue) { + return false; + } + + ILogicalOperator indexSearchOp = createIndexSearchPlan(afterJoinRefs, joinRef, conditionRef, + indexSubTree.getAssignsAndUnnestsRefs(), indexSubTree, probeSubTree, chosenIndex, analysisCtx, true, + isLeftOuterJoin, true, context, newNullPlaceHolderVar); + + if (indexSearchOp == null) { + return false; + } + + return AccessMethodUtils.finalizeJoinPlanTransformation(afterJoinRefs, joinRef, indexSubTree, analysisCtx, + context, isLeftOuterJoin, hasGroupBy, indexSearchOp, newNullPlaceHolderVar, conditionRef, dataset); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/data/nontagged/customerData2.json ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/data/nontagged/customerData2.json b/asterixdb/asterix-app/data/nontagged/customerData2.json new file mode 100644 index 0000000..7624a71 --- /dev/null +++ b/asterixdb/asterix-app/data/nontagged/customerData2.json @@ -0,0 +1,6 @@ +{ "cid": 775, "name": "Jodi Rotruck", "cashBack": 775, "age": null, "address": { "number": 8389, "street": "Hill St.", "city": "Mountain View" }, "lastorder": { "oid": 66, "total": 38.618626f } } +{ "cid": 5, "name": "Jodi Alex", "cashBack": 5, "age": 19, "address": null, "lastorder": { "oid": 48, "total": 318.618626f } } +{ "cid": 1, "name": "Mike Carey", "cashBack": 1, "address": { "number": 389, "street": "Hill St.", "city": "Mountain View" }, "lastorder": { "oid": 18, "total": 338.618626f } } +{ "cid": 0, "name": "Mike ley", "cashBack": 0, "address": null, "lastorder": { "oid": 0258, "total": 368.618626f } } +{ "cid": 4, "name": "Mary Carey", "cashBack": 4, "age": 12, "address": { "number": 8, "street": "Hill St.", "city": "Mountain View" }, "lastorder": { "oid": 4545, "total": 87.618626f } } + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index 189a7e1..c15704a 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@ -64,6 +64,7 @@ import org.apache.asterix.lang.common.statement.StartFeedStatement; import org.apache.asterix.lang.common.util.FunctionUtil; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.optimizer.base.FuzzyUtils; +import org.apache.asterix.optimizer.rules.am.AbstractIntroduceAccessMethodRule; import org.apache.asterix.runtime.job.listener.JobEventListenerFactory; import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement; import org.apache.asterix.translator.IStatementExecutor.Stats; @@ -127,7 +128,7 @@ public class APIFramework { FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME, StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME, FeedActivityDetails.COLLECT_LOCATIONS, "inline_with", - "hash_merge", "output-record-type"); + "hash_merge", "output-record-type", AbstractIntroduceAccessMethodRule.NO_INDEX_ONLY_PLAN_OPTION); private final IRewriterFactory rewriterFactory; private final IAstPrintVisitorFactory astPrintVisitorFactory; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01-index-only.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01-index-only.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01-index-only.aql new file mode 100644 index 0000000..f2c4028 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01-index-only.aql @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree + * : and another for secondary btree index in index subtree. + * : In fact, this is an index-only plan from the inner branch's perspective since only PK and SK + * : variables are used and returned. + * Issue : 730, 741 + * Expected Res : Success + * Date : 8th May 2014 + */ + +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type TwitterUserType as closed { + screen-name: string, + lang: string, + friends-count: int32, + statuses-count: int32, + name: string, + followers-count: int32 +} + +create type TweetMessageType as closed { + tweetid: int64, + user: TwitterUserType, + sender-location: point, + send-time: datetime, + referred-topics: {{ string }}, + message-text: string, + countA: int32, + countB: int32 +} + +create dataset TweetMessages(TweetMessageType) +primary key tweetid; + +create index twmSndLocIx on TweetMessages(sender-location) type rtree; +create index msgCountAIx on TweetMessages(countA) type btree; +create index msgCountBIx on TweetMessages(countB) type btree; +create index msgTextIx on TweetMessages(message-text) type keyword; + +for $t1 in dataset('TweetMessages') +where $t1.tweetid < int64("10") +order by $t1.tweetid +return { +"tweetid1": $t1.tweetid, +"count1":$t1.countA, +"t2info": for $t2 in dataset('TweetMessages') + where $t1.countA /* +indexnl */= $t2.countB + order by $t2.tweetid + return {"tweetid2": $t2.tweetid, + "count2":$t2.countB} +}; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.aql index 38032f5..71e94e2 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.aql +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.aql @@ -17,7 +17,8 @@ * under the License. */ /* - * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree. + * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and + * : another for secondary btree index in index subtree. * Issue : 730, 741 * Expected Res : Success * Date : 8th May 2014 @@ -55,7 +56,7 @@ create index msgCountAIx on TweetMessages(countA) type btree; create index msgCountBIx on TweetMessages(countB) type btree; create index msgTextIx on TweetMessages(message-text) type keyword; -write output to asterix_nc1:"rttest/btree-index-join_leftouterjoin-probe-pidx-with-join-btree-sidx_01.adm"; +set noindexonly 'true'; for $t1 in dataset('TweetMessages') where $t1.tweetid < int64("10") http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/primary-to-secondary-indexonly-plan-equi-join_01.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/primary-to-secondary-indexonly-plan-equi-join_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/primary-to-secondary-indexonly-plan-equi-join_01.aql new file mode 100644 index 0000000..08e786f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/primary-to-secondary-indexonly-plan-equi-join_01.aql @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Equi joins two datasets, Customers and Orders, based on the customer id. + * Given the 'indexnl' hint we expect the join to be transformed + * into an indexed nested-loop join using Orders' secondary index. + * In fact, this is an index-only plan. + * Success : Yes + */ + +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type AddressType as open { + number: int64, + street: string, + city: string +} + +create type CustomerType as closed { + cid: int64, + name: string, + cashBack: int64, + age: int64?, + address: AddressType?, + lastorder: { + oid: int64, + total: float + } +} + +create type OrderType as open { + oid: int64, + cid: int64, + orderstatus: string, + orderpriority: string, + clerk: string, + total: float, + items: [int64] +} + +create dataset Customers(CustomerType) primary key cid; +create dataset Orders(OrderType) primary key oid; + +create index CustomerID_idx on Orders(cid); + +count( +for $c in dataset('Customers') +for $o in dataset('Orders') +where $c.cid /*+ indexnl */ = $o.cid +return {"oid": $o.oid, "cid":$c.cid} +); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_04.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_04.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_04.sqlpp new file mode 100644 index 0000000..15107b8 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_04.sqlpp @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Joins three datasets. + * : Since the given dataset in the inner branch has a secondary index on the field + * : that is being joined, we expect this join to be transformed into an indexed nested-loop join. + * Issue : ASTERIXDB-1984 + */ + +drop dataverse test if exists; +create dataverse test; +use test; + +create type TestType as +{ + id : integer, + val : integer +}; + +create dataset testdst(TestType) primary key id; +create dataset testdst2(TestType) primary key id; +create dataset testdst3(TestType) primary key id; + +create index sec3_Idx on testdst3(val) type btree; + +SELECT * FROM +testdst a JOIN testdst2 b ON a.val = b.val JOIN testdst3 c ON b.val /* +indexnl */ = c.val; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_05.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_05.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_05.sqlpp new file mode 100644 index 0000000..31fc953 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_05.sqlpp @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Joins a constant array with a dataset. + * : Since the given dataset in the inner branch has a secondary index on the field that is being joined, + * : we expect this join to be transformed into an indexed nested-loop join. + * Issue : ASTERIXDB-1984 + */ + +drop dataverse test if exists; +create dataverse test; +use test; + +create type TestType as +{ + id : integer, + val : integer +}; + +create dataset testdst(TestType) primary key id; + +create index sec_Idx on testdst (val) type btree; + +SELECT * FROM +[1, 2, 3] AS bar JOIN testdst ON bar /* +indexnl */ = testdst.val; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_06.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_06.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_06.sqlpp new file mode 100644 index 0000000..558e172 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_06.sqlpp @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Joins two datasets. + * : Since the given dataset in the inner branch has a secondary index on the field that is being joined, + * : we expect this join to be transformed into an indexed nested-loop join. + * Issue : ASTERIXDB-1984 + */ + +drop dataverse test if exists; +create dataverse test; +use test; + +create type TestType as +{ + id : integer, + val : integer +}; + +create dataset testdst(TestType) primary key id; +create dataset testdst2(TestType) primary key id; + +create index sec_Idx on testdst2(val) type btree; + +SELECT * FROM +(SELECT val, COUNT(*) FROM testdst GROUP BY val) AS bar JOIN testdst2 ON bar.val /* +indexnl */ = testdst2.val; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-primary-equi-join_01.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-primary-equi-join_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-primary-equi-join_01.aql new file mode 100644 index 0000000..d001bc4 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-primary-equi-join_01.aql @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + /* + * Description : Equi joins two datasets, Customers and Orders, based on the customer id. + * Given the 'indexnl' hint we expect the join to be transformed + * into an indexed nested-loop join using Customers' primary index. + * An index-only plan exists in the outer branch. + * Success : Yes + */ + +drop dataverse test if exists; +create dataverse test; + +use dataverse test; + +create type AddressType as open { + number: int64, + street: string, + city: string +} + +create type CustomerType as closed { + cid: int64, + name: string, + cashBack: int64, + age: int64?, + address: AddressType?, + lastorder: { + oid: int64, + total: float + } +} + +create type OrderType as open { + oid: int64, + cid: int64, + orderstatus: string, + orderpriority: string, + clerk: string, + total: float, + items: [int64] +} + +create dataset Customers(CustomerType) primary key cid; +create dataset Orders(OrderType) primary key oid; + +create index CustomerID_idx on Orders(cid); + +count( +for $o in dataset('Orders') +for $c in dataset('Customers') +where $o.cid < 800 and $o.cid /*+ indexnl */ = $c.cid +return {"oid": $o.oid, "cid":$c.cid} +); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql new file mode 100644 index 0000000..44efb8a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Equi joins two datasets, Customers and Orders, based on the customer id. + * Given the 'indexnl' hint we expect the join to be transformed + * into an indexed nested-loop join using Orders' secondary index. + * Each branch (outer and inner) will be transformed as an index-only plan. + * Success : Yes + */ + +drop dataverse test if exists; +create dataverse test; + +use dataverse test; + +create type AddressType as open { + number: int64, + street: string, + city: string +} + +create type CustomerType as closed { + cid: int64, + name: string, + cashBack: int64, + age: int64?, + address: AddressType?, + lastorder: { + oid: int64, + total: float + } +} + +create type OrderType as open { + oid: int64, + cid: int64, + orderstatus: string, + orderpriority: string, + clerk: string, + total: float, + items: [int64] +} + +create dataset Customers(CustomerType) primary key cid; +create dataset Orders(OrderType) primary key oid; + +create index CustomerID_idx on Orders(cid); +create index Cashback_idx on Customers(cashBack); + +count( +for $o in dataset('Orders') +for $c in dataset('Customers') +where $o.cid < 100000 and $o.cid /*+ indexnl */ = $c.cashBack +return {"oid": $o.oid, "cid":$c.cid} +); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-non-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-non-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-non-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql new file mode 100644 index 0000000..950e451 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-non-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Equi joins two datasets, Customers and Orders, based on the customer id. + * Given the 'indexnl' hint we expect the join to be transformed + * into an indexed nested-loop join using Orders' secondary index. + * Inner branch will be transformed as an index-only plan. + * Outer branch cannot be transformed as an index-only plan as an index can't cover + * all search predicates even excluding the join condition. + * Success : Yes + */ + +drop dataverse test if exists; +create dataverse test; + +use dataverse test; + +create type AddressType as open { + number: int64, + street: string, + city: string +} + +create type CustomerType as closed { + cid: int64, + name: string, + cashBack: int64, + age: int64?, + address: AddressType?, + lastorder: { + oid: int64, + total: float + } +} + +create type OrderType as open { + oid: int64, + cid: int64, + orderstatus: string, + orderpriority: string, + clerk: string, + total: float, + items: [int64] +} + +create dataset Customers(CustomerType) primary key cid; +create dataset Orders(OrderType) primary key oid; + +create index CustomerID_idx on Orders(cid); +create index Cashback_idx on Customers(cashBack); + +count( +for $o in dataset('Orders') +for $c in dataset('Customers') +where $o.cid < 100000 and $o.total >= 0 and $o.cid /*+ indexnl */ = $c.cashBack +return {"oid": $o.oid, "cid":$c.cid} +); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join-index-only.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join-index-only.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join-index-only.aql new file mode 100644 index 0000000..e71101a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join-index-only.aql @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Self-equi joins on a dataset using two fields - countA and countB. + * TweetMessages has a secondary btree index on countB, and given the 'indexnl' hint + * we expect the join to be transformed into an indexed nested-loop join. + * In fact, this is an index-only plan from the inner branch's perspective since only PK and SK + * variables are used and returned. + * Success : Yes + */ + +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type TwitterUserType as closed { + screen-name: string, + lang: string, + friends-count: int64, + statuses-count: int64, + name: string, + followers-count: int64 +} + +create type TweetMessageType as closed { + tweetid: int64, + user: TwitterUserType, + sender-location: point, + send-time: datetime, + referred-topics: {{ string }}, + message-text: string, + countA: int64, + countB: int64 +} + +create dataset TweetMessages(TweetMessageType) +primary key tweetid; + +create index twmSndLocIx on TweetMessages(sender-location) type rtree; +create index msgCountAIx on TweetMessages(countA) type btree; +create index msgCountBIx on TweetMessages(countB) type btree; +create index msgTextIx on TweetMessages(message-text) type keyword; + +for $t1 in dataset('TweetMessages') +for $t2 in dataset('TweetMessages') +let $c := $t1.countA + 20 +where $c /* +indexnl */= $t2.countB +order by $t2.tweetid +return {"tweetid2": $t2.tweetid, "count2":$t2.countB}; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join.aql index abfd197..74f2d87 100644 --- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join.aql +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join.aql @@ -55,7 +55,7 @@ create index msgCountAIx on TweetMessages(countA) type btree; create index msgCountBIx on TweetMessages(countB) type btree; create index msgTextIx on TweetMessages(message-text) type keyword; -write output to asterix_nc1:"rttest/btree-index-join_self-secondary-equi-join.adm"; +set noindexonly 'true'; for $t1 in dataset('TweetMessages') for $t2 in dataset('TweetMessages') http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-composite-index-indexonly-plan-01.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-composite-index-indexonly-plan-01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-composite-index-indexonly-plan-01.aql new file mode 100644 index 0000000..2599bd1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-composite-index-indexonly-plan-01.aql @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Secondary BTree Index index-only selection plan verification test + * : This test is intended to verify that the secondary BTree index is + * : used in the optimized query plan. + * : In this plan, we fetch PK and SK based on a select condition that utilizes a secondary index. + * : The plan should have two paths after the secondary index-lookup. + * : The left path: + * ... -> unnest-map (sidx) -> split -> unnest-map (pidx) -> select -> union -> ... + * : The right path: + * ... -> unnest-map (sidx) -> split -> union -> ... + * Expected Result : Success + * +*/ + +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type MyRecord as closed { + id: int64, + docid: int64, + val1: int64, + title: string, + point: point, + kwds: string, + line1: line, + line2: line, + poly1: polygon, + poly2: polygon, + rec: rectangle, + circle: circle +} + +create dataset MyData(MyRecord) + primary key id; + +create index btree_index_docid_val1 on MyData(docid,val1) type btree; +create index rtree_index_point on MyData(point) type rtree; +create index rtree_index_rec on MyData(rec) type rtree; +create index ngram_index_title on MyData(title) type ngram(3); +create index keyword_index_title on MyData(title) type keyword; + +for $o in dataset('MyData') +where $o.docid < 3 +return {"pk":$o.id, "sk":$o.val1} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01-disable-indexonly-plan.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01-disable-indexonly-plan.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01-disable-indexonly-plan.sqlpp new file mode 100644 index 0000000..0be684e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01-disable-indexonly-plan.sqlpp @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Secondary BTree Index index-only selection plan verification test + * : The test is intended to verify that the secondary BTree index is used in the optimized query plan. + * : In this plan, we fetch PK and SK based on a select condition that utilizes a secondary index. + * : The plan should have two paths after the secondary index-lookup. + * : The left path: + * ... -> unnest-map (sidx) -> split -> unnest-map (pidx) -> select -> union -> ... + * : The right path: + * ... -> unnest-map (sidx) -> split -> -> union -> ... + * : However, we set the "noindexonly" option to true. So, the index-only plan should not be triggered. + * Expected Result : Success + * +*/ + +drop dataverse test if exists; +create dataverse test; +use test; + +create type MyRecord as closed { + id: int64, + docid: int64, + val1: int64, + title: string, + point: point, + kwds: string, + line1: line, + line2: line, + poly1: polygon, + poly2: polygon, + rec: rectangle, + circle: circle +}; + +create dataset MyData(MyRecord) primary key id; + +create index btree_index_docid on MyData(docid) type btree; +create index btree_index_val1 on MyData(val1) type btree; +create index rtree_index_point on MyData(point) type rtree; +create index rtree_index_rec on MyData(rec) type rtree; +create index ngram_index_title on MyData(title) type ngram(3); +create index keyword_index_title on MyData(title) type keyword; + +set noindexonly 'true'; + +select element {"pk":o.id, "sk":o.docid} +from MyData o +where o.docid < 3 +order by o.id; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01.sqlpp new file mode 100644 index 0000000..af1a099 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01.sqlpp @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Secondary BTree Index index-only selection plan verification test + * : The test is intended to verify that the secondary BTree index is used in the optimized query plan. + * : In this plan, we fetch PK and SK based on a select condition that utilizes a secondary index. + * : The plan should have two paths after the secondary index-lookup. + * : The left path: + * ... -> unnest-map (sidx) -> split -> unnest-map (pidx) -> select -> union -> ... + * : The right path: + * ... -> unnest-map (sidx) -> split -> -> union -> ... + * Expected Result : Success + * +*/ + +drop dataverse test if exists; +create dataverse test; +use test; + +create type MyRecord as closed { + id: int64, + docid: int64, + val1: int64, + title: string, + point: point, + kwds: string, + line1: line, + line2: line, + poly1: polygon, + poly2: polygon, + rec: rectangle, + circle: circle +}; + +create dataset MyData(MyRecord) primary key id; + +create index btree_index_docid on MyData(docid) type btree; +create index btree_index_val1 on MyData(val1) type btree; +create index rtree_index_point on MyData(point) type rtree; +create index rtree_index_rec on MyData(rec) type rtree; +create index ngram_index_title on MyData(title) type ngram(3); +create index keyword_index_title on MyData(title) type keyword; + +select element {"pk":o.id, "sk":o.docid} +from MyData o +where o.docid < 3 +order by o.id;