Yingyi Bu has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/685
Change subject: Supprt querying meta record with meta().
......................................................................
Supprt querying meta record with meta().
Change-Id: Ie65417b6baf209ca0ab413cfa4a5f7fc5156ca63
---
M
asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
A
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
M
asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
M
asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
M
asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
M
asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
A
asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
A
asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.2.update.aql
A
asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.3.query.aql
A
asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.4.ddl.aql
A
asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.ddl.aql
A
asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.2.update.aql
A
asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.3.query.aql
A
asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.4.ddl.aql
A
asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.adm
A
asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.adm
M asterix-app/src/test/resources/runtimets/testsuite.xml
M
asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
M
asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
M
asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
M
asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
M
asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
A
asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/OptionalOpenARecordTypeComputer.java
29 files changed, 584 insertions(+), 68 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/85/685/1
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
index 4c0818b..1f5b47e 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
@@ -114,7 +114,8 @@
public void computeDeliveredProperties(ILogicalOperator op,
IOptimizationContext context)
throws AlgebricksException {
AqlDataSource ds = new DatasetDataSource(datasetId,
datasetId.getDataverseName(), datasetId.getDatasourceName(),
- recordType, null /*external dataset doesn't have meta
records.*/, AqlDataSourceType.EXTERNAL_DATASET);
+ recordType, null /*external dataset doesn't have meta
records.*/, AqlDataSourceType.EXTERNAL_DATASET,
+ dataset.getDatasetDetails());
IDataSourcePropertiesProvider dspp = ds.getPropertiesProvider();
AbstractScanOperator as = (AbstractScanOperator) op;
deliveredProperties = dspp.computePropertiesVector(as.getVariables());
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index ad5283a..8627b97 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -50,6 +50,7 @@
import org.apache.asterix.optimizer.rules.IntroduceStaticTypeCastForInsertRule;
import
org.apache.asterix.optimizer.rules.IntroduceUnnestForCollectionToSequenceRule;
import org.apache.asterix.optimizer.rules.LoadRecordFieldsRule;
+import org.apache.asterix.optimizer.rules.MetaFunctionToMetaVariableRule;
import org.apache.asterix.optimizer.rules.NestGroupByRule;
import
org.apache.asterix.optimizer.rules.PushAggFuncIntoStandaloneAggregateRule;
import org.apache.asterix.optimizer.rules.PushAggregateIntoGroupbyRule;
@@ -167,6 +168,7 @@
normalization.add(new ConstantFoldingRule());
normalization.add(new RemoveRedundantSelectRule());
normalization.add(new UnnestToDataScanRule());
+ normalization.add(new MetaFunctionToMetaVariableRule());
normalization.add(new IfElseToSwitchCaseFunctionRule());
normalization.add(new FuzzyEqRule());
normalization.add(new SimilarityCheckRule());
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index ebe01a7..c534454 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -90,7 +90,7 @@
AsterixBuiltinFunctions.GET_RECORD_FIELD_VALUE,
AsterixBuiltinFunctions.FIELD_ACCESS_NESTED,
AsterixBuiltinFunctions.GET_ITEM,
AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR,
AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX,
AsterixBuiltinFunctions.CAST_RECORD,
- AsterixBuiltinFunctions.CAST_LIST);
+ AsterixBuiltinFunctions.CAST_LIST, AsterixBuiltinFunctions.META);
/** Throws exceptions in substituiteProducedVariable, setVarType, and one
getVarType method. */
private static final IVariableTypeEnvironment _emptyTypeEnv = new
IVariableTypeEnvironment() {
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
new file mode 100644
index 0000000..5e57b9c
--- /dev/null
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.metadata.declared.AqlDataSource;
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import
org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import
org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class MetaFunctionToMetaVariableRule implements IAlgebraicRewriteRule {
+ // The rule can only apply once.
+ private boolean hasApplied = false;
+ private boolean rewritten = false;
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef,
IOptimizationContext context)
+ throws AlgebricksException {
+ if (hasApplied) {
+ return false;
+ }
+ hasApplied = true;
+ visit(opRef);
+ return rewritten;
+ }
+
+ private ILogicalExpressionReferenceTransform
visit(Mutable<ILogicalOperator> opRef) throws AlgebricksException {
+ ILogicalOperator op = opRef.getValue();
+
+ // Reaches NTS or ETS.
+ if (op.getInputs().size() == 0) {
+ return NoOpExpressionReferenceTransform.INSTANCE;
+ }
+
+ // Datascan returns an useful transform if the meta part presents in
the dataset.
+ if (op.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+ DataSourceScanOperator scanOp = (DataSourceScanOperator) op;
+ AqlDataSource dataSource = (AqlDataSource) scanOp.getDataSource();
+ if (!dataSource.hasMeta()) {
+ return visit(op.getInputs().get(0));
+ };
+ List<LogicalVariable> allVars = scanOp.getVariables();
+ LogicalVariable dataVar = allVars.get(allVars.size() - 2);
+ LogicalVariable metaVar = allVars.get(allVars.size() - 1);
+ return new LogicalExpressionReferenceTransform(dataVar, metaVar);
+ }
+
+ // Visits children in the depth-first order.
+ List<ILogicalExpressionReferenceTransform> transformers = new
ArrayList<>();
+ for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
+ ILogicalExpressionReferenceTransform transformer = visit(childRef);
+ if
(!transformer.equals(NoOpExpressionReferenceTransform.INSTANCE)) {
+ transformers.add(transformer);
+ }
+ }
+ ILogicalExpressionReferenceTransform currentTransformer = null;
+ if (transformers.size() == 0) {
+ currentTransformer = NoOpExpressionReferenceTransform.INSTANCE;
+ } else if (transformers.size() == 1) {
+ currentTransformer = transformers.get(0);
+ } else {
+ for (ILogicalExpressionReferenceTransform transformer :
transformers) {
+ LogicalExpressionReferenceTransform logicalTransformer =
(LogicalExpressionReferenceTransform) transformer;
+ logicalTransformer.setVariableRequired();
+ }
+ currentTransformer = new
CompositeExpressionReferenceTransform(transformers);
+ }
+ rewritten |= op.acceptExpressionTransform(currentTransformer);
+ return currentTransformer;
+ }
+}
+
+class NoOpExpressionReferenceTransform implements
ILogicalExpressionReferenceTransform {
+ static NoOpExpressionReferenceTransform INSTANCE = new
NoOpExpressionReferenceTransform();
+
+ private NoOpExpressionReferenceTransform() {
+
+ }
+
+ @Override
+ public boolean transform(Mutable<ILogicalExpression> expression) throws
AlgebricksException {
+ return false;
+ }
+}
+
+class LogicalExpressionReferenceTransform implements
ILogicalExpressionReferenceTransform {
+ private final LogicalVariable dataVar;
+ private final LogicalVariable metaVar;
+ private boolean variableRequired = false;
+
+ LogicalExpressionReferenceTransform(LogicalVariable dataVar,
LogicalVariable metaVar) {
+ this.dataVar = dataVar;
+ this.metaVar = metaVar;
+ }
+
+ void setVariableRequired() {
+ this.variableRequired = true;
+ }
+
+ @Override
+ public boolean transform(Mutable<ILogicalExpression> exprRef) throws
AlgebricksException {
+ ILogicalExpression expr = exprRef.getValue();
+ if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+ AbstractFunctionCallExpression funcExpr =
(AbstractFunctionCallExpression) expr;
+ List<Mutable<ILogicalExpression>> argRefs = funcExpr.getArguments();
+
+ // Recursively transform argument expressions.
+ for (Mutable<ILogicalExpression> argRef : argRefs) {
+ transform(argRef);
+ }
+
+ if
(!funcExpr.getFunctionIdentifier().equals(AsterixBuiltinFunctions.META)) {
+ return false;
+ }
+ // The user query provides more than one parameter for the meta
function.
+ if (argRefs.size() > 1) {
+ throw new AlgebricksException("The meta function can at most have
one argument!");
+ }
+
+ // The user query provides exact one parameter for the meta function.
+ if (argRefs.size() == 1) {
+ ILogicalExpression argExpr = argRefs.get(0).getValue();
+ if (argExpr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ return false;
+ }
+ VariableReferenceExpression argVarExpr =
(VariableReferenceExpression) argExpr;
+ LogicalVariable argVar = argVarExpr.getVariableReference();
+ if (!dataVar.equals(argVar)) {
+ return false;
+ }
+ exprRef.setValue(new VariableReferenceExpression(metaVar));
+ return true;
+ }
+
+ // The user query provides zero parameter for the meta function.
+ if (variableRequired) {
+ throw new AlgebricksException("Cannot resolve to ambiguity on the
meta function call --"
+ + " there are more than once dataset choices!");
+ }
+ exprRef.setValue(new VariableReferenceExpression(metaVar));
+ return true;
+ }
+}
+
+class CompositeExpressionReferenceTransform implements
ILogicalExpressionReferenceTransform {
+ private final List<ILogicalExpressionReferenceTransform> transformers;
+
+ public
CompositeExpressionReferenceTransform(List<ILogicalExpressionReferenceTransform>
transforms) {
+ this.transformers = transforms;
+ }
+
+ @Override
+ public boolean transform(Mutable<ILogicalExpression> expression) throws
AlgebricksException {
+ // Tries transfomations one by one.
+ for (ILogicalExpressionReferenceTransform transformer : transformers) {
+ if (transformer.transform(expression)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+}
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index f39d190..5cb01eb 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -35,6 +35,7 @@
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.om.base.ABoolean;
import org.apache.asterix.om.base.AInt32;
@@ -82,14 +83,25 @@
*/
public class AccessMethodUtils {
- public static void appendPrimaryIndexTypes(Dataset dataset, IAType
itemType, List<Object> target)
- throws IOException, AlgebricksException {
+ public static void appendPrimaryIndexTypes(Dataset dataset, IAType
itemType, IAType metaItemType,
+ List<Object> target) throws IOException, AlgebricksException {
ARecordType recordType = (ARecordType) itemType;
+ ARecordType metaRecordType = (ARecordType) metaItemType;
+ List<Integer> keySourceIndicators = ((InternalDatasetDetails)
dataset.getDatasetDetails())
+ .getKeySourceIndicator();
List<List<String>> partitioningKeys =
DatasetUtils.getPartitioningKeys(dataset);
+ int index = 0;
for (List<String> partitioningKey : partitioningKeys) {
- target.add(recordType.getSubFieldType(partitioningKey));
+ target.add(keySourceIndicators.get(index) == 0 ?
recordType.getSubFieldType(partitioningKey)
+ : metaRecordType.getSubFieldType(partitioningKey));
+ ++index;
}
+ // Adds data record type.
target.add(itemType);
+ // Adds metea record type if any.
+ if (dataset.hasMetaPart()) {
+ target.add(metaItemType);
+ }
}
public static ConstantExpression createStringConstant(String str) {
@@ -172,8 +184,9 @@
constantExpressionType);
for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
//avoid additional optFuncExpressions in case of a join
- if (optFuncExpr.getFuncExpr().equals(funcExpr))
+ if (optFuncExpr.getFuncExpr().equals(funcExpr)) {
return true;
+ }
}
analysisCtx.matchedFuncExprs.add(newOptFuncExpr);
return true;
@@ -196,8 +209,9 @@
new LogicalVariable[] { fieldVar1, fieldVar2 }, new
ILogicalExpression[0], new IAType[0]);
for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
//avoid additional optFuncExpressions in case of a join
- if (optFuncExpr.getFuncExpr().equals(funcExpr))
+ if (optFuncExpr.getFuncExpr().equals(funcExpr)) {
return true;
+ }
}
analysisCtx.matchedFuncExprs.add(newOptFuncExpr);
return true;
@@ -228,8 +242,9 @@
/**
* Appends the types of the fields produced by the given secondary index
to dest.
*/
- public static void appendSecondaryIndexTypes(Dataset dataset, ARecordType
recordType, Index index,
- boolean primaryKeysOnly, List<Object> dest) throws
AlgebricksException {
+ public static void appendSecondaryIndexTypes(Dataset dataset, ARecordType
recordType, ARecordType metaRecordType,
+ Index index, boolean primaryKeysOnly, List<Object> dest) throws
AlgebricksException {
+ List<Integer> keySourceIndicators =
index.getKeyFieldSourceIndicators();
if (!primaryKeysOnly) {
switch (index.getIndexType()) {
case BTREE:
@@ -237,17 +252,19 @@
case SINGLE_PARTITION_NGRAM_INVIX: {
for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
Pair<IAType, Boolean> keyPairType =
Index.getNonNullableOpenFieldType(
- index.getKeyFieldTypes().get(i),
index.getKeyFieldNames().get(i), recordType);
+ index.getKeyFieldTypes().get(i),
index.getKeyFieldNames().get(i),
+ keySourceIndicators.get(i) == 0 ? recordType :
metaRecordType);
dest.add(keyPairType.first);
}
break;
}
case RTREE: {
+ ARecordType targetRecType = keySourceIndicators.get(0) ==
0 ? recordType : metaRecordType;
Pair<IAType, Boolean> keyPairType =
Index.getNonNullableOpenFieldType(
- index.getKeyFieldTypes().get(0),
index.getKeyFieldNames().get(0), recordType);
+ index.getKeyFieldTypes().get(0),
index.getKeyFieldNames().get(0), targetRecType);
IAType keyType = keyPairType.first;
IAType nestedKeyType =
NonTaggedFormatUtil.getNestedSpatialType(keyType.getTypeTag());
- int numKeys = getNumSecondaryKeys(index, recordType);
+ int numKeys = getNumSecondaryKeys(index, targetRecType);
for (int i = 0; i < numKeys; i++) {
dest.add(nestedKeyType);
}
@@ -269,12 +286,18 @@
}
} else {
List<List<String>> partitioningKeys =
DatasetUtils.getPartitioningKeys(dataset);
+ List<Integer> partitioningKeySourceIndicators =
((InternalDatasetDetails) dataset.getDatasetDetails())
+ .getKeySourceIndicator();
+ int keyIndex = 0;
for (List<String> partitioningKey : partitioningKeys) {
try {
- dest.add(recordType.getSubFieldType(partitioningKey));
+ dest.add(partitioningKeySourceIndicators.get(keyIndex) == 0
+ ? recordType.getSubFieldType(partitioningKey)
+ : metaRecordType.getSubFieldType(partitioningKey));
} catch (IOException e) {
throw new AlgebricksException(e);
}
+ ++keyIndex;
}
}
}
@@ -422,9 +445,10 @@
return indexExprs.get(0).second;
}
- public static UnnestMapOperator createSecondaryIndexUnnestMap(Dataset
dataset, ARecordType recordType, Index index,
- ILogicalOperator inputOp, AccessMethodJobGenParams jobGenParams,
IOptimizationContext context,
- boolean outputPrimaryKeysOnly, boolean retainInput) throws
AlgebricksException {
+ public static UnnestMapOperator createSecondaryIndexUnnestMap(Dataset
dataset, ARecordType recordType,
+ ARecordType metaRecordType, Index index, ILogicalOperator inputOp,
AccessMethodJobGenParams jobGenParams,
+ IOptimizationContext context, boolean outputPrimaryKeysOnly,
boolean retainInput)
+ throws AlgebricksException {
// The job gen parameters are transferred to the actual job gen via
the UnnestMapOperator's function arguments.
ArrayList<Mutable<ILogicalExpression>> secondaryIndexFuncArgs = new
ArrayList<Mutable<ILogicalExpression>>();
jobGenParams.writeToFuncArgs(secondaryIndexFuncArgs);
@@ -434,7 +458,8 @@
// Append output variables/types generated by the secondary-index
search (not forwarded from input).
appendSecondaryIndexOutputVars(dataset, recordType, index,
outputPrimaryKeysOnly, context,
secondaryIndexUnnestVars);
- appendSecondaryIndexTypes(dataset, recordType, index,
outputPrimaryKeysOnly, secondaryIndexOutputTypes);
+ appendSecondaryIndexTypes(dataset, recordType, metaRecordType, index,
outputPrimaryKeysOnly,
+ secondaryIndexOutputTypes);
// An index search is expressed as an unnest over an index-search
function.
IFunctionInfo secondaryIndexSearch =
FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.INDEX_SEARCH);
UnnestingFunctionCallExpression secondaryIndexSearchFunc = new
UnnestingFunctionCallExpression(
@@ -452,9 +477,9 @@
}
public static UnnestMapOperator
createPrimaryIndexUnnestMap(AbstractDataSourceOperator dataSourceOp,
- Dataset dataset, ARecordType recordType, ILogicalOperator inputOp,
IOptimizationContext context,
- boolean sortPrimaryKeys, boolean retainInput, boolean retainNull,
boolean requiresBroadcast)
- throws AlgebricksException {
+ Dataset dataset, ARecordType recordType, ARecordType
metaRecordType, ILogicalOperator inputOp,
+ IOptimizationContext context, boolean sortPrimaryKeys, boolean
retainInput, boolean retainNull,
+ boolean requiresBroadcast) throws AlgebricksException {
List<LogicalVariable> primaryKeyVars =
AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset,
inputOp);
// Optionally add a sort on the primary-index keys before searching
the primary index.
@@ -489,7 +514,7 @@
// Append output variables/types generated by the primary-index search
(not forwarded from input).
primaryIndexUnnestVars.addAll(dataSourceOp.getVariables());
try {
- appendPrimaryIndexTypes(dataset, recordType,
primaryIndexOutputTypes);
+ appendPrimaryIndexTypes(dataset, recordType, metaRecordType,
primaryIndexOutputTypes);
} catch (IOException e) {
throw new AlgebricksException(e);
}
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index 1bbb13b..9825b9d 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -481,7 +481,7 @@
}
UnnestMapOperator secondaryIndexUnnestOp =
AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
- chosenIndex, inputOp, jobGenParams, context, false,
retainInput);
+ metaRecordType, chosenIndex, inputOp, jobGenParams, context,
false, retainInput);
// Generate the rest of the upstream plan which feeds the search
results into the primary index.
UnnestMapOperator primaryIndexUnnestOp = null;
@@ -494,7 +494,7 @@
return externalDataAccessOp;
} else if (!isPrimaryIndex) {
primaryIndexUnnestOp =
AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType,
- secondaryIndexUnnestOp, context, true, retainInput,
retainNull, false);
+ metaRecordType, secondaryIndexUnnestOp, context, true,
retainInput, retainNull, false);
// Adds equivalence classes --- one equivalent class between a
primary key
// variable and a record field-access expression.
@@ -503,7 +503,7 @@
} else {
List<Object> primaryIndexOutputTypes = new ArrayList<Object>();
try {
- AccessMethodUtils.appendPrimaryIndexTypes(dataset, recordType,
primaryIndexOutputTypes);
+ AccessMethodUtils.appendPrimaryIndexTypes(dataset, recordType,
metaRecordType, primaryIndexOutputTypes);
} catch (IOException e) {
throw new AlgebricksException(e);
}
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
index 824fe7a..70d7088 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
@@ -66,7 +66,6 @@
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import
org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
import
org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
@@ -364,13 +363,13 @@
@Override
public ILogicalOperator
createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef,
OptimizableOperatorSubTree indexSubTree,
OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
- AccessMethodAnalysisContext analysisCtx,
- boolean retainInput, boolean retainNull, boolean
requiresBroadcast, IOptimizationContext context)
- throws AlgebricksException {
+ AccessMethodAnalysisContext analysisCtx, boolean retainInput,
boolean retainNull, boolean requiresBroadcast,
+ IOptimizationContext context) throws AlgebricksException {
IOptimizableFuncExpr optFuncExpr =
AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
Dataset dataset = indexSubTree.dataset;
ARecordType recordType = indexSubTree.recordType;
+ ARecordType metaRecordType = indexSubTree.metaRecordType;
// we made sure indexSubTree has datasource scan
DataSourceScanOperator dataSourceScan = (DataSourceScanOperator)
indexSubTree.dataSourceRef.getValue();
@@ -407,11 +406,11 @@
}
jobGenParams.setKeyVarList(keyVarList);
UnnestMapOperator secondaryIndexUnnestOp =
AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
- chosenIndex, inputOp, jobGenParams, context, true,
retainInput);
+ metaRecordType, chosenIndex, inputOp, jobGenParams, context,
true, retainInput);
// Generate the rest of the upstream plan which feeds the search
results into the primary index.
UnnestMapOperator primaryIndexUnnestOp =
AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceScan, dataset,
- recordType, secondaryIndexUnnestOp, context, true,
retainInput, retainNull, false);
+ recordType, metaRecordType, secondaryIndexUnnestOp, context,
true, retainInput, retainNull, false);
return primaryIndexUnnestOp;
}
@@ -436,8 +435,8 @@
public boolean applySelectPlanTransformation(Mutable<ILogicalOperator>
selectRef,
OptimizableOperatorSubTree subTree, Index chosenIndex,
AccessMethodAnalysisContext analysisCtx,
IOptimizationContext context) throws AlgebricksException {
- ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null,
subTree, null, chosenIndex, analysisCtx, false,
- false, false, context);
+ ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null,
subTree, null, chosenIndex, analysisCtx,
+ false, false, false, context);
// Replace the datasource scan with the new plan rooted at
primaryIndexUnnestMap.
subTree.dataSourceRef.setValue(indexPlanRootOp);
return true;
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
index 3c925e2..111fcf4 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -179,6 +179,7 @@
IOptimizableFuncExpr optFuncExpr =
AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
Dataset dataset = indexSubTree.dataset;
ARecordType recordType = indexSubTree.recordType;
+ ARecordType metaRecordType = indexSubTree.metaRecordType;
int optFieldIdx = AccessMethodUtils.chooseFirstOptFuncVar(chosenIndex,
analysisCtx);
Pair<IAType, Boolean> keyPairType =
Index.getNonNullableOpenFieldType(optFuncExpr.getFieldType(optFieldIdx),
@@ -239,7 +240,7 @@
}
UnnestMapOperator secondaryIndexUnnestOp =
AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
- chosenIndex, assignSearchKeys, jobGenParams, context, false,
retainInput);
+ metaRecordType, chosenIndex, assignSearchKeys, jobGenParams,
context, false, retainInput);
// Generate the rest of the upstream plan which feeds the search
results into the primary index.
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
@@ -248,8 +249,8 @@
return externalDataAccessOp;
} else {
UnnestMapOperator primaryIndexUnnestOp =
AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp,
- dataset, recordType, secondaryIndexUnnestOp, context,
true, retainInput, false, false);
-
+ dataset, recordType, metaRecordType,
secondaryIndexUnnestOp, context, true, retainInput, false,
+ false);
return primaryIndexUnnestOp;
}
}
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
index e85fecf..242e15b 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
@@ -103,7 +103,7 @@
for (int pkIndex = 0; pkIndex < primaryKey.size(); ++pkIndex) {
LogicalVariable referredRecordVar = recordVar;
String pkFieldName = primaryKey.get(pkIndex).get(0);
- int source = keySourceIndicators == null ? 0 :
keySourceIndicators.get(pkIndex);
+ int source = keySourceIndicators.get(pkIndex);
Integer fieldIndexInRecord;
if (source == 0) {
// The field is from the main record.
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index c635125..28de56b 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -457,8 +457,8 @@
IAType metaItemType =
metadataProvider.findType(dataset.getMetaItemTypeDataverseName(),
dataset.getMetaItemTypeName());
DatasetDataSource dataSource = new DatasetDataSource(sourceId,
dataset.getDataverseName(),
- dataset.getDatasetName(), itemType, metaItemType,
AqlDataSourceType.INTERNAL_DATASET);
-
+ dataset.getDatasetName(), itemType, metaItemType,
AqlDataSourceType.INTERNAL_DATASET,
+ dataset.getDatasetDetails());
return dataSource;
}
diff --git
a/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
b/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
index 95a8802..9020e27 100644
---
a/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
+++
b/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
@@ -151,12 +151,14 @@
* @throws AsterixException
* (if the validation failed), IOException
*/
- public static void validateKeyFields(ARecordType recType,
List<List<String>> keyFieldNames,
- List<IAType> keyFieldTypes, IndexType indexType) throws
AsterixException, IOException {
+ public static void validateKeyFields(ARecordType recType, ARecordType
metaRecType, List<List<String>> keyFieldNames,
+ List<Integer> keySourceIndicators, List<IAType> keyFieldTypes,
IndexType indexType)
+ throws AsterixException, IOException {
int pos = 0;
boolean openFieldCompositeIdx = false;
for (List<String> fieldName : keyFieldNames) {
- IAType fieldType = recType.getSubFieldType(fieldName);
+ IAType fieldType = keySourceIndicators.get(pos) == 0 ?
recType.getSubFieldType(fieldName)
+ : metaRecType.getSubFieldType(fieldName);
if (fieldType == null) {
fieldType = keyFieldTypes.get(pos);
if (keyFieldTypes.get(pos) == BuiltinType.ANULL) {
diff --git
a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 75f325b..d2a0d03 100644
---
a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++
b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -811,6 +811,7 @@
CreateIndexStatement stmtCreateIndex = (CreateIndexStatement) stmt;
String dataverseName =
getActiveDataverse(stmtCreateIndex.getDataverseName());
String datasetName = stmtCreateIndex.getDatasetName().getValue();
+ List<Integer> keySourceIndicators =
stmtCreateIndex.getFieldSourceIndicators();
MetadataTransactionContext mdTxnCtx =
MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
@@ -838,19 +839,23 @@
indexName = stmtCreateIndex.getIndexName().getValue();
Index idx =
MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(),
dataverseName,
datasetName, indexName);
-
- String itemTypeName = ds.getItemTypeName();
Datatype dt =
MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
- ds.getItemTypeDataverseName(), itemTypeName);
- IAType itemType = dt.getDatatype();
- ARecordType aRecordType = (ARecordType) itemType;
+ ds.getItemTypeDataverseName(), ds.getItemTypeName());
+ ARecordType aRecordType = (ARecordType) dt.getDatatype();
+ ARecordType metaRecordType = null;
+ if (ds.hasMetaPart()) {
+ Datatype metaDt =
MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
+ ds.getMetaItemTypeDataverseName(),
ds.getMetaItemTypeName());
+ metaRecordType = (ARecordType) metaDt.getDatatype();
+ }
List<List<String>> indexFields = new ArrayList<List<String>>();
List<IAType> indexFieldTypes = new ArrayList<IAType>();
+ int keyIndex = 0;
for (Pair<List<String>, TypeExpression> fieldExpr :
stmtCreateIndex.getFieldExprs()) {
IAType fieldType = null;
- boolean isOpen = aRecordType.isOpen();
- ARecordType subType = aRecordType;
+ ARecordType subType = keySourceIndicators.get(keyIndex) == 0 ?
aRecordType : metaRecordType;
+ boolean isOpen = subType.isOpen();
int i = 0;
if (fieldExpr.first.size() > 1 && !isOpen) {
for (; i < fieldExpr.first.size() - 1;) {
@@ -885,9 +890,11 @@
indexFields.add(fieldExpr.first);
indexFieldTypes.add(fieldType);
+ ++keyIndex;
}
- ValidateUtil.validateKeyFields(aRecordType, indexFields,
indexFieldTypes, stmtCreateIndex.getIndexType());
+ ValidateUtil.validateKeyFields(aRecordType, metaRecordType,
indexFields, keySourceIndicators,
+ indexFieldTypes, stmtCreateIndex.getIndexType());
if (idx != null) {
if (stmtCreateIndex.getIfNotExists()) {
@@ -995,8 +1002,8 @@
//#. add a new index with PendingAddOp
Index index = new Index(dataverseName, datasetName, indexName,
stmtCreateIndex.getIndexType(), indexFields,
- stmtCreateIndex.getFieldSourceIndicators(),
indexFieldTypes, stmtCreateIndex.getGramLength(),
- stmtCreateIndex.isEnforced(), false,
IMetadataEntity.PENDING_ADD_OP);
+ keySourceIndicators, indexFieldTypes,
stmtCreateIndex.getGramLength(), stmtCreateIndex.isEnforced(),
+ false, IMetadataEntity.PENDING_ADD_OP);
MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(),
index);
ARecordType enforcedType = null;
diff --git
a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
new file mode 100644
index 0000000..a341b29
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type EmptyType as open {
+ "key":int32
+}
+
+create type LineType as open {
+ id:int32,
+ text: string
+}
+
+create dataset Book(LineType) with meta(EmptyType)
+primary key meta()."key";
diff --git
a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.2.update.aql
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.2.update.aql
new file mode 100644
index 0000000..bd244d0
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.2.update.aql
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
diff --git
a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.3.query.aql
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.3.query.aql
new file mode 100644
index 0000000..246046d
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.3.query.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse test;
+
+for $x in dataset Book
+where meta($x)."key" >10
+return $x;
+
diff --git
a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.4.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.4.ddl.aql
new file mode 100644
index 0000000..dc10acd
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.4.ddl.aql
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
diff --git
a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.ddl.aql
new file mode 100644
index 0000000..a341b29
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.ddl.aql
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type EmptyType as open {
+ "key":int32
+}
+
+create type LineType as open {
+ id:int32,
+ text: string
+}
+
+create dataset Book(LineType) with meta(EmptyType)
+primary key meta()."key";
diff --git
a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.2.update.aql
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.2.update.aql
new file mode 100644
index 0000000..bd244d0
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.2.update.aql
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
diff --git
a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.3.query.aql
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.3.query.aql
new file mode 100644
index 0000000..6b8e4a9
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.3.query.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse test;
+
+for $x in dataset Book
+where meta()."key" >10
+return $x;
+
diff --git
a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.4.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.4.ddl.aql
new file mode 100644
index 0000000..dc10acd
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.4.ddl.aql
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
diff --git
a/asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.adm
b/asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.adm
diff --git
a/asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.adm
b/asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++
b/asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.adm
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml
b/asterix-app/src/test/resources/runtimets/testsuite.xml
index 05f131e..fd4f32f 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -2916,6 +2916,16 @@
<output-dir
compare="Text">query_dataset_with_meta-2</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="meta">
+ <compilation-unit name="query_dataset_with_meta_primary_index-1">
+ <output-dir
compare="Text">query_dataset_with_meta_primary_index-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="meta">
+ <compilation-unit name="query_dataset_with_meta_primary_index-2">
+ <output-dir
compare="Text">query_dataset_with_meta_primary_index-2</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="misc">
<test-case FilePath="misc">
diff --git
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
index 7ce7ba3..c7c1d1e 100644
---
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
+++
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
@@ -989,7 +989,7 @@
AqlDataSourceType datasourceType =
dataset.getDatasetType().equals(DatasetType.EXTERNAL)
? AqlDataSourceType.EXTERNAL_DATASET :
AqlDataSourceType.INTERNAL_DATASET;
return new DatasetDataSource(aqlId, aqlId.getDataverseName(),
aqlId.getDatasourceName(), itemType, metaItemType,
- datasourceType);
+ datasourceType, dataset.getDatasetDetails());
}
@Override
diff --git
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index 8b5956f..8e63478 100644
---
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -21,9 +21,11 @@
import java.io.IOException;
import java.util.List;
+import org.apache.asterix.metadata.IDatasetDetails;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
@@ -36,7 +38,8 @@
private Dataset dataset;
public DatasetDataSource(AqlSourceId id, String datasourceDataverse,
String datasourceName, IAType itemType,
- IAType metaItemType, AqlDataSourceType datasourceType) throws
AlgebricksException {
+ IAType metaItemType, AqlDataSourceType datasourceType,
IDatasetDetails datasetDetails)
+ throws AlgebricksException {
super(id, itemType, metaItemType, datasourceType);
MetadataTransactionContext ctx = null;
try {
@@ -49,7 +52,7 @@
MetadataManager.INSTANCE.commitTransaction(ctx);
switch (dataset.getDatasetType()) {
case INTERNAL:
- initInternalDataset(itemType, metaItemType);
+ initInternalDataset(itemType, metaItemType,
datasetDetails);
break;
case EXTERNAL:
initExternalDataset(itemType);
@@ -74,13 +77,21 @@
return dataset;
}
- private void initInternalDataset(IAType itemType, IAType metaItemType)
throws IOException, AlgebricksException {
+ private void initInternalDataset(IAType itemType, IAType metaItemType,
IDatasetDetails datasetDetails)
+ throws IOException, AlgebricksException {
+ InternalDatasetDetails internalDatasetDetails =
(InternalDatasetDetails) datasetDetails;
List<List<String>> partitioningKeys =
DatasetUtils.getPartitioningKeys(dataset);
ARecordType recordType = (ARecordType) itemType;
+ ARecordType metaRecordType = (ARecordType) metaItemType;
int n = partitioningKeys.size();
schemaTypes = metaItemType == null ? new IAType[n + 1] : new IAType[n
+ 2];
+ List<Integer> keySourceIndicators =
internalDatasetDetails.getKeySourceIndicator();
for (int i = 0; i < n; i++) {
- schemaTypes[i] =
recordType.getSubFieldType(partitioningKeys.get(i));
+ if (keySourceIndicators.get(i) == 0) {
+ schemaTypes[i] =
recordType.getSubFieldType(partitioningKeys.get(i));
+ } else {
+ schemaTypes[i] =
metaRecordType.getSubFieldType(partitioningKeys.get(i));
+ }
}
schemaTypes[n] = itemType;
if (metaItemType != null) {
diff --git
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
index 6971037..b0c6c51 100644
---
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
+++
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
@@ -20,6 +20,7 @@
package org.apache.asterix.metadata.entities;
import java.io.DataOutput;
+import java.util.ArrayList;
import java.util.List;
import org.apache.asterix.builders.IARecordBuilder;
@@ -75,6 +76,13 @@
this.partitioningStrategy = partitioningStrategy;
this.partitioningKeys = partitioningKey;
this.primaryKeys = primaryKey;
+ if (keyFieldIndicators == null) {
+ // Create a dummy list.
+ keyFieldIndicators = new ArrayList<>();
+ for (int index = 0; index < partitioningKey.size(); ++index) {
+ keyFieldIndicators.add(0);
+ }
+ }
this.keySourceIndicators = keyFieldIndicators;
this.primaryKeyTypes = primaryKeyType;
this.autogenerated = autogenerated;
diff --git
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
index f310a03..241a68d 100644
---
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
+++
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
@@ -142,18 +142,7 @@
IAType fieldType =
AsterixBuiltinTypeMap.getTypeFromTypeName(metadataNode, jobId, dvName,
typeName, false);
searchKeyType.add(fieldType);
}
- // index key type information is not persisted, thus we extract type
information from the record metadata
- if (searchKeyType.isEmpty()) {
- Dataset dSet = metadataNode.getDataset(jobId, dvName, dsName);
- String datatypeName = dSet.getItemTypeName();
- String datatypeDataverseName = dSet.getItemTypeDataverseName();
- ARecordType recordDt = (ARecordType)
metadataNode.getDatatype(jobId, datatypeDataverseName, datatypeName)
- .getDatatype();
- for (int i = 0; i < searchKey.size(); i++) {
- IAType fieldType = recordDt.getSubFieldType(searchKey.get(i));
- searchKeyType.add(fieldType);
- }
- }
+
int isEnforcedFieldPos =
rec.getType().getFieldIndex(INDEX_ISENFORCED_FIELD_NAME);
Boolean isEnforcingKeys = false;
if (isEnforcedFieldPos > 0) {
@@ -183,6 +172,27 @@
keyFieldSourceIndicator.add(0);
}
}
+
+ // index key type information is not persisted, thus we extract type
information from the record metadata
+ if (searchKeyType.isEmpty()) {
+ Dataset dSet = metadataNode.getDataset(jobId, dvName, dsName);
+ String datatypeName = dSet.getItemTypeName();
+ String datatypeDataverseName = dSet.getItemTypeDataverseName();
+ ARecordType recordDt = (ARecordType)
metadataNode.getDatatype(jobId, datatypeDataverseName, datatypeName)
+ .getDatatype();
+ String metatypeName = dSet.getMetaItemTypeName();
+ String metatypeDataverseName = dSet.getMetaItemTypeDataverseName();
+ ARecordType metaDt = null;
+ if (metatypeName != null && metatypeDataverseName != null) {
+ metaDt = (ARecordType) metadataNode.getDatatype(jobId,
metatypeDataverseName, metatypeName)
+ .getDatatype();
+ }
+ for (int i = 0; i < searchKey.size(); i++) {
+ IAType fieldType = keyFieldSourceIndicator.get(i) == 0 ?
recordDt.getSubFieldType(searchKey.get(i))
+ : metaDt.getSubFieldType(searchKey.get(i));
+ searchKeyType.add(fieldType);
+ }
+ }
return new Index(dvName, dsName, indexName, indexStructure, searchKey,
keyFieldSourceIndicator, searchKeyType,
gramLength, isEnforcingKeys, isPrimaryIndex, pendingOp);
}
diff --git
a/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
b/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
index 997dccd..9f8d5d7 100644
---
a/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
+++
b/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
@@ -90,6 +90,7 @@
import
org.apache.asterix.om.typecomputer.impl.OptionalATemporalInstanceTypeComputer;
import org.apache.asterix.om.typecomputer.impl.OptionalATimeTypeComputer;
import
org.apache.asterix.om.typecomputer.impl.OptionalAYearMonthDurationTypeComputer;
+import org.apache.asterix.om.typecomputer.impl.OptionalOpenARecordTypeComputer;
import
org.apache.asterix.om.typecomputer.impl.OrderedListConstructorResultType;
import org.apache.asterix.om.typecomputer.impl.OrderedListOfAInt32TypeComputer;
import org.apache.asterix.om.typecomputer.impl.OrderedListOfAInt64TypeComputer;
@@ -723,6 +724,9 @@
public static final FunctionIdentifier EXTERNAL_LOOKUP = new
FunctionIdentifier(FunctionConstants.ASTERIX_NS,
"external-lookup", FunctionIdentifier.VARARGS);
+ public static final FunctionIdentifier META = new
FunctionIdentifier(FunctionConstants.ASTERIX_NS, "meta",
+ FunctionIdentifier.VARARGS);
+
public static IFunctionInfo getAsterixFunctionInfo(FunctionIdentifier fid)
{
return registeredFunctions.get(fid);
}
@@ -1028,6 +1032,9 @@
addFunction(INTERVAL_CONSTRUCTOR_START_FROM_DATETIME,
OptionalAIntervalTypeComputer.INSTANCE, true);
addFunction(INTERVAL_CONSTRUCTOR_START_FROM_TIME,
OptionalAIntervalTypeComputer.INSTANCE, true);
+ // meta() function
+ addFunction(META, OptionalOpenARecordTypeComputer.INSTANCE, true);
+
addPrivateFunction(COLLECTION_TO_SEQUENCE,
CollectionToSequenceTypeComputer.INSTANCE, true);
// external lookup
diff --git
a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/OptionalOpenARecordTypeComputer.java
b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/OptionalOpenARecordTypeComputer.java
new file mode 100644
index 0000000..b630f44
--- /dev/null
+++
b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/OptionalOpenARecordTypeComputer.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.typecomputer.impl;
+
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import
org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public class OptionalOpenARecordTypeComputer implements IResultTypeComputer {
+
+ public static final OptionalOpenARecordTypeComputer INSTANCE = new
OptionalOpenARecordTypeComputer();
+
+ private OptionalOpenARecordTypeComputer() {
+ }
+
+ @Override
+ public IAType computeType(ILogicalExpression expression,
IVariableTypeEnvironment env,
+ IMetadataProvider<?, ?> metadataProvider) throws
AlgebricksException {
+ return
AUnionType.createNullableType(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE,
"OptionalOpenRecord");
+ }
+
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/685
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie65417b6baf209ca0ab413cfa4a5f7fc5156ca63
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <[email protected]>
