Ildar Absalyamov has submitted this change and it was merged. Change subject: - Reapplied fix for ASTERIXDB-1109 on merged master - Cleaned up runtime testsuite, made it validate against XSD ......................................................................
- Reapplied fix for ASTERIXDB-1109 on merged master - Cleaned up runtime testsuite, made it validate against XSD Change-Id: I8fcf41c009f9faf51bc8bccc0c3c7217b7769121 Reviewed-on: https://asterix-gerrit.ics.uci.edu/712 Tested-by: Jenkins <[email protected]> Reviewed-by: Ian Maxon <[email protected]> --- M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java M asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java R asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-collision/index-type-collision.1.ddl.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-promotion-collision/index-type-promotion-collision.1.ddl.aql R asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/record-type-collision/record-collision.1.ddl.aql A asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.ddl.aql A asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.update.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.3.ddl.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.4.query.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.5.ddl.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.6.query.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.1.ddl.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.2.update.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.3.ddl.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.4.query.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.5.query.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.6.query.aql A asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.adm A asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.adm A asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.1.adm A asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.2.adm A asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.3.adm M asterix-app/src/test/resources/runtimets/testsuite.xml M asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java M asterix-test-framework/src/main/resources/Catalog.xsd 25 files changed, 459 insertions(+), 268 deletions(-) Approvals: Ian Maxon: Looks good to me, approved Jenkins: Verified diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java index ab09188..8cc18ba 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java @@ -21,7 +21,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Set; import java.util.Stack; import org.apache.asterix.common.config.DatasetConfig.DatasetType; @@ -47,6 +50,7 @@ import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; +import org.apache.asterix.om.types.hierachy.ATypeHierarchy; import org.apache.asterix.om.util.NonTaggedFormatUtil; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.mutable.Mutable; @@ -101,9 +105,13 @@ /** find the record variable */ InsertDeleteUpsertOperator insertOp = (InsertDeleteUpsertOperator) op1; ILogicalExpression recordExpr = insertOp.getPayloadExpression().getValue(); - List<LogicalVariable> recordVar = new ArrayList<LogicalVariable>(); + LogicalVariable recordVar = null; + List<LogicalVariable> usedRecordVars = new ArrayList<>(); /** assume the payload is always a single variable expression */ - recordExpr.getUsedVariables(recordVar); + recordExpr.getUsedVariables(usedRecordVars); + if (usedRecordVars.size() == 1) { + recordVar = usedRecordVars.get(0); + } /** * op2 is the assign operator which extract primary keys from the record @@ -111,7 +119,7 @@ */ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue(); - if (recordVar.size() == 0) { + if (recordVar == null) { /** * For the case primary key-assignment expressions are constant * expressions, find assign op that creates record to be @@ -134,7 +142,7 @@ } } AssignOperator assignOp2 = (AssignOperator) op2; - recordVar.addAll(assignOp2.getVariables()); + recordVar = assignOp2.getVariables().get(0); } /* @@ -196,20 +204,6 @@ op0.getInputs().clear(); } - // Replicate Operator is applied only when doing the bulk-load. - ReplicateOperator replicateOp = null; - - // No need to take care of the upsert case in the bulk load since it doesn't exist as of now - if (secondaryIndexTotalCnt > 1 && insertOp.isBulkload()) { - // Split the logical plan into "each secondary index update branch" - // to replicate each <PK,RECORD> pair. - replicateOp = new ReplicateOperator(secondaryIndexTotalCnt); - replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); - replicateOp.setExecutionMode(ExecutionMode.PARTITIONED); - context.computeAndSetTypeEnvironmentForOperator(replicateOp); - currentTop = replicateOp; - } - // Prepare filtering field information (This is the filter created using the "filter with" key word in the // create dataset ddl) List<String> filteringFields = ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterField(); @@ -228,33 +222,22 @@ } } } + LogicalVariable enforcedRecordVar = recordVar; - // Iterate each secondary index and applying Index Update operations. - // At first, op1 is the index insert op insertOp - for (Index index : indexes) { - List<LogicalVariable> projectVars = new ArrayList<LogicalVariable>(); - // Q. Why do we add these used variables to the projectVars? - // A. We want to always keep the primary keys, the record, and the filtering values - // In addition to those, we want to extract and keep the secondary key - VariableUtilities.getUsedVariables(insertOp, projectVars); - if (!index.isSecondaryIndex()) { - continue; - } - LogicalVariable enforcedRecordVar = recordVar.get(0); - hasSecondaryIndex = true; - /* - * if the index is enforcing field types (For open indexes), We add a cast - * operator to ensure type safety - */ - if (index.isEnforcingKeyFileds()) { - try { - DatasetDataSource ds = (DatasetDataSource) (insertOp.getDataSource()); - ARecordType insertRecType = (ARecordType) ds.getSchemaTypes()[ds.getSchemaTypes().length - 1]; - // A new variable which represents the casted record - LogicalVariable castVar = context.newVar(); - // create the expected record type = the original + the optional open field - ARecordType enforcedType = createEnforcedType(insertRecType, index); - // introduce casting to enforced type + /* + * if the index is enforcing field types (For open indexes), We add a cast + * operator to ensure type safety + */ + if (insertOp.getOperation() == Kind.INSERT || insertOp.getOperation() == Kind.UPSERT) { + try { + DatasetDataSource ds = (DatasetDataSource) (insertOp.getDataSource()); + ARecordType insertRecType = (ARecordType) ds.getSchemaTypes()[ds.getSchemaTypes().length - 1]; + // A new variable which represents the casted record + LogicalVariable castedRecVar = context.newVar(); + // create the expected record type = the original + the optional open field + ARecordType enforcedType = createEnforcedType(insertRecType, indexes); + if (!enforcedType.equals(insertRecType)) { + //introduce casting to enforced type AbstractFunctionCallExpression castFunc = new ScalarFunctionCallExpression( FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.CAST_RECORD)); // The first argument is the record @@ -262,23 +245,57 @@ .add(new MutableObject<ILogicalExpression>(insertOp.getPayloadExpression().getValue())); TypeComputerUtilities.setRequiredAndInputTypes(castFunc, enforcedType, insertRecType); // AssignOperator puts in the cast var the casted record - AssignOperator newAssignOperator = new AssignOperator(castVar, + AssignOperator castedRecordAssignOperator = new AssignOperator(castedRecVar, new MutableObject<ILogicalExpression>(castFunc)); // Connect the current top of the plan to the cast operator - newAssignOperator.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); - currentTop = newAssignOperator; - projectVars.add(castVar); - enforcedRecordVar = castVar; - context.computeAndSetTypeEnvironmentForOperator(newAssignOperator); - context.computeAndSetTypeEnvironmentForOperator(currentTop); + castedRecordAssignOperator.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); + currentTop = castedRecordAssignOperator; + enforcedRecordVar = castedRecVar; recType = enforcedType; + context.computeAndSetTypeEnvironmentForOperator(castedRecordAssignOperator); // We don't need to cast the old rec, we just need an assignment function that extracts the SK // and an expression which reference the new variables. - } catch (AsterixException e) { - throw new AlgebricksException(e); } + } catch (AsterixException e) { + throw new AlgebricksException(e); } + } + Set<LogicalVariable> projectVars = new HashSet<LogicalVariable>(); + VariableUtilities.getUsedVariables(op1, projectVars); + if (enforcedRecordVar != null) { + projectVars.add(enforcedRecordVar); + } + if (insertOp.getOperation() == Kind.UPSERT) { + projectVars.add(insertOp.getPrevRecordVar()); + if (filteringFields != null) { + // project prev filter value + projectVars.add(insertOp.getPrevFilterVar()); + } + } + ProjectOperator project = new ProjectOperator(new ArrayList<LogicalVariable>(projectVars)); + project.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); + context.computeAndSetTypeEnvironmentForOperator(project); + currentTop = project; + // Replicate Operator is applied only when doing the bulk-load. + AbstractLogicalOperator replicateOp = null; + if (secondaryIndexTotalCnt > 1 && insertOp.isBulkload()) { + // Split the logical plan into "each secondary index update branch" + // to replicate each <PK,RECORD> pair. + replicateOp = new ReplicateOperator(secondaryIndexTotalCnt); + replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); + replicateOp.setExecutionMode(ExecutionMode.PARTITIONED); + context.computeAndSetTypeEnvironmentForOperator(replicateOp); + currentTop = replicateOp; + } + + // Iterate each secondary index and applying Index Update operations. + // At first, op1 is the index insert op insertOp + for (Index index : indexes) { + if (!index.isSecondaryIndex()) { + continue; + } + hasSecondaryIndex = true; // Get the secondary fields names and types List<List<String>> secondaryKeyFields = index.getKeyFieldNames(); List<IAType> secondaryKeyTypes = index.getKeyFieldTypes(); @@ -307,26 +324,19 @@ prevSecondaryKeyAssign = new AssignOperator(prevSecondaryKeyVars, prevExpressions); } AssignOperator assign = new AssignOperator(secondaryKeyVars, expressions); - ProjectOperator project = new ProjectOperator(projectVars); + AssignOperator topAssign = assign; - assign.getInputs().add(new MutableObject<ILogicalOperator>(project)); if (insertOp.getOperation() == Kind.UPSERT) { - projectVars.add(insertOp.getPrevRecordVar()); - if (filteringFields != null) { - // project prev filter value - projectVars.add(insertOp.getPrevFilterVar()); - } - prevSecondaryKeyAssign.getInputs().add(new MutableObject<ILogicalOperator>(assign)); + prevSecondaryKeyAssign.getInputs().add(new MutableObject<ILogicalOperator>(topAssign)); topAssign = prevSecondaryKeyAssign; } // Only apply replicate operator when doing bulk-load if (secondaryIndexTotalCnt > 1 && insertOp.isBulkload()) { - project.getInputs().add(new MutableObject<ILogicalOperator>(replicateOp)); + assign.getInputs().add(new MutableObject<ILogicalOperator>(replicateOp)); } else { - project.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); + assign.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); } - context.computeAndSetTypeEnvironmentForOperator(project); context.computeAndSetTypeEnvironmentForOperator(assign); if (insertOp.getOperation() == Kind.UPSERT) { context.computeAndSetTypeEnvironmentForOperator(prevSecondaryKeyAssign); @@ -553,65 +563,92 @@ return true; } - public static ARecordType createEnforcedType(ARecordType initialType, Index index) + // Merges typed index fields with specified recordType, allowing indexed fields to be optional. + // I.e. the type { "personId":int32, "name": string, "address" : { "street": string } } with typed indexes on age:int32, address.state:string + // will be merged into type { "personId":int32, "name": string, "age": int32? "address" : { "street": string, "state": string? } } + // Used by open indexes to enforce the type of an indexed record + public static ARecordType createEnforcedType(ARecordType initialType, List<Index> indexes) throws AsterixException, AlgebricksException { ARecordType enforcedType = initialType; - for (int i = 0; i < index.getKeyFieldNames().size(); i++) { - - Stack<Pair<ARecordType, String>> nestedTypeStack = new Stack<Pair<ARecordType, String>>(); - List<String> splits = index.getKeyFieldNames().get(i); - ARecordType nestedFieldType = enforcedType; - boolean openRecords = false; - String bridgeName = nestedFieldType.getTypeName(); - int j; - // Build the stack for the enforced type - for (j = 1; j < splits.size(); j++) { - nestedTypeStack.push(new Pair<ARecordType, String>(nestedFieldType, splits.get(j - 1))); - bridgeName = nestedFieldType.getTypeName(); - nestedFieldType = (ARecordType) enforcedType.getSubFieldType(splits.subList(0, j)); - if (nestedFieldType == null) { - openRecords = true; - break; - } + for (Index index : indexes) { + if (!index.isSecondaryIndex() || !index.isEnforcingKeyFileds()) { + continue; } - if (openRecords == true) { - // create the smallest record - enforcedType = new ARecordType(splits.get(splits.size() - 2), - new String[] { splits.get(splits.size() - 1) }, - new IAType[] { AUnionType.createNullableType(index.getKeyFieldTypes().get(i)) }, true); - // create the open part of the nested field - for (int k = splits.size() - 3; k > (j - 2); k--) { - enforcedType = new ARecordType(splits.get(k), new String[] { splits.get(k + 1) }, - new IAType[] { AUnionType.createNullableType(enforcedType) }, true); + for (int i = 0; i < index.getKeyFieldNames().size(); i++) { + Stack<Pair<ARecordType, String>> nestedTypeStack = new Stack<Pair<ARecordType, String>>(); + List<String> splits = index.getKeyFieldNames().get(i); + ARecordType nestedFieldType = enforcedType; + boolean openRecords = false; + String bridgeName = nestedFieldType.getTypeName(); + int j; + // Build the stack for the enforced type + for (j = 1; j < splits.size(); j++) { + nestedTypeStack.push(new Pair<ARecordType, String>(nestedFieldType, splits.get(j - 1))); + bridgeName = nestedFieldType.getTypeName(); + nestedFieldType = (ARecordType) enforcedType.getSubFieldType(splits.subList(0, j)); + if (nestedFieldType == null) { + openRecords = true; + break; + } } - // Bridge the gap - Pair<ARecordType, String> gapPair = nestedTypeStack.pop(); - ARecordType parent = gapPair.first; + if (openRecords == true) { + // create the smallest record + enforcedType = new ARecordType(splits.get(splits.size() - 2), + new String[] { splits.get(splits.size() - 1) }, + new IAType[] { AUnionType.createNullableType(index.getKeyFieldTypes().get(i)) }, true); + // create the open part of the nested field + for (int k = splits.size() - 3; k > (j - 2); k--) { + enforcedType = new ARecordType(splits.get(k), new String[] { splits.get(k + 1) }, + new IAType[] { AUnionType.createNullableType(enforcedType) }, true); + } + // Bridge the gap + Pair<ARecordType, String> gapPair = nestedTypeStack.pop(); + ARecordType parent = gapPair.first; - IAType[] parentFieldTypes = ArrayUtils.addAll(parent.getFieldTypes().clone(), - new IAType[] { AUnionType.createNullableType(enforcedType) }); - enforcedType = new ARecordType(bridgeName, - ArrayUtils.addAll(parent.getFieldNames(), enforcedType.getTypeName()), parentFieldTypes, true); + IAType[] parentFieldTypes = ArrayUtils.addAll(parent.getFieldTypes().clone(), + new IAType[] { AUnionType.createNullableType(enforcedType) }); + enforcedType = new ARecordType(bridgeName, + ArrayUtils.addAll(parent.getFieldNames(), enforcedType.getTypeName()), parentFieldTypes, + true); - } else { - // Schema is closed all the way to the field - // enforced fields are either null or strongly typed - enforcedType = new ARecordType(nestedFieldType.getTypeName(), - ArrayUtils.addAll(nestedFieldType.getFieldNames(), splits.get(splits.size() - 1)), - ArrayUtils.addAll(nestedFieldType.getFieldTypes(), - AUnionType.createNullableType(index.getKeyFieldTypes().get(i))), - nestedFieldType.isOpen()); - } + } else { + //Schema is closed all the way to the field + //enforced fields are either null or strongly typed + LinkedHashMap<String, IAType> recordNameTypesMap = new LinkedHashMap<String, IAType>(); + for (j = 0; j < nestedFieldType.getFieldNames().length; j++) { + recordNameTypesMap.put(nestedFieldType.getFieldNames()[j], nestedFieldType.getFieldTypes()[j]); + } + // if a an enforced field already exists and the type is correct + IAType enforcedFieldType = recordNameTypesMap.get(splits.get(splits.size() - 1)); + if (enforcedFieldType != null && enforcedFieldType.getTypeTag() == ATypeTag.UNION + && ((AUnionType) enforcedFieldType).isNullableType()) { + enforcedFieldType = ((AUnionType) enforcedFieldType).getNullableType(); + } + if (enforcedFieldType != null && !ATypeHierarchy.canPromote(enforcedFieldType.getTypeTag(), + index.getKeyFieldTypes().get(i).getTypeTag())) { + throw new AlgebricksException("Cannot enforce field " + index.getKeyFieldNames().get(i) + + " to have type " + index.getKeyFieldTypes().get(i)); + } + if (enforcedFieldType == null) { + recordNameTypesMap.put(splits.get(splits.size() - 1), + AUnionType.createNullableType(index.getKeyFieldTypes().get(i))); + } + enforcedType = new ARecordType(nestedFieldType.getTypeName(), + recordNameTypesMap.keySet().toArray(new String[recordNameTypesMap.size()]), + recordNameTypesMap.values().toArray(new IAType[recordNameTypesMap.size()]), + nestedFieldType.isOpen()); + } - // Create the enforcedtype for the nested fields in the schema, from the ground up - if (nestedTypeStack.size() > 0) { - while (!nestedTypeStack.isEmpty()) { - Pair<ARecordType, String> nestedTypePair = nestedTypeStack.pop(); - ARecordType nestedRecType = nestedTypePair.first; - IAType[] nestedRecTypeFieldTypes = nestedRecType.getFieldTypes().clone(); - nestedRecTypeFieldTypes[nestedRecType.getFieldIndex(nestedTypePair.second)] = enforcedType; - enforcedType = new ARecordType(nestedRecType.getTypeName(), nestedRecType.getFieldNames(), - nestedRecTypeFieldTypes, nestedRecType.isOpen()); + // Create the enforcedtype for the nested fields in the schema, from the ground up + if (nestedTypeStack.size() > 0) { + while (!nestedTypeStack.isEmpty()) { + Pair<ARecordType, String> nestedTypePair = nestedTypeStack.pop(); + ARecordType nestedRecType = nestedTypePair.first; + IAType[] nestedRecTypeFieldTypes = nestedRecType.getFieldTypes().clone(); + nestedRecTypeFieldTypes[nestedRecType.getFieldIndex(nestedTypePair.second)] = enforcedType; + enforcedType = new ARecordType(nestedRecType.getTypeName() + "_enforced", + nestedRecType.getFieldNames(), nestedRecTypeFieldTypes, nestedRecType.isOpen()); + } } } } diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java index 6b54fbc..213b090 100644 --- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java +++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java @@ -190,6 +190,8 @@ import org.json.JSONException; import org.json.JSONObject; +import com.google.common.collect.Lists; + /* * Provides functionality for executing a batch of Query statements (queries included) * sequentially. @@ -991,9 +993,9 @@ for (Index index : indexes) { if (index.getKeyFieldNames().equals(indexFields) && !index.getKeyFieldTypes().equals(indexFieldTypes) && index.isEnforcingKeyFileds()) { - throw new AsterixException( - "Cannot create index " + indexName + " , enforced index " + index.getIndexName() - + " on field \"" + StringUtils.join(indexFields, ',') + "\" already exist"); + throw new AsterixException("Cannot create index " + indexName + " , enforced index " + + index.getIndexName() + " on field \"" + StringUtils.join(indexFields, ',') + + "\" is already defined with type \"" + index.getKeyFieldTypes() + "\""); } } } @@ -1006,7 +1008,8 @@ ARecordType enforcedType = null; if (stmtCreateIndex.isEnforced()) { - enforcedType = IntroduceSecondaryIndexInsertDeleteRule.createEnforcedType(aRecordType, index); + enforcedType = IntroduceSecondaryIndexInsertDeleteRule.createEnforcedType(aRecordType, + Lists.newArrayList(index)); } //#. prepare to create the index artifact in NC. @@ -2467,11 +2470,15 @@ "Cannot compact the extrenal dataset " + datasetName + " because it has no indexes"); } + Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(), + dataverseName); + jobsToExecute.add(DatasetOperations.compactDatasetJobSpec(dataverse, datasetName, metadataProvider)); + ARecordType aRecordType = (ARecordType) dt.getDatatype(); + ARecordType enforcedType = IntroduceSecondaryIndexInsertDeleteRule.createEnforcedType(aRecordType, indexes); + if (ds.getDatasetType() == DatasetType.INTERNAL) { for (int j = 0; j < indexes.size(); j++) { if (indexes.get(j).isSecondaryIndex()) { - Dataverse dataverse = MetadataManager.INSTANCE - .getDataverse(metadataProvider.getMetadataTxnContext(), dataverseName); jobsToExecute .add(DatasetOperations.compactDatasetJobSpec(dataverse, datasetName, metadataProvider)); } @@ -2483,15 +2490,8 @@ datasetName, indexes.get(j).getIndexName(), indexes.get(j).getKeyFieldNames(), indexes.get(j).getKeyFieldTypes(), indexes.get(j).isEnforcingKeyFileds(), indexes.get(j).getGramLength(), indexes.get(j).getIndexType()); - ARecordType aRecordType = (ARecordType) dt.getDatatype(); - ARecordType enforcedType = null; - if (cics.isEnforced()) { - enforcedType = IntroduceSecondaryIndexInsertDeleteRule.createEnforcedType(aRecordType, - indexes.get(j)); - } jobsToExecute.add(IndexOperations.buildSecondaryIndexCompactJobSpec(cics, aRecordType, enforcedType, metadataProvider, ds)); - } } diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-collision/index-type-collision.1.ddl.aql similarity index 87% rename from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql rename to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-collision/index-type-collision.1.ddl.aql index 853386d..84d35b7 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-collision/index-type-collision.1.ddl.aql @@ -21,9 +21,9 @@ use dataverse test; create type testType as open { - "id": int32, - "value": string + "id": int32 } create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +create index testIdx1 on testDS(value: int32) enforced; +create index testIdx2 on testDS(value: string) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-promotion-collision/index-type-promotion-collision.1.ddl.aql similarity index 87% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-promotion-collision/index-type-promotion-collision.1.ddl.aql index 853386d..6c47032 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-promotion-collision/index-type-promotion-collision.1.ddl.aql @@ -21,9 +21,9 @@ use dataverse test; create type testType as open { - "id": int32, - "value": string + "id": int32 } create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +create index testIdx1 on testDS(value: int64) enforced; +create index testIdx2 on testDS(value: int32) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-type-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/record-type-collision/record-collision.1.ddl.aql similarity index 100% rename from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-type-collision/enforced-field-name-collision.1.ddl.aql rename to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/record-type-collision/record-collision.1.ddl.aql diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.ddl.aql new file mode 100644 index 0000000..10a3530 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.ddl.aql @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Test that BTree open index is used in query plan + * : define the BTree open index on a composite key (fname,lanme) + * : predicate => where $l.fname="Julio" and $l.lname="Isa" + * Expected Result : Success + * Issue : Issue 162 + * Date : 27th March 2014 + */ + +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type Emp as closed { +id:int64, +fname:string, +lname:string, +age:int64, +dept:string +} + +create type EmpOpen as open { +id:int64, +fname:string, +age:int64, +dept:string +} + +create dataset employee(Emp) primary key id; + +create dataset employeeOpen(EmpOpen) primary key id; + diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.update.aql new file mode 100644 index 0000000..50b2269 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.update.aql @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Test that BTree enforced open index is used in query plan + * : define the BTree enforced open index on a composite key (fname,lanme) + * : predicate => where $l.fname="Julio" and $l.lname="Isa" + * Expected Result : Success + * Issue : Issue 162 + * Date : 27th March 2014 + */ + +use dataverse test; + +load dataset employee +using localfs +(("path"="asterix_nc1://data/names.adm"),("format"="delimited-text"),("delimiter"="|")); + +insert into dataset employeeOpen ( + for $x in dataset employee return $x +); diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.3.ddl.aql similarity index 60% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.3.ddl.aql index 853386d..11b0baa 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.3.ddl.aql @@ -16,14 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; +/* + * Description : Test that BTree enforced open index is used in query plan + * : define the BTree enforced open index on a composite key (fname,lanme) + * : predicate => where $l.fname="Julio" and $l.lname="Isa" + * Expected Result : Success + * Issue : Issue 162 + * Date : 27th March 2014 + */ + use dataverse test; -create type testType as open { - "id": int32, - "value": string -} +// create secondary index -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +create index idx_employee_f_l_name on employeeOpen(fname,lname:string) enforced; +create index idx_employee_l_f_name on employeeOpen(lname:string, fname) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.4.query.aql similarity index 60% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.4.query.aql index 853386d..b1f2adf 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.4.query.aql @@ -16,14 +16,23 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; +/* + * Description : Test that BTree enforced open index is used in query plan + * : define the BTree enforced open index on a composite key (fname,lanme) + * : predicate => where $l.fname="Julio" and $l.lname="Isa" + * Expected Result : Success + * Issue : Issue 162 + * Date : 27th March 2014 + */ + use dataverse test; -create type testType as open { - "id": int32, - "value": string +for $l in dataset('employeeOpen') +where $l.fname="Julio" and $l.lname="Isa" +return { + "id": $l.id, + "fname": $l.fname, + "lname": $l.lname, + "age": $l.age, + "dept": $l.dept } - -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.5.ddl.aql similarity index 65% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.5.ddl.aql index 853386d..4eaf8e8 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.5.ddl.aql @@ -16,14 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; +/* + * Description : Test that BTree enforced open index is used in query plan + * : define the BTree enforced open index on a composite key (fname,lanme) + * : predicate => where $l.fname="Julio" and $l.lname="Isa" + * Expected Result : Success + * Issue : Issue 162 + * Date : 27th March 2014 + */ + use dataverse test; -create type testType as open { - "id": int32, - "value": string -} +// create secondary index -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +drop index employeeOpen.idx_employee_f_l_name; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.6.query.aql similarity index 60% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.6.query.aql index 853386d..b1f2adf 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.6.query.aql @@ -16,14 +16,23 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; +/* + * Description : Test that BTree enforced open index is used in query plan + * : define the BTree enforced open index on a composite key (fname,lanme) + * : predicate => where $l.fname="Julio" and $l.lname="Isa" + * Expected Result : Success + * Issue : Issue 162 + * Date : 27th March 2014 + */ + use dataverse test; -create type testType as open { - "id": int32, - "value": string +for $l in dataset('employeeOpen') +where $l.fname="Julio" and $l.lname="Isa" +return { + "id": $l.id, + "fname": $l.fname, + "lname": $l.lname, + "age": $l.age, + "dept": $l.dept } - -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.1.ddl.aql similarity index 68% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.1.ddl.aql index 853386d..0efffb2 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.1.ddl.aql @@ -20,10 +20,24 @@ create dataverse test; use dataverse test; -create type testType as open { - "id": int32, - "value": string +create type DBLPType as closed { + id: int64, + dblpid: string, + title: string, + authors: string, + misc: string } -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +create type DBLPOpenType as open { + id: int64, + dblpid: string, + authors: string, + misc: string +} + +create nodegroup group1 if not exists on nc1, nc2; + +create dataset DBLP(DBLPType) + primary key id on group1; +create dataset DBLPOpen(DBLPOpenType) + primary key id on group1; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.2.update.aql similarity index 64% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.2.update.aql index 853386d..feb951a 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.2.update.aql @@ -16,14 +16,25 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; use dataverse test; -create type testType as open { - "id": int32, - "value": string -} +load dataset DBLP +using localfs +(("path"="asterix_nc1://data/dblp-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":")) pre-sorted; -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +insert into dataset test.DBLPOpen ( + for $x in dataset test.DBLP + where $x.id <= 50 + return $x +); + +insert into dataset test.DBLPOpen ( + for $c in dataset test.DBLP + where $c.id > 50 + return { + "id": $c.id, + "dblpid": $c.dblpid, + "authors": $c.authors, + "misc": $c.misc + } +); diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.3.ddl.aql similarity index 78% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.3.ddl.aql index 853386d..98338b2 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.3.ddl.aql @@ -16,14 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; + use dataverse test; -create type testType as open { - "id": int32, - "value": string -} - -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +create index ngram_index on DBLPOpen(title:string) type ngram(3) enforced; +create index keyword_index on DBLPOpen(title:string) type keyword enforced; +create index btree_index on DBLPOpen(title:string) enforced; \ No newline at end of file diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.4.query.aql similarity index 78% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.4.query.aql index 853386d..775e97d 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.4.query.aql @@ -16,14 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; use dataverse test; -create type testType as open { - "id": int32, - "value": string +for $o in dataset('DBLPOpen') +where contains($o.title, "Multimedia") +order by $o.id +return { + "id": $o.id, + "dblpid": $o.dblpid, + "title": $o.title, + "authors": $o.authors, + "misc": $o.misc } -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.5.query.aql similarity index 74% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.5.query.aql index 853386d..b7423e2 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.5.query.aql @@ -16,14 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; + use dataverse test; -create type testType as open { - "id": int32, - "value": string +for $o in dataset('DBLPOpen') +let $jacc := similarity-jaccard-check(word-tokens($o.title), word-tokens("Transactions for Cooperative Environments"), 0.5f) +where $jacc[0] +return { + "id": $o.id, + "dblpid": $o.dblpid, + "title": $o.title, + "authors": $o.authors, + "misc": $o.misc } - -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.6.query.aql similarity index 77% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.6.query.aql index 853386d..0e1ef29 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.6.query.aql @@ -16,14 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; use dataverse test; -create type testType as open { - "id": int32, - "value": string +for $o in dataset('DBLPOpen') +where $o.title = "Multimedia Information Systems Issues and Approaches." +order by $o.title +return { + "id": $o.id, + "dblpid": $o.dblpid, + "title": $o.title, + "authors": $o.authors, + "misc": $o.misc } - -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; diff --git a/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.adm b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.adm new file mode 100644 index 0000000..3a3fb2f --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.adm @@ -0,0 +1 @@ +{ "id": 881, "fname": "Julio", "age": 38, "dept": "Sales", "lname": "Isa" } diff --git a/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.adm b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.adm new file mode 100644 index 0000000..3a3fb2f --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.adm @@ -0,0 +1 @@ +{ "id": 881, "fname": "Julio", "age": 38, "dept": "Sales", "lname": "Isa" } diff --git a/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.1.adm b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.1.adm new file mode 100644 index 0000000..5ef01a1 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.1.adm @@ -0,0 +1 @@ +{ "id": 4, "dblpid": "books/acm/kim95/ChristodoulakisK95", "authors": "Stavros Christodoulakis Leonidas Koveos", "misc": "2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95", "title": "Multimedia Information Systems Issues and Approaches." } diff --git a/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.2.adm b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.2.adm new file mode 100644 index 0000000..5f1f708 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.2.adm @@ -0,0 +1 @@ +{ "id": 9, "dblpid": "books/acm/kim95/Kaiser95", "authors": "Gail E. Kaiser", "misc": "2002-01-03 409-433 1995 Modern Database Systems db/books/collections/kim95.html#Kaiser95", "title": "Cooperative Transactions for Multiuser Environments." } diff --git a/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.3.adm b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.3.adm new file mode 100644 index 0000000..5ef01a1 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.3.adm @@ -0,0 +1 @@ +{ "id": 4, "dblpid": "books/acm/kim95/ChristodoulakisK95", "authors": "Stavros Christodoulakis Leonidas Koveos", "misc": "2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95", "title": "Multimedia Information Systems Issues and Approaches." } diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml index 9dcd883..81480fd 100644 --- a/asterix-app/src/test/resources/runtimets/testsuite.xml +++ b/asterix-app/src/test/resources/runtimets/testsuite.xml @@ -289,11 +289,6 @@ </compilation-unit> </test-case> </test-group> - <test-case FilePath="flwor"> - <compilation-unit name="let33"> - <output-dir compare="Text">let33</output-dir> - </compilation-unit> - </test-case> <test-group name="aggregate"> <test-case FilePath="aggregate"> <compilation-unit name="issue531_string_min_max"> @@ -2118,6 +2113,11 @@ </compilation-unit> </test-case> <test-case FilePath="flwor"> + <compilation-unit name="let33"> + <output-dir compare="Text">let33</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="flwor"> <compilation-unit name="order-by-01"> <output-dir compare="Text">order-by-01</output-dir> </compilation-unit> @@ -2482,7 +2482,6 @@ <output-dir compare="Text">dblp-csx-3_5</output-dir> </compilation-unit> </test-case> - --> <test-case FilePath="fuzzyjoin"> <compilation-unit name="dblp-csx-aqlplus_1"> <output-dir compare="Text">dblp-csx-aqlplus_1</output-dir> @@ -3070,17 +3069,23 @@ </test-case> </test-group> <test-group name="open-index-enforced"> - <test-group FilePath="open-index-enforced/error-checking"> - <test-case FilePath="open-index-enforced/error-checking"><!-- This test case doesn't throw an exception? commenting it out --> - <compilation-unit name="enforced-field-name-collision"> - <output-dir compare="Text">enforced-field-name-collision</output-dir> - <!-- <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error> --> + <test-group name="open-index-enforced/error-checking"> + <test-case FilePath="open-index-enforced/error-checking"> + <compilation-unit name="index-on-closed-type"> + <output-dir compare="Text">index-on-closed-type</output-dir> + <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Typed index on "[value]" field could be created only for open datatype</expected-error> </compilation-unit> </test-case> <test-case FilePath="open-index-enforced/error-checking"> - <compilation-unit name="enforced-field-type-collision"> - <output-dir compare="Text">enforced-field-type-collision</output-dir> - <expected-error>SyntaxError: A field "[value]" is already defined with the type "STRING"</expected-error> + <compilation-unit name="index-type-collision"> + <output-dir compare="Text">index-type-collision</output-dir> + <expected-error>org.apache.asterix.common.exceptions.AsterixException: Cannot create index testIdx2 , enforced index testIdx1 on field "[value]" is already defined with type "[INT32]"</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="open-index-enforced/error-checking"> + <compilation-unit name="index-type-promotion-collision"> + <output-dir compare="Text">index-type-promotion-collision</output-dir> + <expected-error>org.apache.asterix.common.exceptions.AsterixException: Cannot create index testIdx2 , enforced index testIdx1 on field "[value]" is already defined with type "[INT64]"</expected-error> </compilation-unit> </test-case> <test-case FilePath="open-index-enforced/error-checking"> @@ -3090,13 +3095,13 @@ </compilation-unit> </test-case> <test-case FilePath="open-index-enforced/error-checking"> - <compilation-unit name="index-on-closed-type"> - <output-dir compare="Text">index-on-closed-type</output-dir> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Typed index on "[value]" field could be created only for open datatype</expected-error> + <compilation-unit name="record-type-collision"> + <output-dir compare="Text">record-type-collision</output-dir> + <expected-error>org.apache.asterix.common.exceptions.AsterixException: A field "[value]" is already defined with the type "STRING"</expected-error> </compilation-unit> </test-case> </test-group> - <test-group FilePath="open-index-enforced/index-join"> + <test-group name="open-index-enforced/index-join"> <test-case FilePath="open-index-enforced/index-join"> <compilation-unit name="btree-secondary-equi-join"> <output-dir compare="Text">btree-secondary-equi-join</output-dir> @@ -3138,7 +3143,7 @@ </compilation-unit> </test-case> </test-group> - <test-group FilePath="open-index-enforced/index-leftouterjoin"> + <test-group name="open-index-enforced/index-leftouterjoin"> <test-case FilePath="open-index-enforced/index-leftouterjoin"> <compilation-unit name="probe-pidx-with-join-btree-sidx1"> <output-dir compare="Text">probe-pidx-with-join-btree-sidx1</output-dir> @@ -3170,7 +3175,7 @@ </compilation-unit> </test-case> </test-group> - <test-group FilePath="open-index-enforced/index-selection"> + <test-group name="open-index-enforced/index-selection"> <test-case FilePath="open-index-enforced/index-selection"> <compilation-unit name="btree-index-composite-key"> <output-dir compare="Text">btree-index-composite-key</output-dir> @@ -3227,6 +3232,16 @@ </compilation-unit> </test-case> <test-case FilePath="open-index-enforced/index-selection"> + <compilation-unit name="multi-index"> + <output-dir compare="Text">multi-index</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="open-index-enforced/index-selection"> + <compilation-unit name="multi-index-composite-key"> + <output-dir compare="Text">multi-index-composite-key</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="open-index-enforced/index-selection"> <compilation-unit name="orders-index-custkey"> <output-dir compare="Text">orders-index-custkey</output-dir> </compilation-unit> @@ -3271,8 +3286,8 @@ </test-group> </test-group> <test-group name="nested-open-index"> - <test-group FilePath="nested-open-index/index-join"> - <test-case FilePath="nested-open-index/index-join"> + <test-group name="nested-open-index/index-join"> + <test-case FilePath="index-join"> <compilation-unit name="btree-secondary-equi-join"> <output-dir compare="Text">btree-secondary-equi-join</output-dir> </compilation-unit> @@ -3313,7 +3328,7 @@ </compilation-unit> </test-case> </test-group> - <test-group FilePath="nested-open-index/index-leftouterjoin"> + <test-group name="nested-open-index/index-leftouterjoin"> <test-case FilePath="nested-open-index/index-leftouterjoin"> <compilation-unit name="probe-pidx-with-join-btree-sidx1"> <output-dir compare="Text">probe-pidx-with-join-btree-sidx1</output-dir> @@ -3345,7 +3360,7 @@ </compilation-unit> </test-case> </test-group> - <test-group FilePath="nested-open-index/index-selection"> + <test-group name="nested-open-index/index-selection"> <test-case FilePath="nested-open-index/index-selection"> <compilation-unit name="btree-index-composite-key"> <output-dir compare="Text">btree-index-composite-key</output-dir> @@ -3444,7 +3459,7 @@ </compilation-unit> </test-case> </test-group> - <test-group name="nested-open-index/highly-open-highly-nested"> + <test-group name="highly-open-highly-nested"> <test-case FilePath="nested-open-index/highly-open-highly-nested"> <compilation-unit name="bottom-closed-top-closed"> <output-dir compare="Text">bottom-closed-top-closed</output-dir> @@ -3468,7 +3483,7 @@ </test-group> </test-group> <test-group name="nested-index"> - <test-group FilePath="nested-index/index-join"> + <test-group name="nested-index/index-join"> <test-case FilePath="nested-index/index-join"> <compilation-unit name="btree-primary-equi-join"> <output-dir compare="Text">btree-primary-equi-join</output-dir> @@ -3515,7 +3530,7 @@ </compilation-unit> </test-case> </test-group> - <test-group FilePath="nested-index/index-leftouterjoin"> + <test-group name="nested-index/index-leftouterjoin"> <test-case FilePath="nested-index/index-leftouterjoin"> <compilation-unit name="probe-pidx-with-join-btree-sidx1"> <output-dir compare="Text">probe-pidx-with-join-btree-sidx1</output-dir> @@ -3547,7 +3562,7 @@ </compilation-unit> </test-case> </test-group> - <test-group FilePath="nested-index/index-selection"> + <test-group name="nested-index/index-selection"> <test-case FilePath="nested-index/index-selection"> <compilation-unit name="btree-index-composite-key"> <output-dir compare="Text">btree-index-composite-key</output-dir> @@ -6338,7 +6353,7 @@ </compilation-unit> </test-case> </test-group> - <test-group> + <test-group name="big-object"> <test-case FilePath="big-object"> <compilation-unit name="big_object_sort"> <output-dir compare="Text">big_object_sort</output-dir> diff --git a/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java b/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java index 0d8a953..b102e0b 100644 --- a/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java +++ b/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java @@ -47,7 +47,6 @@ import org.apache.asterix.testframework.context.TestCaseContext.OutputFormat; import org.apache.asterix.testframework.context.TestFileContext; import org.apache.asterix.testframework.xml.TestCase.CompilationUnit; -import org.apache.asterix.testframework.xml.TestCase.CompilationUnit.ExpectedError; import org.apache.asterix.testframework.xml.TestGroup; import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler; import org.apache.commons.httpclient.HttpClient; @@ -766,8 +765,8 @@ unExpectedFailure = true; } else { // Get the expected exception - ExpectedError expectedError = cUnit.getExpectedError().get(numOfErrors - 1); - if (e.toString().contains(expectedError.getValue())) { + String expectedError = cUnit.getExpectedError().get(numOfErrors - 1); + if (e.toString().contains(expectedError)) { System.err.println("...but that was expected."); } else { unExpectedFailure = true; diff --git a/asterix-test-framework/src/main/resources/Catalog.xsd b/asterix-test-framework/src/main/resources/Catalog.xsd index 80836a3..a18f41c 100644 --- a/asterix-test-framework/src/main/resources/Catalog.xsd +++ b/asterix-test-framework/src/main/resources/Catalog.xsd @@ -37,8 +37,6 @@ <xs:element ref="test:test-group" maxOccurs="unbounded"/> </xs:sequence> - <xs:attribute name="CatalogDesignDate" type="xs:date" use="required"/> - <xs:attribute name="ResultOffsetPath" type="test:SimplifiedRelativeFilePath" use="required"> <xs:annotation> <xs:documentation> @@ -52,6 +50,15 @@ <xs:annotation> <xs:documentation> offset from root to Query expression files + </xs:documentation> + </xs:annotation> + </xs:attribute> + + <xs:attribute name="QueryFileExtension" type="xs:string" + use="required"> + <xs:annotation> + <xs:documentation> + extension of query expression files </xs:documentation> </xs:annotation> </xs:attribute> @@ -70,7 +77,7 @@ <xs:simpleType name="SimplifiedRelativeFilePath"> <xs:restriction base="xs:anyURI"> - <xs:pattern value="([a-zA-Z0-9\-\.]+/)+"/> + <xs:pattern value="([a-zA-Z0-9\-_\.]+\/?)+"/> </xs:restriction> </xs:simpleType> @@ -111,12 +118,23 @@ <xs:complexType name="test-case"> <xs:sequence> - <xs:element name="description" type="test:description"/> + <xs:element name="description" type="test:description" minOccurs="0" /> <xs:element name="compilation-unit" minOccurs="1" maxOccurs="unbounded"> <xs:complexType> <xs:sequence> <xs:element name="description" type="test:description" minOccurs="0"/> + <xs:element name="parameter" minOccurs="0" maxOccurs="unbounded"> + <xs:annotation> + <xs:documentation> + Parameter to send to the HTTP API. + </xs:documentation> + </xs:annotation> + <xs:complexType> + <xs:attribute name="name" type="xs:string" /> + <xs:attribute name="value" type="xs:string" /> + </xs:complexType> + </xs:element> <xs:element name="output-dir" minOccurs="0"> <xs:annotation> <xs:documentation> @@ -131,33 +149,15 @@ </xs:simpleContent> </xs:complexType> </xs:element> - <xs:element name="parameter" minOccurs="0" maxOccurs="unbounded"> - <xs:annotation> - <xs:documentation> - Parameter to send to the HTTP API. - </xs:documentation> - </xs:annotation> - <xs:complexType> - <xs:attribute name="name" type="xs:string" /> - <xs:attribute name="value" type="xs:string" /> - </xs:complexType> - </xs:element> <!-- Zero or more expected errors for this query --> - <xs:element name="expected-error" minOccurs="0" maxOccurs="unbounded"> + <xs:element name="expected-error" type="xs:string" minOccurs="0" maxOccurs="unbounded"> <xs:annotation> <xs:documentation> Zero or more expected errors for this query. </xs:documentation> </xs:annotation> - - <xs:complexType> - <xs:simpleContent> - <xs:extension base="test:ErrorCode"> - </xs:extension> - </xs:simpleContent> - </xs:complexType> </xs:element> </xs:sequence> @@ -179,7 +179,6 @@ <!-- the QueryFileExtension --> <xs:attribute name="FilePath" type="test:SimplifiedRelativeFilePath" use="required"/> - <xs:attribute name="date" type="xs:date" use="required"/> <xs:attribute name="category" type="test:category-enum"/> </xs:complexType> -- To view, visit https://asterix-gerrit.ics.uci.edu/712 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I8fcf41c009f9faf51bc8bccc0c3c7217b7769121 Gerrit-PatchSet: 20 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Ildar Absalyamov <[email protected]> Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Ildar Absalyamov <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
