Till Westmann has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/544
Change subject: merge release-0.8.8 ...................................................................... merge release-0.8.8 Change-Id: Ic12be5fffee10d2a4ff2a185f988b0bd2c2fe44f --- M README.md M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceMaterializationForInsertWithSelfScanRule.java M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java M asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java M asterix-app/data/csv/sample_01.csv M asterix-app/pom.xml M asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java M asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java M asterix-app/src/test/resources/optimizerts/results/disjunction-to-join-delete-2.plan R asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-collision/index-type-collision.1.ddl.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-promotion-collision/index-type-promotion-collision.1.ddl.aql R asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/record-type-collision/record-collision.1.ddl.aql A asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.ddl.aql A asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.update.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.3.ddl.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.4.query.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.5.ddl.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.6.query.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.1.ddl.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.2.update.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.3.ddl.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.4.query.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.5.query.aql C asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.6.query.aql A asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.adm A asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.adm A asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.1.adm A asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.2.adm A asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.3.adm M asterix-app/src/test/resources/runtimets/testsuite.xml M asterix-doc/src/site/markdown/aql/manual.md M asterix-installer/pom.xml M asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java A asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.1.script.aql A asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.2.ddl.aql A asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.3.script.aql C asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.4.query.aql A asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.5.script.aql A asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm A asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/external_index/create_and_start.sh A asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/external_index/stop_and_delete.sh A asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/external_index/stop_and_start.sh M asterix-installer/src/test/resources/transactionts/testsuite.xml M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java 44 files changed, 592 insertions(+), 285 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/44/544/1 diff --git a/README.md b/README.md index b4506a7..89cebd0 100644 --- a/README.md +++ b/README.md @@ -16,40 +16,51 @@ ! specific language governing permissions and limitations ! under the License. !--> -#AsterixDB +#Apache AsterixDB -AsterixDB is a BDMS (Big Data Management System) with a rich feature set that sets it apart from other Big Data platforms. Its feature set makes it well-suited to modern needs such as web data warehousing and social data storage and analysis. AsterixDB has: +Apache AsterixDB is a BDMS (Big Data Management System) with a rich feature set that sets it apart from +other Big Data platforms. Its feature set makes it well-suited to modern needs such as +web data warehousing and social data storage and analysis. + +Apache AsterixDB has: * A semistructured NoSQL style data model (ADM) resulting from extending JSON with object database ideas - * An expressive and declarative query language (AQL) that supports a broad range of queries and analysis over semistructured data + * An expressive and declarative query language (AQL) that supports + a broad range of queries and analysis over semistructured data * A parallel runtime query execution engine, Hyracks, that has been scale-tested on up to 1000+ cores and 500+ disks * Partitioned LSM-based data storage and indexing to support efficient ingestion and management of semistructured data - * Support for query access to externally stored data (e.g., data in HDFS) as well as to data stored natively by AsterixDB - * A rich set of primitive data types, including spatial and temporal data in addition to integer, floating point, and textual data + * Support for query access to externally stored data (e.g., data in HDFS) as well + as to data stored natively by Apache AsterixDB + * A rich set of primitive data types, including spatial and temporal data in addition + to integer, floating point, and textual data * Secondary indexing options that include B+ trees, R trees, and inverted keyword (exact and fuzzy) index types * Support for fuzzy and spatial queries as well as for more traditional parametric queries * Basic transactional (concurrency and recovery) capabilities akin to those of a NoSQL store -Learn more about AsterixDB at [http://asterixdb.ics.uci.edu/] (http://asterixdb.ics.uci.edu/) +Learn more about Apache AsterixDB at [http://asterixdb.ics.uci.edu/] (http://asterixdb.ics.uci.edu/) -##Building AsterixDB +##Building Apache AsterixDB -To build AsterixDB from source, you should have a platform with the following: +To build Apache AsterixDB from source, you should have a platform with the following: * A Unix-ish environment (Linux, OS X, will all do). * git * Maven 3.1.1 or newer. -* Java 7 or newer. +* Java 8 or newer. -Additionally to run all the integration tests you should be running `sshd` locally, and have passwordless ssh logins enabled for the account which is running the tests. +Additionally to run all the integration tests you should be running `sshd` locally, and have passwordless ssh +logins enabled for the account which is running the tests. ##Documentation -AsterixDB's official documentation resides at [http://asterixdb.ics.uci.edu/documentation/index.html] (http://asterixdb.ics.uci.edu/documentation/index.html). This is built from the maven project under `asterix-doc/` as a maven site. The documentation on the official website refers to the most stable release version, so for pre-release versions one should refer to the compiled documentation. - +Apache AsterixDB's official documentation resides at [https://ci.apache.org/projects/asterixdb/] (http://asterixdb.ics.uci.edu/documentation/index.html). +This is built from the maven project under `asterix-doc/` as a maven site. ##Support/Contact -If you have any questions, please feel free to ask on our mailing list, [us...@asterixdb.incubator.apache.org](mailto:us...@asterixdb.incubator.apache.org). Join the list by sending an email to [users-subscr...@asterixdb.incubator.apache.org](mailto:users-subscr...@asterixdb.incubator.apache.org). If you are interested in the internals or developement of AsterixDB, also please feel free to subscribe to our developer mailing list, [d...@asterixdb.incubator.apache.org](mailto:d...@asterixdb.incubator.apache.org), by sending an email to [dev-subscr...@asterixdb.incubator.apache.org](mailto:dev-subscr...@asterixdb.incubator.apache.org). +If you have any questions, please feel free to ask on our mailing list, [us...@asterixdb.incubator.apache.org](mailto:us...@asterixdb.incubator.apache.org). +Join the list by sending an email to [users-subscr...@asterixdb.incubator.apache.org](mailto:users-subscr...@asterixdb.incubator.apache.org). +If you are interested in the internals or developement of Apache AsterixDB, also please feel free to subscribe +to our developer mailing list, [d...@asterixdb.incubator.apache.org](mailto:d...@asterixdb.incubator.apache.org), by sending an email to [dev-subscr...@asterixdb.incubator.apache.org](mailto:dev-subscr...@asterixdb.incubator.apache.org). diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceMaterializationForInsertWithSelfScanRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceMaterializationForInsertWithSelfScanRule.java index adce8ce..43ec793 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceMaterializationForInsertWithSelfScanRule.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceMaterializationForInsertWithSelfScanRule.java @@ -42,6 +42,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.physical.MaterializePOperator; import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; +// TODO: Reconsider if materialization is needed in delete pipeline public class IntroduceMaterializationForInsertWithSelfScanRule implements IAlgebraicRewriteRule { @Override diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java index bb5f659..3760bbf 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java @@ -22,7 +22,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Set; import java.util.Stack; import org.apache.asterix.common.config.DatasetConfig.DatasetType; @@ -48,8 +51,10 @@ import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; +import org.apache.asterix.om.types.hierachy.ATypeHierarchy; import org.apache.asterix.om.util.NonTaggedFormatUtil; import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -71,6 +76,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind; import org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator; @@ -99,11 +105,14 @@ FunctionIdentifier fid = null; /** find the record variable */ - InsertDeleteOperator insertOp = (InsertDeleteOperator) op1; - ILogicalExpression recordExpr = insertOp.getPayloadExpression().getValue(); - List<LogicalVariable> recordVar = new ArrayList<LogicalVariable>(); + InsertDeleteOperator insertDeleteOp = (InsertDeleteOperator) op1; + ILogicalExpression recordExpr = insertDeleteOp.getPayloadExpression().getValue(); + LogicalVariable recordVar = null; + List<LogicalVariable> usedRecordVars = new ArrayList<>(); /** assume the payload is always a single variable expression */ - recordExpr.getUsedVariables(recordVar); + recordExpr.getUsedVariables(usedRecordVars); + if (usedRecordVars.size() == 1) + recordVar = usedRecordVars.get(0); /** * op2 is the assign operator which extract primary keys from the record @@ -111,7 +120,7 @@ */ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue(); - if (recordVar.size() == 0) { + if (recordVar == null) { /** * For the case primary key-assignment expressions are constant * expressions, find assign op that creates record to be @@ -134,9 +143,9 @@ } } AssignOperator assignOp2 = (AssignOperator) op2; - recordVar.addAll(assignOp2.getVariables()); + recordVar = assignOp2.getVariables().get(0); } - AqlDataSource datasetSource = (AqlDataSource) insertOp.getDataSource(); + AqlDataSource datasetSource = (AqlDataSource) insertDeleteOp.getDataSource(); AqlMetadataProvider mp = (AqlMetadataProvider) context.getMetadataProvider(); String dataverseName = datasetSource.getId().getDataverseName(); String datasetName = datasetSource.getId().getDatasourceName(); @@ -181,19 +190,6 @@ op0.getInputs().clear(); } - // Replicate Operator is applied only when doing the bulk-load. - AbstractLogicalOperator replicateOp = null; - - if (secondaryIndexTotalCnt > 1 && insertOp.isBulkload()) { - // Split the logical plan into "each secondary index update branch" - // to replicate each <PK,RECORD> pair. - replicateOp = new ReplicateOperator(secondaryIndexTotalCnt); - replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); - replicateOp.setExecutionMode(ExecutionMode.PARTITIONED); - context.computeAndSetTypeEnvironmentForOperator(replicateOp); - currentTop = replicateOp; - } - // Prepare filtering field information List<String> additionalFilteringField = ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterField(); List<LogicalVariable> additionalFilteringVars = null; @@ -205,7 +201,7 @@ additionalFilteringVars = new ArrayList<LogicalVariable>(); additionalFilteringAssignExpressions = new ArrayList<Mutable<ILogicalExpression>>(); additionalFilteringExpressions = new ArrayList<Mutable<ILogicalExpression>>(); - prepareVarAndExpression(additionalFilteringField, recType.getFieldNames(), recordVar.get(0), + prepareVarAndExpression(additionalFilteringField, recType.getFieldNames(), recordVar, additionalFilteringAssignExpressions, additionalFilteringVars, context); additionalFilteringAssign = new AssignOperator(additionalFilteringVars, additionalFilteringAssignExpressions); @@ -214,44 +210,61 @@ .add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(var))); } } + LogicalVariable enforcedRecordVar = recordVar; - // Iterate each secondary index and applying Index Update operations. - for (Index index : indexes) { - List<LogicalVariable> projectVars = new ArrayList<LogicalVariable>(); - VariableUtilities.getUsedVariables(op1, projectVars); - if (!index.isSecondaryIndex()) { - continue; - } - LogicalVariable enforcedRecordVar = recordVar.get(0); - hasSecondaryIndex = true; - //if the index is enforcing field types - if (index.isEnforcingKeyFileds()) { - try { - DatasetDataSource ds = (DatasetDataSource) (insertOp.getDataSource()); - ARecordType insertRecType = (ARecordType) ds.getSchemaTypes()[ds.getSchemaTypes().length - 1]; - LogicalVariable castVar = context.newVar(); - ARecordType enforcedType = createEnforcedType(insertRecType, index); + if (insertDeleteOp.getOperation() == Kind.INSERT) { + try { + DatasetDataSource ds = (DatasetDataSource) (insertDeleteOp.getDataSource()); + ARecordType insertRecType = (ARecordType) ds.getSchemaTypes()[ds.getSchemaTypes().length - 1]; + LogicalVariable castVar = context.newVar(); + ARecordType enforcedType = createEnforcedType(insertRecType, indexes); + if (!enforcedType.equals(insertRecType)) { //introduce casting to enforced type AbstractFunctionCallExpression castFunc = new ScalarFunctionCallExpression( FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.CAST_RECORD)); - castFunc.getArguments() - .add(new MutableObject<ILogicalExpression>(insertOp.getPayloadExpression().getValue())); + castFunc.getArguments().add( + new MutableObject<ILogicalExpression>(insertDeleteOp.getPayloadExpression().getValue())); TypeComputerUtilities.setRequiredAndInputTypes(castFunc, enforcedType, insertRecType); - AssignOperator newAssignOperator = new AssignOperator(castVar, + AssignOperator castedRecordAssignOperator = new AssignOperator(castVar, new MutableObject<ILogicalExpression>(castFunc)); - newAssignOperator.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); - currentTop = newAssignOperator; - //project out casted record - projectVars.add(castVar); + castedRecordAssignOperator.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); + currentTop = castedRecordAssignOperator; enforcedRecordVar = castVar; - context.computeAndSetTypeEnvironmentForOperator(newAssignOperator); - context.computeAndSetTypeEnvironmentForOperator(currentTop); recType = enforcedType; - } catch (AsterixException e) { - throw new AlgebricksException(e); + context.computeAndSetTypeEnvironmentForOperator(castedRecordAssignOperator); } + } catch (AsterixException e) { + throw new AlgebricksException(e); } + } + Set<LogicalVariable> projectVars = new HashSet<LogicalVariable>(); + VariableUtilities.getUsedVariables(op1, projectVars); + if (enforcedRecordVar != null) + projectVars.add(enforcedRecordVar); + ProjectOperator project = new ProjectOperator(new ArrayList<LogicalVariable>(projectVars)); + project.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); + context.computeAndSetTypeEnvironmentForOperator(project); + currentTop = project; + + // Replicate Operator is applied only when doing the bulk-load. + AbstractLogicalOperator replicateOp = null; + if (secondaryIndexTotalCnt > 1 && insertDeleteOp.isBulkload()) { + // Split the logical plan into "each secondary index update branch" + // to replicate each <PK,RECORD> pair. + replicateOp = new ReplicateOperator(secondaryIndexTotalCnt); + replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); + replicateOp.setExecutionMode(ExecutionMode.PARTITIONED); + context.computeAndSetTypeEnvironmentForOperator(replicateOp); + currentTop = replicateOp; + } + + // Iterate each secondary index and applying Index Update operations. + for (Index index : indexes) { + if (!index.isSecondaryIndex()) { + continue; + } + hasSecondaryIndex = true; List<List<String>> secondaryKeyFields = index.getKeyFieldNames(); List<IAType> secondaryKeyTypes = index.getKeyFieldTypes(); @@ -265,27 +278,24 @@ } AssignOperator assign = new AssignOperator(secondaryKeyVars, expressions); - ProjectOperator project = new ProjectOperator(projectVars); + ILogicalOperator filterOrAssignOp = null; if (additionalFilteringAssign != null) { - additionalFilteringAssign.getInputs().add(new MutableObject<ILogicalOperator>(project)); + filterOrAssignOp = additionalFilteringAssign; assign.getInputs().add(new MutableObject<ILogicalOperator>(additionalFilteringAssign)); } else { - assign.getInputs().add(new MutableObject<ILogicalOperator>(project)); + filterOrAssignOp = assign; } // Only apply replicate operator when doing bulk-load - if (secondaryIndexTotalCnt > 1 && insertOp.isBulkload()) - project.getInputs().add(new MutableObject<ILogicalOperator>(replicateOp)); + if (secondaryIndexTotalCnt > 1 && insertDeleteOp.isBulkload()) + filterOrAssignOp.getInputs().add(new MutableObject<ILogicalOperator>(replicateOp)); else - project.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); - - context.computeAndSetTypeEnvironmentForOperator(project); + filterOrAssignOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); if (additionalFilteringAssign != null) { context.computeAndSetTypeEnvironmentForOperator(additionalFilteringAssign); } - context.computeAndSetTypeEnvironmentForOperator(assign); currentTop = assign; @@ -304,7 +314,7 @@ // Introduce the TokenizeOperator only when doing bulk-load, // and index type is keyword or n-gram. - if (index.getIndexType() != IndexType.BTREE && insertOp.isBulkload()) { + if (index.getIndexType() != IndexType.BTREE && insertDeleteOp.isBulkload()) { // Check whether the index is length-partitioned or not. // If partitioned, [input variables to TokenizeOperator, @@ -350,14 +360,15 @@ // TokenizeOperator to tokenize [SK, PK] pairs TokenizeOperator tokenUpdate = new TokenizeOperator(dataSourceIndex, - insertOp.getPrimaryKeyExpressions(), secondaryExpressions, tokenizeKeyVars, - filterExpression, insertOp.getOperation(), insertOp.isBulkload(), isPartitioned, varTypes); + insertDeleteOp.getPrimaryKeyExpressions(), secondaryExpressions, tokenizeKeyVars, + filterExpression, insertDeleteOp.getOperation(), insertDeleteOp.isBulkload(), isPartitioned, + varTypes); tokenUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assign)); context.computeAndSetTypeEnvironmentForOperator(tokenUpdate); IndexInsertDeleteOperator indexUpdate = new IndexInsertDeleteOperator(dataSourceIndex, - insertOp.getPrimaryKeyExpressions(), tokenizeKeyExprs, filterExpression, - insertOp.getOperation(), insertOp.isBulkload()); + insertDeleteOp.getPrimaryKeyExpressions(), tokenizeKeyExprs, filterExpression, + insertDeleteOp.getOperation(), insertDeleteOp.isBulkload()); indexUpdate.setAdditionalFilteringExpressions(additionalFilteringExpressions); indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(tokenUpdate)); @@ -369,15 +380,15 @@ } else { // When TokenizeOperator is not needed IndexInsertDeleteOperator indexUpdate = new IndexInsertDeleteOperator(dataSourceIndex, - insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression, - insertOp.getOperation(), insertOp.isBulkload()); + insertDeleteOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression, + insertDeleteOp.getOperation(), insertDeleteOp.isBulkload()); indexUpdate.setAdditionalFilteringExpressions(additionalFilteringExpressions); indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); currentTop = indexUpdate; context.computeAndSetTypeEnvironmentForOperator(indexUpdate); - if (insertOp.isBulkload()) + if (insertDeleteOp.isBulkload()) op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); } @@ -417,14 +428,14 @@ context.getOutputTypeEnvironment(assignCoordinates), forceFilter); AqlIndex dataSourceIndex = new AqlIndex(index, dataverseName, datasetName, mp); IndexInsertDeleteOperator indexUpdate = new IndexInsertDeleteOperator(dataSourceIndex, - insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression, - insertOp.getOperation(), insertOp.isBulkload()); + insertDeleteOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression, + insertDeleteOp.getOperation(), insertDeleteOp.isBulkload()); indexUpdate.setAdditionalFilteringExpressions(additionalFilteringExpressions); indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assignCoordinates)); currentTop = indexUpdate; context.computeAndSetTypeEnvironmentForOperator(indexUpdate); - if (insertOp.isBulkload()) + if (insertDeleteOp.isBulkload()) op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); } @@ -434,78 +445,103 @@ return false; } - if (!insertOp.isBulkload()) { + if (!insertDeleteOp.isBulkload()) { op0.getInputs().clear(); op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop)); } return true; } - public static ARecordType createEnforcedType(ARecordType initialType, Index index) + // Merges typed index fields with specified recordType, allowing indexed fields to be optional. + // I.e. the type { "personId":int32, "name": string, "address" : { "street": string } } with typed indexes on age:int32, address.state:string + // will be merged into type { "personId":int32, "name": string, "age": int32? "address" : { "street": string, "state": string? } } + // Used by open indexes to enforce the type of an indexed record + public static ARecordType createEnforcedType(ARecordType initialType, List<Index> indexes) throws AsterixException, AlgebricksException { ARecordType enforcedType = initialType; - for (int i = 0; i < index.getKeyFieldNames().size(); i++) { - try { - Stack<Pair<ARecordType, String>> nestedTypeStack = new Stack<Pair<ARecordType, String>>(); - List<String> splits = index.getKeyFieldNames().get(i); - ARecordType nestedFieldType = enforcedType; - boolean openRecords = false; - String bridgeName = nestedFieldType.getTypeName(); - int j; - //Build the stack for the enforced type - for (j = 1; j < splits.size(); j++) { - nestedTypeStack.push(new Pair<ARecordType, String>(nestedFieldType, splits.get(j - 1))); - bridgeName = nestedFieldType.getTypeName(); - nestedFieldType = (ARecordType) enforcedType.getSubFieldType(splits.subList(0, j)); - if (nestedFieldType == null) { - openRecords = true; - break; + for (Index index : indexes) { + if (!index.isSecondaryIndex() || !index.isEnforcingKeyFileds()) { + continue; + } + for (int i = 0; i < index.getKeyFieldNames().size(); i++) { + try { + Stack<Pair<ARecordType, String>> nestedTypeStack = new Stack<Pair<ARecordType, String>>(); + List<String> splits = index.getKeyFieldNames().get(i); + ARecordType nestedFieldType = enforcedType; + boolean openRecords = false; + String bridgeName = nestedFieldType.getTypeName(); + int j; + //Build the stack for the enforced type + for (j = 1; j < splits.size(); j++) { + nestedTypeStack.push(new Pair<ARecordType, String>(nestedFieldType, splits.get(j - 1))); + bridgeName = nestedFieldType.getTypeName(); + nestedFieldType = (ARecordType) enforcedType.getSubFieldType(splits.subList(0, j)); + if (nestedFieldType == null) { + openRecords = true; + break; + } } - } - if (openRecords == true) { - //create the smallest record - enforcedType = new ARecordType(splits.get(splits.size() - 2), - new String[] { splits.get(splits.size() - 1) }, - new IAType[] { AUnionType.createNullableType(index.getKeyFieldTypes().get(i)) }, true); - //create the open part of the nested field - for (int k = splits.size() - 3; k > (j - 2); k--) { - enforcedType = new ARecordType(splits.get(k), new String[] { splits.get(k + 1) }, - new IAType[] { AUnionType.createNullableType(enforcedType) }, true); + if (openRecords == true) { + //create the smallest record + enforcedType = new ARecordType(splits.get(splits.size() - 2), + new String[] { splits.get(splits.size() - 1) }, + new IAType[] { AUnionType.createNullableType(index.getKeyFieldTypes().get(i)) }, true); + //create the open part of the nested field + for (int k = splits.size() - 3; k > (j - 2); k--) { + enforcedType = new ARecordType(splits.get(k), new String[] { splits.get(k + 1) }, + new IAType[] { AUnionType.createNullableType(enforcedType) }, true); + } + //Bridge the gap + Pair<ARecordType, String> gapPair = nestedTypeStack.pop(); + ARecordType parent = gapPair.first; + + IAType[] parentFieldTypes = ArrayUtils.addAll(parent.getFieldTypes().clone(), + new IAType[] { AUnionType.createNullableType(enforcedType) }); + enforcedType = new ARecordType(bridgeName, + ArrayUtils.addAll(parent.getFieldNames(), enforcedType.getTypeName()), parentFieldTypes, + true); + + } else { + //Schema is closed all the way to the field + //enforced fields are either null or strongly typed + LinkedHashMap<String, IAType> recordNameTypesMap = new LinkedHashMap<String, IAType>(); + for (j = 0; j < nestedFieldType.getFieldNames().length; j++) { + recordNameTypesMap.put(nestedFieldType.getFieldNames()[j], + nestedFieldType.getFieldTypes()[j]); + } + // if a an enforced field already exists and the type is correct + IAType enforcedFieldType = recordNameTypesMap.get(splits.get(splits.size() - 1)); + if (enforcedFieldType != null && enforcedFieldType.getTypeTag() == ATypeTag.UNION + && ((AUnionType) enforcedFieldType).isNullableType()) + enforcedFieldType = ((AUnionType) enforcedFieldType).getNullableType(); + if (enforcedFieldType != null && !ATypeHierarchy.canPromote(enforcedFieldType.getTypeTag(), + index.getKeyFieldTypes().get(i).getTypeTag())) + throw new AlgebricksException("Cannot enforce field " + index.getKeyFieldNames().get(i) + + " to have type " + index.getKeyFieldTypes().get(i)); + if (enforcedFieldType == null) + recordNameTypesMap.put(splits.get(splits.size() - 1), + AUnionType.createNullableType(index.getKeyFieldTypes().get(i))); + enforcedType = new ARecordType(nestedFieldType.getTypeName(), + recordNameTypesMap.keySet().toArray(new String[recordNameTypesMap.size()]), + recordNameTypesMap.values().toArray(new IAType[recordNameTypesMap.size()]), + nestedFieldType.isOpen()); } - //Bridge the gap - Pair<ARecordType, String> gapPair = nestedTypeStack.pop(); - ARecordType parent = gapPair.first; - IAType[] parentFieldTypes = ArrayUtils.addAll(parent.getFieldTypes().clone(), - new IAType[] { AUnionType.createNullableType(enforcedType) }); - enforcedType = new ARecordType(bridgeName, - ArrayUtils.addAll(parent.getFieldNames(), enforcedType.getTypeName()), parentFieldTypes, - true); - - } else { - //Schema is closed all the way to the field - //enforced fields are either null or strongly typed - enforcedType = new ARecordType(nestedFieldType.getTypeName(), - ArrayUtils.addAll(nestedFieldType.getFieldNames(), splits.get(splits.size() - 1)), - ArrayUtils.addAll(nestedFieldType.getFieldTypes(), - AUnionType.createNullableType(index.getKeyFieldTypes().get(i))), - nestedFieldType.isOpen()); - } - - //Create the enforcedtype for the nested fields in the schema, from the ground up - if (nestedTypeStack.size() > 0) { - while (!nestedTypeStack.isEmpty()) { - Pair<ARecordType, String> nestedTypePair = nestedTypeStack.pop(); - ARecordType nestedRecType = nestedTypePair.first; - IAType[] nestedRecTypeFieldTypes = nestedRecType.getFieldTypes().clone(); - nestedRecTypeFieldTypes[nestedRecType.getFieldIndex(nestedTypePair.second)] = enforcedType; - enforcedType = new ARecordType(nestedRecType.getTypeName(), nestedRecType.getFieldNames(), - nestedRecTypeFieldTypes, nestedRecType.isOpen()); + //Create the enforcedtype for the nested fields in the schema, from the ground up + if (nestedTypeStack.size() > 0) { + while (!nestedTypeStack.isEmpty()) { + Pair<ARecordType, String> nestedTypePair = nestedTypeStack.pop(); + ARecordType nestedRecType = nestedTypePair.first; + IAType[] nestedRecTypeFieldTypes = nestedRecType.getFieldTypes().clone(); + nestedRecTypeFieldTypes[nestedRecType + .getFieldIndex(nestedTypePair.second)] = enforcedType; + enforcedType = new ARecordType(nestedRecType.getTypeName() + "_enforced", + nestedRecType.getFieldNames(), nestedRecTypeFieldTypes, nestedRecType.isOpen()); + } } + } catch (IOException e) { + throw new AsterixException(e); } - - } catch (IOException e) { - throw new AsterixException(e); } } return enforcedType; diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java index d4825c4..d595e2d 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java @@ -66,8 +66,8 @@ // added by yasser public static class CompiledCreateDataverseStatement implements ICompiledStatement { - private String dataverseName; - private String format; + private final String dataverseName; + private final String format; public CompiledCreateDataverseStatement(String dataverseName, String format) { this.dataverseName = dataverseName; @@ -89,7 +89,7 @@ } public static class CompiledNodeGroupDropStatement implements ICompiledStatement { - private String nodeGroupName; + private final String nodeGroupName; public CompiledNodeGroupDropStatement(String nodeGroupName) { this.nodeGroupName = nodeGroupName; @@ -106,9 +106,9 @@ } public static class CompiledIndexDropStatement implements ICompiledStatement { - private String dataverseName; - private String datasetName; - private String indexName; + private final String dataverseName; + private final String datasetName; + private final String indexName; public CompiledIndexDropStatement(String dataverseName, String datasetName, String indexName) { this.dataverseName = dataverseName; @@ -135,8 +135,8 @@ } public static class CompiledDataverseDropStatement implements ICompiledStatement { - private String dataverseName; - private boolean ifExists; + private final String dataverseName; + private final boolean ifExists; public CompiledDataverseDropStatement(String dataverseName, boolean ifExists) { this.dataverseName = dataverseName; @@ -158,7 +158,7 @@ } public static class CompiledTypeDropStatement implements ICompiledStatement { - private String typeName; + private final String typeName; public CompiledTypeDropStatement(String nodeGroupName) { this.typeName = nodeGroupName; @@ -247,11 +247,11 @@ } public static class CompiledLoadFromFileStatement implements ICompiledDmlStatement { - private String dataverseName; - private String datasetName; - private boolean alreadySorted; - private String adapter; - private Map<String, String> properties; + private final String dataverseName; + private final String datasetName; + private final boolean alreadySorted; + private final String adapter; + private final Map<String, String> properties; public CompiledLoadFromFileStatement(String dataverseName, String datasetName, String adapter, Map<String, String> properties, boolean alreadySorted) { @@ -328,12 +328,12 @@ } public static class CompiledConnectFeedStatement implements ICompiledDmlStatement { - private String dataverseName; - private String feedName; - private String datasetName; - private String policyName; + private final String dataverseName; + private final String feedName; + private final String datasetName; + private final String policyName; private Query query; - private int varCounter; + private final int varCounter; public CompiledConnectFeedStatement(String dataverseName, String feedName, String datasetName, String policyName, Query query, int varCounter) { @@ -423,9 +423,9 @@ } public static class CompiledDisconnectFeedStatement implements ICompiledDmlStatement { - private String dataverseName; - private String datasetName; - private String feedName; + private final String dataverseName; + private final String datasetName; + private final String feedName; private Query query; private int varCounter; diff --git a/asterix-app/data/csv/sample_01.csv b/asterix-app/data/csv/sample_01.csv index 4dd437a..fbba382 100644 --- a/asterix-app/data/csv/sample_01.csv +++ b/asterix-app/data/csv/sample_01.csv @@ -1,8 +1,8 @@ -1,0.899682764,5.6256,2013-08-07,07:22:35,1979-02-25T23:48:27.034 -2,0.669052398,,-1923-03-29,19:33:34,-1979-02-25T23:48:27.002 -3,0.572733058,192674,-1923-03-28,19:33:34,-1979-02-25T23:48:27.001 -4,,192674,-1923-03-27,19:33:34,-1979-02-25T23:48:27.001 -5,0.572733058,192674,,19:33:34,-1979-02-25T23:48:27.001 -6,0.572733058,192674,-1923-03-25,,-1979-02-25T23:48:27.001 -7,0.572733058,192674,-1923-03-24,19:33:34, +1,0.899682764,5.6256,2013-08-07,07:22:35,1979-02-25T23:48:27.034 +2,0.669052398,,-1923-03-29,19:33:34,-1979-02-25T23:48:27.002 +3,0.572733058,192674,-1923-03-28,19:33:34,-1979-02-25T23:48:27.001 +4,,192674,-1923-03-27,19:33:34,-1979-02-25T23:48:27.001 +5,0.572733058,192674,,19:33:34,-1979-02-25T23:48:27.001 +6,0.572733058,192674,-1923-03-25,,-1979-02-25T23:48:27.001 +7,0.572733058,192674,-1923-03-24,19:33:34, 8,,,,, \ No newline at end of file diff --git a/asterix-app/pom.xml b/asterix-app/pom.xml index 66382e6..0360529 100644 --- a/asterix-app/pom.xml +++ b/asterix-app/pom.xml @@ -83,6 +83,18 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>jar</goal> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java index 51e1612..06921cd 100644 --- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java +++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java @@ -39,6 +39,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.google.common.collect.Lists; import org.apache.asterix.api.common.APIFramework; import org.apache.asterix.api.common.SessionConfig; import org.apache.asterix.api.common.SessionConfig.OutputFormat; @@ -968,7 +969,7 @@ ARecordType enforcedType = null; if (stmtCreateIndex.isEnforced()) { - enforcedType = IntroduceSecondaryIndexInsertDeleteRule.createEnforcedType(aRecordType, index); + enforcedType = IntroduceSecondaryIndexInsertDeleteRule.createEnforcedType(aRecordType, Lists.newArrayList(index)); } //#. prepare to create the index artifact in NC. @@ -2400,6 +2401,9 @@ "There is no dataset with this name " + datasetName + " in dataverse " + dataverseName + "."); } + + + String itemTypeName = ds.getItemTypeName(); Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(), dataverseName, itemTypeName); @@ -2431,7 +2435,7 @@ ARecordType enforcedType = null; if (cics.isEnforced()) { enforcedType = IntroduceSecondaryIndexInsertDeleteRule.createEnforcedType(aRecordType, - indexes.get(j)); + indexes); } jobsToExecute.add(IndexOperations.buildSecondaryIndexCompactJobSpec(cics, aRecordType, enforcedType, metadataProvider, ds)); diff --git a/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java b/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java index 9961dc8..a3be1cf 100644 --- a/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java +++ b/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java @@ -60,25 +60,27 @@ * Called prior to running the Runtime test suite. */ public void setup() throws Exception { - conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml")); - conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml")); - conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml")); + setup(""); + } + + public void setup(String basePath) throws Exception { + conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/core-site.xml")); + conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/mapred-site.xml")); + conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/hdfs-site.xml")); cleanupLocal(); - //this constructor is deprecated in hadoop 2x - //dfsCluster = new MiniDFSCluster(nameNodePort, conf, numDataNodes, true, true, StartupOption.REGULAR, null); MiniDFSCluster.Builder build = new MiniDFSCluster.Builder(conf); build.nameNodePort(nameNodePort); build.numDataNodes(numDataNodes); build.startupOption(StartupOption.REGULAR); dfsCluster = build.build(); dfs = FileSystem.get(conf); - loadData(); + loadData(basePath); } - private void loadData() throws IOException { + private void loadData(String localDataRoot) throws IOException { Path destDir = new Path(HDFS_PATH); dfs.mkdirs(destDir); - File srcDir = new File(DATA_PATH); + File srcDir = new File(localDataRoot + DATA_PATH); File[] listOfFiles = srcDir.listFiles(); for (File srcFile : listOfFiles) { Path path = new Path(srcFile.getAbsolutePath()); @@ -99,5 +101,4 @@ cleanupLocal(); } } - } diff --git a/asterix-app/src/test/resources/optimizerts/results/disjunction-to-join-delete-2.plan b/asterix-app/src/test/resources/optimizerts/results/disjunction-to-join-delete-2.plan index 25b3396..4b9e8a2 100644 --- a/asterix-app/src/test/resources/optimizerts/results/disjunction-to-join-delete-2.plan +++ b/asterix-app/src/test/resources/optimizerts/results/disjunction-to-join-delete-2.plan @@ -9,7 +9,7 @@ -- INSERT_DELETE |PARTITIONED| -- ONE_TO_ONE_EXCHANGE |PARTITIONED| -- MATERIALIZE |PARTITIONED| - -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + -- HASH_PARTITION_EXCHANGE [$$8] |PARTITIONED| -- ASSIGN |PARTITIONED| -- STREAM_PROJECT |PARTITIONED| -- STREAM_SELECT |PARTITIONED| diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-collision/index-type-collision.1.ddl.aql similarity index 87% rename from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql rename to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-collision/index-type-collision.1.ddl.aql index 853386d..84d35b7 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-collision/index-type-collision.1.ddl.aql @@ -21,9 +21,9 @@ use dataverse test; create type testType as open { - "id": int32, - "value": string + "id": int32 } create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +create index testIdx1 on testDS(value: int32) enforced; +create index testIdx2 on testDS(value: string) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-promotion-collision/index-type-promotion-collision.1.ddl.aql similarity index 87% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-promotion-collision/index-type-promotion-collision.1.ddl.aql index 853386d..6c47032 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/index-type-promotion-collision/index-type-promotion-collision.1.ddl.aql @@ -21,9 +21,9 @@ use dataverse test; create type testType as open { - "id": int32, - "value": string + "id": int32 } create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +create index testIdx1 on testDS(value: int64) enforced; +create index testIdx2 on testDS(value: int32) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-type-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/record-type-collision/record-collision.1.ddl.aql similarity index 100% rename from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-type-collision/enforced-field-name-collision.1.ddl.aql rename to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/record-type-collision/record-collision.1.ddl.aql diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.ddl.aql new file mode 100644 index 0000000..10a3530 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.ddl.aql @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Test that BTree open index is used in query plan + * : define the BTree open index on a composite key (fname,lanme) + * : predicate => where $l.fname="Julio" and $l.lname="Isa" + * Expected Result : Success + * Issue : Issue 162 + * Date : 27th March 2014 + */ + +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type Emp as closed { +id:int64, +fname:string, +lname:string, +age:int64, +dept:string +} + +create type EmpOpen as open { +id:int64, +fname:string, +age:int64, +dept:string +} + +create dataset employee(Emp) primary key id; + +create dataset employeeOpen(EmpOpen) primary key id; + diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.update.aql new file mode 100644 index 0000000..d28adff --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.update.aql @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Test that BTree enforced open index is used in query plan + * : define the BTree enforced open index on a composite key (fname,lanme) + * : predicate => where $l.fname="Julio" and $l.lname="Isa" + * Expected Result : Success + * Issue : Issue 162 + * Date : 27th March 2014 + */ + +use dataverse test; + +load dataset employee +using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter" +(("path"="nc1://data/names.adm"),("format"="delimited-text"),("delimiter"="|")); + +insert into dataset employeeOpen ( + for $x in dataset employee return $x +); diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.3.ddl.aql similarity index 60% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.3.ddl.aql index 853386d..11b0baa 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.3.ddl.aql @@ -16,14 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; +/* + * Description : Test that BTree enforced open index is used in query plan + * : define the BTree enforced open index on a composite key (fname,lanme) + * : predicate => where $l.fname="Julio" and $l.lname="Isa" + * Expected Result : Success + * Issue : Issue 162 + * Date : 27th March 2014 + */ + use dataverse test; -create type testType as open { - "id": int32, - "value": string -} +// create secondary index -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +create index idx_employee_f_l_name on employeeOpen(fname,lname:string) enforced; +create index idx_employee_l_f_name on employeeOpen(lname:string, fname) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.4.query.aql similarity index 60% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.4.query.aql index 853386d..b1f2adf 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.4.query.aql @@ -16,14 +16,23 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; +/* + * Description : Test that BTree enforced open index is used in query plan + * : define the BTree enforced open index on a composite key (fname,lanme) + * : predicate => where $l.fname="Julio" and $l.lname="Isa" + * Expected Result : Success + * Issue : Issue 162 + * Date : 27th March 2014 + */ + use dataverse test; -create type testType as open { - "id": int32, - "value": string +for $l in dataset('employeeOpen') +where $l.fname="Julio" and $l.lname="Isa" +return { + "id": $l.id, + "fname": $l.fname, + "lname": $l.lname, + "age": $l.age, + "dept": $l.dept } - -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.5.ddl.aql similarity index 65% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.5.ddl.aql index 853386d..4eaf8e8 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.5.ddl.aql @@ -16,14 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; +/* + * Description : Test that BTree enforced open index is used in query plan + * : define the BTree enforced open index on a composite key (fname,lanme) + * : predicate => where $l.fname="Julio" and $l.lname="Isa" + * Expected Result : Success + * Issue : Issue 162 + * Date : 27th March 2014 + */ + use dataverse test; -create type testType as open { - "id": int32, - "value": string -} +// create secondary index -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +drop index employeeOpen.idx_employee_f_l_name; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.6.query.aql similarity index 60% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.6.query.aql index 853386d..b1f2adf 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.6.query.aql @@ -16,14 +16,23 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; +/* + * Description : Test that BTree enforced open index is used in query plan + * : define the BTree enforced open index on a composite key (fname,lanme) + * : predicate => where $l.fname="Julio" and $l.lname="Isa" + * Expected Result : Success + * Issue : Issue 162 + * Date : 27th March 2014 + */ + use dataverse test; -create type testType as open { - "id": int32, - "value": string +for $l in dataset('employeeOpen') +where $l.fname="Julio" and $l.lname="Isa" +return { + "id": $l.id, + "fname": $l.fname, + "lname": $l.lname, + "age": $l.age, + "dept": $l.dept } - -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.1.ddl.aql similarity index 68% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.1.ddl.aql index 853386d..0efffb2 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.1.ddl.aql @@ -20,10 +20,24 @@ create dataverse test; use dataverse test; -create type testType as open { - "id": int32, - "value": string +create type DBLPType as closed { + id: int64, + dblpid: string, + title: string, + authors: string, + misc: string } -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +create type DBLPOpenType as open { + id: int64, + dblpid: string, + authors: string, + misc: string +} + +create nodegroup group1 if not exists on nc1, nc2; + +create dataset DBLP(DBLPType) + primary key id on group1; +create dataset DBLPOpen(DBLPOpenType) + primary key id on group1; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.2.update.aql similarity index 62% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.2.update.aql index 853386d..3ee5d4e 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.2.update.aql @@ -16,14 +16,25 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; use dataverse test; -create type testType as open { - "id": int32, - "value": string -} +load dataset DBLP +using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter" +(("path"="nc1://data/dblp-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":")) pre-sorted; -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +insert into dataset test.DBLPOpen ( + for $x in dataset test.DBLP + where $x.id <= 50 + return $x +); + +insert into dataset test.DBLPOpen ( + for $c in dataset test.DBLP + where $c.id > 50 + return { + "id": $c.id, + "dblpid": $c.dblpid, + "authors": $c.authors, + "misc": $c.misc + } +); diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.3.ddl.aql similarity index 78% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.3.ddl.aql index 853386d..98338b2 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.3.ddl.aql @@ -16,14 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; + use dataverse test; -create type testType as open { - "id": int32, - "value": string -} - -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; +create index ngram_index on DBLPOpen(title:string) type ngram(3) enforced; +create index keyword_index on DBLPOpen(title:string) type keyword enforced; +create index btree_index on DBLPOpen(title:string) enforced; \ No newline at end of file diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.4.query.aql similarity index 78% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.4.query.aql index 853386d..775e97d 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.4.query.aql @@ -16,14 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; use dataverse test; -create type testType as open { - "id": int32, - "value": string +for $o in dataset('DBLPOpen') +where contains($o.title, "Multimedia") +order by $o.id +return { + "id": $o.id, + "dblpid": $o.dblpid, + "title": $o.title, + "authors": $o.authors, + "misc": $o.misc } -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.5.query.aql similarity index 74% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.5.query.aql index 853386d..b7423e2 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.5.query.aql @@ -16,14 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; + use dataverse test; -create type testType as open { - "id": int32, - "value": string +for $o in dataset('DBLPOpen') +let $jacc := similarity-jaccard-check(word-tokens($o.title), word-tokens("Transactions for Cooperative Environments"), 0.5f) +where $jacc[0] +return { + "id": $o.id, + "dblpid": $o.dblpid, + "title": $o.title, + "authors": $o.authors, + "misc": $o.misc } - -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.6.query.aql similarity index 77% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.6.query.aql index 853386d..0e1ef29 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/index-selection/multi-index/multi-index.6.query.aql @@ -16,14 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; use dataverse test; -create type testType as open { - "id": int32, - "value": string +for $o in dataset('DBLPOpen') +where $o.title = "Multimedia Information Systems Issues and Approaches." +order by $o.title +return { + "id": $o.id, + "dblpid": $o.dblpid, + "title": $o.title, + "authors": $o.authors, + "misc": $o.misc } - -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; diff --git a/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.adm b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.adm new file mode 100644 index 0000000..3a3fb2f --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.1.adm @@ -0,0 +1 @@ +{ "id": 881, "fname": "Julio", "age": 38, "dept": "Sales", "lname": "Isa" } diff --git a/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.adm b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.adm new file mode 100644 index 0000000..3a3fb2f --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index-composite-key/multi-index-composite-key.2.adm @@ -0,0 +1 @@ +{ "id": 881, "fname": "Julio", "age": 38, "dept": "Sales", "lname": "Isa" } diff --git a/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.1.adm b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.1.adm new file mode 100644 index 0000000..5ef01a1 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.1.adm @@ -0,0 +1 @@ +{ "id": 4, "dblpid": "books/acm/kim95/ChristodoulakisK95", "authors": "Stavros Christodoulakis Leonidas Koveos", "misc": "2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95", "title": "Multimedia Information Systems Issues and Approaches." } diff --git a/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.2.adm b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.2.adm new file mode 100644 index 0000000..5f1f708 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.2.adm @@ -0,0 +1 @@ +{ "id": 9, "dblpid": "books/acm/kim95/Kaiser95", "authors": "Gail E. Kaiser", "misc": "2002-01-03 409-433 1995 Modern Database Systems db/books/collections/kim95.html#Kaiser95", "title": "Cooperative Transactions for Multiuser Environments." } diff --git a/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.3.adm b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.3.adm new file mode 100644 index 0000000..5ef01a1 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/results/open-index-enforced/index-selection/multi-index/multi-index.3.adm @@ -0,0 +1 @@ +{ "id": 4, "dblpid": "books/acm/kim95/ChristodoulakisK95", "authors": "Stavros Christodoulakis Leonidas Koveos", "misc": "2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95", "title": "Multimedia Information Systems Issues and Approaches." } diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml index 430fd8e..c6264aa 100644 --- a/asterix-app/src/test/resources/runtimets/testsuite.xml +++ b/asterix-app/src/test/resources/runtimets/testsuite.xml @@ -2960,15 +2960,22 @@ <test-group name="open-index-enforced"> <test-group FilePath="open-index-enforced/error-checking"> <test-case FilePath="open-index-enforced/error-checking"> - <compilation-unit name="enforced-field-name-collision"> - <output-dir compare="Text">enforced-field-name-collision</output-dir> + <compilation-unit name="index-on-closed-type"> + <output-dir compare="Text">index-on-closed-type</output-dir> <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException </expected-error> </compilation-unit> </test-case> <test-case FilePath="open-index-enforced/error-checking"> - <compilation-unit name="enforced-field-type-collision"> - <output-dir compare="Text">enforced-field-type-collision</output-dir> + <compilation-unit name="index-type-collision"> + <output-dir compare="Text">index-type-collision</output-dir> + <expected-error>org.apache.asterix.common.exceptions.AsterixException + </expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="open-index-enforced/error-checking"> + <compilation-unit name="index-type-promotion-collision"> + <output-dir compare="Text">index-type-promotion-collision</output-dir> <expected-error>org.apache.asterix.common.exceptions.AsterixException</expected-error> </compilation-unit> </test-case> @@ -2980,10 +2987,9 @@ </compilation-unit> </test-case> <test-case FilePath="open-index-enforced/error-checking"> - <compilation-unit name="index-on-closed-type"> - <output-dir compare="Text">index-on-closed-type</output-dir> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException - </expected-error> + <compilation-unit name="record-type-collision"> + <output-dir compare="Text">record-type-collision</output-dir> + <expected-error>org.apache.asterix.common.exceptions.AsterixException</expected-error> </compilation-unit> </test-case> </test-group> @@ -3118,6 +3124,16 @@ </compilation-unit> </test-case> <test-case FilePath="open-index-enforced/index-selection"> + <compilation-unit name="multi-index"> + <output-dir compare="Text">multi-index</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="open-index-enforced/index-selection"> + <compilation-unit name="multi-index-composite-key"> + <output-dir compare="Text">multi-index-composite-key</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="open-index-enforced/index-selection"> <compilation-unit name="orders-index-custkey"> <output-dir compare="Text">orders-index-custkey</output-dir> </compilation-unit> diff --git a/asterix-doc/src/site/markdown/aql/manual.md b/asterix-doc/src/site/markdown/aql/manual.md index af45f76..883e285 100644 --- a/asterix-doc/src/site/markdown/aql/manual.md +++ b/asterix-doc/src/site/markdown/aql/manual.md @@ -793,7 +793,7 @@ drop dataset FacebookUsers if exists; - drop index fbSenderLocIndex; + drop index FacebookUsers.fbSenderLocIndex; drop type FacebookUserType; diff --git a/asterix-installer/pom.xml b/asterix-installer/pom.xml index 61345a3..2c80677 100644 --- a/asterix-installer/pom.xml +++ b/asterix-installer/pom.xml @@ -253,6 +253,13 @@ <type>test-jar</type> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.asterix</groupId> + <artifactId>asterix-app</artifactId> + <version>0.8.8-SNAPSHOT</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> <dependency> <groupId>org.apache.asterix</groupId> <artifactId>asterix-server</artifactId> diff --git a/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java b/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java index edf6e5e..61cc5cc 100644 --- a/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java +++ b/asterix-installer/src/test/java/org/apache/asterix/installer/transaction/RecoveryIT.java @@ -26,6 +26,7 @@ import java.util.logging.Logger; import org.apache.asterix.test.aql.TestExecutor; +import org.apache.asterix.test.runtime.HDFSCluster; import org.apache.asterix.testframework.context.TestCaseContext; import org.apache.commons.io.FileUtils; import org.junit.AfterClass; @@ -41,6 +42,7 @@ private static final Logger LOGGER = Logger.getLogger(RecoveryIT.class.getName()); private static final String PATH_ACTUAL = "rttest/"; private static final String PATH_BASE = "src/test/resources/transactionts/"; + private static final String HDFS_BASE = "../asterix-app/"; private TestCaseContext tcCtx; private static File asterixInstallerPath; private static File installerTargetPath; @@ -79,6 +81,7 @@ scriptHomePath + File.separator + "setup_teardown" + File.separator + "configure_and_validate.sh"); TestExecutor.executeScript(pb, scriptHomePath + File.separator + "setup_teardown" + File.separator + "stop_and_delete.sh"); + HDFSCluster.getInstance().setup(HDFS_BASE); } @AfterClass @@ -92,6 +95,7 @@ scriptHomePath + File.separator + "setup_teardown" + File.separator + "stop_and_delete.sh"); TestExecutor.executeScript(pb, scriptHomePath + File.separator + "setup_teardown" + File.separator + "shutdown.sh"); + HDFSCluster.getInstance().cleanup(); } @Parameters diff --git a/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.1.script.aql b/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.1.script.aql new file mode 100644 index 0000000..cc46136 --- /dev/null +++ b/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.1.script.aql @@ -0,0 +1 @@ +create_and_start.sh diff --git a/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.2.ddl.aql b/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.2.ddl.aql new file mode 100644 index 0000000..7aa1129 --- /dev/null +++ b/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.2.ddl.aql @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* +* Description : Create an external dataset that contains records stored with text hdfs file format. + Build an index over the external dataset age attribute + Perform a query over the dataset using the index. +* Expected Res : Success +* Date : 3rd Jan 2014 +*/ +drop dataverse test if exists; +create dataverse test; + +use dataverse test; + +create type EmployeeType as closed { + id: int64, + name: string, + age: int64 +}; + +create external dataset EmployeeDataset(EmployeeType) +using hdfs +(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/external-indexing-test.txt"),("input-format"="text-input-format"),("format"="delimited-text"),("delimiter"="|")); + +create index EmployeeAgeIdx on EmployeeDataset(age); + diff --git a/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.3.script.aql b/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.3.script.aql new file mode 100644 index 0000000..3ba1dc0 --- /dev/null +++ b/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.3.script.aql @@ -0,0 +1 @@ +stop_and_start.sh diff --git a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql b/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.4.query.aql similarity index 67% copy from asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql copy to asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.4.query.aql index 853386d..edfb290 100644 --- a/asterix-app/src/test/resources/runtimets/queries/open-index-enforced/error-checking/enforced-field-name-collision/enforced-field-name-collision.1.ddl.aql +++ b/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.4.query.aql @@ -16,14 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -drop dataverse test if exists; -create dataverse test; +/* +* Description : Create an external dataset that contains records stored with text hdfs file format. + Build an index over the external dataset age attribute + Perform a query over the dataset using the index. +* Expected Res : Success +* Date : 3rd Jan 2014 +*/ use dataverse test; -create type testType as open { - "id": int32, - "value": string -} +for $emp in dataset EmployeeDataset +where $emp.age = 22 +order by $emp.id +return $emp; -create dataset testDS(testType) primary key id; -create index testIdx on testDS(value: string) enforced; diff --git a/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.5.script.aql b/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.5.script.aql new file mode 100644 index 0000000..10e1a51 --- /dev/null +++ b/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.5.script.aql @@ -0,0 +1 @@ +stop_and_delete.sh diff --git a/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm b/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm new file mode 100644 index 0000000..36b836c --- /dev/null +++ b/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm @@ -0,0 +1,4 @@ +[ { "id": 3, "name": "Samuel", "age": 22 } +, { "id": 10, "name": "David", "age": 22 } + ] + diff --git a/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/external_index/create_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/external_index/create_and_start.sh new file mode 100755 index 0000000..945f01d --- /dev/null +++ b/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/external_index/create_and_start.sh @@ -0,0 +1 @@ +$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml; diff --git a/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/external_index/stop_and_delete.sh b/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/external_index/stop_and_delete.sh new file mode 100755 index 0000000..d7deea3 --- /dev/null +++ b/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/external_index/stop_and_delete.sh @@ -0,0 +1,3 @@ +$MANAGIX_HOME/bin/managix stop -n nc1; +$MANAGIX_HOME/bin/managix delete -n nc1; + diff --git a/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/external_index/stop_and_start.sh b/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/external_index/stop_and_start.sh new file mode 100755 index 0000000..1271a2b --- /dev/null +++ b/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/external_index/stop_and_start.sh @@ -0,0 +1,2 @@ +$MANAGIX_HOME/bin/managix stop -n nc1; +$MANAGIX_HOME/bin/managix start -n nc1; diff --git a/asterix-installer/src/test/resources/transactionts/testsuite.xml b/asterix-installer/src/test/resources/transactionts/testsuite.xml index cf95132..e3774f8 100644 --- a/asterix-installer/src/test/resources/transactionts/testsuite.xml +++ b/asterix-installer/src/test/resources/transactionts/testsuite.xml @@ -18,6 +18,13 @@ !--> <test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries" QueryFileExtension=".aql"> + <test-group name="query_after_restart"> + <test-case FilePath="query_after_restart"> + <compilation-unit name="external_index"> + <output-dir compare="Text">external_index</output-dir> + </compilation-unit> + </test-case> + </test-group> <test-group name="recover_after_abort"> <test-case FilePath="recover_after_abort"> diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java index d31dfcf..0dc1013 100644 --- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java +++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java @@ -709,7 +709,7 @@ } Pair<IBinaryComparatorFactory[], ITypeTraits[]> comparatorFactoriesAndTypeTraits = getComparatorFactoriesAndTypeTraitsOfSecondaryBTreeIndex( secondaryIndex.getIndexType(), secondaryIndex.getKeyFieldNames(), - secondaryIndex.getKeyFieldTypes(), DatasetUtils.getPartitioningKeys(dataset), itemType); + secondaryIndex.getKeyFieldTypes(), DatasetUtils.getPartitioningKeys(dataset), itemType, dataset.getDatasetType()); comparatorFactories = comparatorFactoriesAndTypeTraits.first; typeTraits = comparatorFactoriesAndTypeTraits.second; if (filterTypeTraits != null) { @@ -811,7 +811,7 @@ private Pair<IBinaryComparatorFactory[], ITypeTraits[]> getComparatorFactoriesAndTypeTraitsOfSecondaryBTreeIndex( IndexType indexType, List<List<String>> sidxKeyFieldNames, List<IAType> sidxKeyFieldTypes, - List<List<String>> pidxKeyFieldNames, ARecordType recType) throws AlgebricksException { + List<List<String>> pidxKeyFieldNames, ARecordType recType, DatasetType dsType) throws AlgebricksException { IBinaryComparatorFactory[] comparatorFactories; ITypeTraits[] typeTraits; @@ -833,8 +833,17 @@ for (int j = 0; j < pidxKeyFieldCount; ++j, ++i) { IAType keyType = null; try { - keyType = recType.getSubFieldType(pidxKeyFieldNames.get(j)); - } catch (IOException e) { + switch (dsType) { + case INTERNAL: + keyType = recType.getSubFieldType(pidxKeyFieldNames.get(j)); + break; + case EXTERNAL: + keyType = IndexingConstants.getFieldType(j); + break; + default: + throw new AlgebricksException("Unknown Dataset Type"); + } + } catch (IOException| AsterixException e) { throw new AlgebricksException(e); } comparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, -- To view, visit https://asterix-gerrit.ics.uci.edu/544 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ic12be5fffee10d2a4ff2a185f988b0bd2c2fe44f Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Till Westmann <ti...@apache.org>
