>From Peeyush Gupta <[email protected]>: Peeyush Gupta has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20427?usp=email )
Change subject: [ASTERIXDB-3652][STO] Fixing storage assembling issues ...................................................................... [ASTERIXDB-3652][STO] Fixing storage assembling issues - user model changes: no - storage format changes: no - interface changes: no Details: 1. Fixed incorrect setting of maxLevel 2. The inner array was missing when querying a nested array. Ext-ref: MB-68738 Change-Id: Ia9cab2d1471ea131e3e4f34dbc44c0df381e6e8a Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20427 Integration-Tests: Jenkins <[email protected]> Tested-by: Peeyush Gupta <[email protected]> Reviewed-by: Peeyush Gupta <[email protected]> --- A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.001.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.002.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.003.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/missing-inner-array/missing-null-array.003.adm M asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ArrayWithUnionValueAssembler.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/RepeatedPrimitiveValueAssembler.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java 9 files changed, 120 insertions(+), 14 deletions(-) Approvals: Peeyush Gupta: Looks good to me, approved; Verified Jenkins: Verified Objections: Anon. E. Moose #1000171: Violations found diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.001.ddl.sqlpp new file mode 100644 index 0000000..d151eb1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.001.ddl.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + DROP DATAVERSE test IF EXISTS; + CREATE DATAVERSE test; + + USE test; + + CREATE DATASET ColumnDataset + PRIMARY KEY (uid:int) WITH { + "storage-format": {"format": "column"} + }; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.002.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.002.update.sqlpp new file mode 100644 index 0000000..5d56f26 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.002.update.sqlpp @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +UPSERT INTO ColumnDataset( + { + "uid": 1, + "key_0": [ + [ + [] + ], + [] + ] + } +); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.003.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.003.query.sqlpp new file mode 100644 index 0000000..05ce765 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.003.query.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +SELECT * FROM ColumnDataset; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/missing-inner-array/missing-null-array.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/missing-inner-array/missing-null-array.003.adm new file mode 100644 index 0000000..73d2e0d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/missing-inner-array/missing-null-array.003.adm @@ -0,0 +1 @@ +{ "ColumnDataset": { "uid": 1, "key_0": [ [ [ ] ], [ ] ] } } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml index 7b9e4e6..e7d0c76 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml @@ -16601,6 +16601,11 @@ </compilation-unit> </test-case> <test-case FilePath="column"> + <compilation-unit name="assembly/missing-inner-array"> + <output-dir compare="Text">assembly/missing-inner-array</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="column"> <compilation-unit name="filter/ASTERIXDB-3499"> <output-dir compare="Text">filter/ASTERIXDB-3499</output-dir> </compilation-unit> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml index 5b230da..f48b00d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml @@ -85,6 +85,11 @@ </compilation-unit> </test-case> <test-case FilePath="column"> + <compilation-unit name="assembly/missing-inner-array"> + <output-dir compare="Text">assembly/missing-inner-array</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="column"> <compilation-unit name="filter/ASTERIXDB-3499"> <output-dir compare="Text">filter/ASTERIXDB-3499</output-dir> </compilation-unit> diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ArrayWithUnionValueAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ArrayWithUnionValueAssembler.java index dcd240b..c2d6c42 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ArrayWithUnionValueAssembler.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/ArrayWithUnionValueAssembler.java @@ -44,6 +44,11 @@ nonMissingValueAdded = true; numberOfAddedValues++; super.addValue(value); + if (numberOfAddedValues == numberOfUnionChildren) { + // Completed a chunk; since we saw a non-missing, just reset the counters + nonMissingValueAdded = false; + numberOfAddedValues = 0; + } } @Override @@ -51,17 +56,23 @@ nonMissingValueAdded = true; numberOfAddedValues++; super.addNull(value); + if (numberOfAddedValues == numberOfUnionChildren) { + // Completed a chunk; since we saw a non-missing, just reset the counters + nonMissingValueAdded = false; + numberOfAddedValues = 0; + } } @Override void addMissing() throws HyracksDataException { numberOfAddedValues++; - if (nonMissingValueAdded && numberOfAddedValues >= numberOfUnionChildren) { - nonMissingValueAdded = false; - numberOfAddedValues = numberOfAddedValues % numberOfUnionChildren; - } else if (numberOfAddedValues == numberOfUnionChildren) { - super.addMissing(); + if (numberOfAddedValues == numberOfUnionChildren) { + if (!nonMissingValueAdded) { + super.addMissing(); + } + // Reset for the next chunk numberOfAddedValues = 0; + nonMissingValueAdded = false; } } -} +} \ No newline at end of file diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/RepeatedPrimitiveValueAssembler.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/RepeatedPrimitiveValueAssembler.java index 8b44534..61d0741 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/RepeatedPrimitiveValueAssembler.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/assembler/RepeatedPrimitiveValueAssembler.java @@ -27,14 +27,15 @@ import com.fasterxml.jackson.databind.node.ObjectNode; final class RepeatedPrimitiveValueAssembler extends AbstractPrimitiveValueAssembler { + private int arrayDelegateLevels; private boolean arrayDelegate; - private int arrayLevel; RepeatedPrimitiveValueAssembler(int level, AssemblerInfo info, IColumnValuesReader reader, IValueGetter primitiveValue) { super(level, info, reader, primitiveValue); this.arrayDelegate = false; - arrayLevel = 0; + this.arrayDelegateLevels = 0; + } @Override @@ -70,7 +71,7 @@ public void setAsDelegate(int arrayLevel) { // This assembler is responsible for adding null values this.arrayDelegate = true; - this.arrayLevel = arrayLevel; + this.arrayDelegateLevels |= (1 << arrayLevel); } private void next() throws HyracksDataException { @@ -84,7 +85,7 @@ * (i.e., arrayDelegate is true) */ addNullToAncestor(reader.getLevel()); - } else if (reader.isMissing() && (arrayLevel == reader.getLevel() || reader.getLevel() + 1 == level)) { + } else if (reader.isMissing() && (isArrayDelegate(reader.getLevel()) || reader.getLevel() + 1 == level)) { /* * Add a missing item in either * - the array item is MISSING @@ -96,6 +97,10 @@ } } + private boolean isArrayDelegate(int level) { + return (arrayDelegateLevels & (1 << level)) != 0; + } + private ColumnarValueException createException() { ColumnarValueException e = new ColumnarValueException(); ObjectNode assemblerNode = e.createNode(getClass().getSimpleName()); @@ -107,4 +112,4 @@ return e; } -} +} \ No newline at end of file diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java index 62d45a4..e8c6d3e 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java @@ -118,9 +118,8 @@ } allMissing = false; try { - int actualLevel = BytesUtils.readZigZagVarInt(in); - maxLevel = ColumnValuesUtil.clearNullBit(nullBitMask, actualLevel); - nullBitMask = ColumnValuesUtil.getNullMask(actualLevel); + maxLevel = BytesUtils.readZigZagVarInt(in); + nullBitMask = ColumnValuesUtil.getNullMask(maxLevel); currentDefinitionLevels = definitionLevels.get(maxLevel); if (currentDefinitionLevels == null) { -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20427?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: asterixdb Gerrit-Branch: ionic Gerrit-Change-Id: Ia9cab2d1471ea131e3e4f34dbc44c0df381e6e8a Gerrit-Change-Number: 20427 Gerrit-PatchSet: 5 Gerrit-Owner: Ritik Raj <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Peeyush Gupta <[email protected]>
