>From Ritik Raj <[email protected]>: Ritik Raj has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20427?usp=email )
Change subject: [ASTERIXDB-3652][STO] Fixing storage assembling issues - user model changes: no - storage format changes: no - interface changes: no ...................................................................... [ASTERIXDB-3652][STO] Fixing storage assembling issues - user model changes: no - storage format changes: no - interface changes: no Details: 1. Fixed incorrect setting of maxLevel 2. The inner array was missing when querying a nested array. Change-Id: Ia9cab2d1471ea131e3e4f34dbc44c0df381e6e8a --- M asterixdb/asterix-app/src/test/resources/cc-main.conf A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.001.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.002.update.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.003.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/missing-inner-array/missing-null-array.003.adm M asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/RepeatedPrimitiveColumnValuesReader.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java 11 files changed, 99 insertions(+), 24 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/27/20427/1 diff --git a/asterixdb/asterix-app/src/test/resources/cc-main.conf b/asterixdb/asterix-app/src/test/resources/cc-main.conf index 623c82a..cdc3e6b 100644 --- a/asterixdb/asterix-app/src/test/resources/cc-main.conf +++ b/asterixdb/asterix-app/src/test/resources/cc-main.conf @@ -19,16 +19,16 @@ txn.log.dir=target/tmp/asterix_nc1/txnlog core.dump.dir=target/tmp/asterix_nc1/coredump iodevices=target/tmp/asterix_nc1/iodevice1, -; iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice2 +iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice2 nc.api.port=19004 #jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006 -; [nc/asterix_nc2] -; ncservice.port=9091 -; txn.log.dir=target/tmp/asterix_nc2/txnlog -; core.dump.dir=target/tmp/asterix_nc2/coredump -; iodevices=target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2 -; nc.api.port=19005 +[nc/asterix_nc2] +ncservice.port=9091 +txn.log.dir=target/tmp/asterix_nc2/txnlog +core.dump.dir=target/tmp/asterix_nc2/coredump +iodevices=target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2 +nc.api.port=19005 #jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007 [nc] diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.001.ddl.sqlpp new file mode 100644 index 0000000..d151eb1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.001.ddl.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + DROP DATAVERSE test IF EXISTS; + CREATE DATAVERSE test; + + USE test; + + CREATE DATASET ColumnDataset + PRIMARY KEY (uid:int) WITH { + "storage-format": {"format": "column"} + }; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.002.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.002.update.sqlpp new file mode 100644 index 0000000..5d56f26 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.002.update.sqlpp @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +UPSERT INTO ColumnDataset( + { + "uid": 1, + "key_0": [ + [ + [] + ], + [] + ] + } +); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.003.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.003.query.sqlpp new file mode 100644 index 0000000..05ce765 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/assembly/missing-inner-array/missing-inner-array.003.query.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +SELECT * FROM ColumnDataset; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/missing-inner-array/missing-null-array.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/missing-inner-array/missing-null-array.003.adm new file mode 100644 index 0000000..73d2e0d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/assembly/missing-inner-array/missing-null-array.003.adm @@ -0,0 +1 @@ +{ "ColumnDataset": { "uid": 1, "key_0": [ [ [ ] ], [ ] ] } } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml index 7b9e4e6..e7d0c76 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml @@ -16601,6 +16601,11 @@ </compilation-unit> </test-case> <test-case FilePath="column"> + <compilation-unit name="assembly/missing-inner-array"> + <output-dir compare="Text">assembly/missing-inner-array</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="column"> <compilation-unit name="filter/ASTERIXDB-3499"> <output-dir compare="Text">filter/ASTERIXDB-3499</output-dir> </compilation-unit> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml index 5b230da..f48b00d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_single_partition_sqlpp.xml @@ -85,6 +85,11 @@ </compilation-unit> </test-case> <test-case FilePath="column"> + <compilation-unit name="assembly/missing-inner-array"> + <output-dir compare="Text">assembly/missing-inner-array</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="column"> <compilation-unit name="filter/ASTERIXDB-3499"> <output-dir compare="Text">filter/ASTERIXDB-3499</output-dir> </compilation-unit> diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java index 8bc6827..e8c6d3e 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java @@ -119,7 +119,6 @@ allMissing = false; try { maxLevel = BytesUtils.readZigZagVarInt(in); - // maxLevel = ColumnValuesUtil.clearNullBit(nullBitMask, actualLevel); nullBitMask = ColumnValuesUtil.getNullMask(maxLevel); currentDefinitionLevels = definitionLevels.get(maxLevel); diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/RepeatedPrimitiveColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/RepeatedPrimitiveColumnValuesReader.java index 1043533..0f3b817 100644 --- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/RepeatedPrimitiveColumnValuesReader.java +++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/RepeatedPrimitiveColumnValuesReader.java @@ -63,10 +63,6 @@ return false; } - if (columnIndex == 14289) { - System.out.println("waiting for debugger..."); - } - consumeDelimiterIfAny(); nextLevel(); setDelimiterIndex(); diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java index f71053e..1e3fe75 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java @@ -48,7 +48,7 @@ CLOUD_STORAGE_REGION(STRING, ""), CLOUD_STORAGE_ENDPOINT(STRING, ""), CLOUD_STORAGE_ANONYMOUS_AUTH(BOOLEAN, false), - CLOUD_STORAGE_CACHE_POLICY(STRING, "lazy"), + CLOUD_STORAGE_CACHE_POLICY(STRING, "selective"), // 80% of the total disk space CLOUD_STORAGE_ALLOCATION_PERCENTAGE(DOUBLE, 0.8d), // 90% of the allocated space for storage (i.e., 90% of the 80% of the total disk space) diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java index 3e88402..916b48c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeBulkloader.java @@ -264,10 +264,6 @@ try { columnarFrame.flush(columnWriter, tupleCount, maxColumnsInPageZerothSegment, lowKey.getTuple(), splitKey.getTuple(), this); - logNode.removeAll(); - logNode.put("leafCount", numberOfLeafNodes); - columnarFrame.dumpBuffer(logNode); - LOGGER.debug("Wrote leaf page: {}", logNode); updatePageLayoutStat(); } catch (Exception e) { logState(e); @@ -293,8 +289,6 @@ } } - ObjectNode logNode = JSONUtil.createObject(); - @Override protected void writeFullLeafPage() throws HyracksDataException { try { @@ -332,11 +326,6 @@ write(c); } - logNode.removeAll(); - logNode.put("leafCount", numberOfLeafNodes); - columnarFrame.dumpBuffer(logNode); - LOGGER.debug("Wrote leaf page: {}", logNode); - // For logging maxNumberOfPagesInALeafNode = Math.max(maxNumberOfPagesInALeafNode, numberOfPagesInCurrentLeafNode); maxTupleCount = Math.max(maxTupleCount, tupleCount); -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20427?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: newchange Gerrit-Project: asterixdb Gerrit-Branch: ionic Gerrit-Change-Id: Ia9cab2d1471ea131e3e4f34dbc44c0df381e6e8a Gerrit-Change-Number: 20427 Gerrit-PatchSet: 1 Gerrit-Owner: Ritik Raj <[email protected]>
