Wail Alkowaileet has uploaded a new change for review. https://asterix-gerrit.ics.uci.edu/3140
Change subject: [ASTERIXDB-2491][TXN] Fix(hack) to 16-bit offset overflow ...................................................................... [ASTERIXDB-2491][TXN] Fix(hack) to 16-bit offset overflow - user model changes: no - storage format changes: no - interface changes: no Details: - Add unsigned short deserializer - Try to detect overflowed offsets and try to ensure offsets are monotonically increasing. - As a last resort, it sets the last field's (payload's) end offset as the size of the value. - Add recovery test (passed 100KB record) Change-Id: I2ab49af8ce7c4753404dbb38ac58e28f75cae8fe --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/LogRecord.java A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.1.script.aql A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.2.ddl.aql A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.3.update.aql A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.4.txnqbc.aql A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.5.script.aql A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.6.script.aql A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.7.txnqar.aql A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.8.script.aql A asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/create_and_start.sh A asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/kill_cc_and_nc.sh A asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_delete.sh A asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_start.sh M asterixdb/asterix-server/src/test/resources/transactionts/testsuite.xml M hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/ShortPointable.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleReference.java M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleWriter.java 17 files changed, 342 insertions(+), 19 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/40/3140/1 diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/LogRecord.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/LogRecord.java index 7a1079d..c8da50a 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/LogRecord.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/LogRecord.java @@ -415,8 +415,8 @@ if (srcBuffer.position() + size > srcBuffer.limit()) { throw new BufferUnderflowException(); } + destTuple.resetByTupleOffset(srcBuffer.array(), srcBuffer.position(), size); destTuple.setFieldCount(fieldCnt); - destTuple.resetByTupleOffset(srcBuffer.array(), srcBuffer.position()); srcBuffer.position(srcBuffer.position() + size); return destTuple; } diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.1.script.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.1.script.aql new file mode 100644 index 0000000..7d441cd --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.1.script.aql @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +create_and_start.sh \ No newline at end of file diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.2.ddl.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.2.ddl.aql new file mode 100644 index 0000000..02f3a72 --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.2.ddl.aql @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Test case Name : large_object_100K + * Description : ASTERIXDB-2491 (Recovery fails for objects with size larger than 32KB) + * Expected Result : Success + * Date : December 25 2018 + */ + +drop dataverse recovery if exists; +create dataverse recovery; +use dataverse recovery; + +create type RecoveryType as { + uid: uuid, + numbers: [int] +}; + +create dataset RecoveryDataset (RecoveryType) +primary key uid autogenerated; diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.3.update.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.3.update.aql new file mode 100644 index 0000000..1e24bcf --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.3.update.aql @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Test case Name : large_object_100K + * Description : ASTERIXDB-2491 (Recovery fails for objects with size larger than 32KB) + * Expected Result : Success + * Date : December 25 2018 + */ + +use dataverse recovery; + +//Create a 100KB record (8-bytes * 12800 = 102400 bytes) +insert into dataset RecoveryDataset ( + {"numbers": range(1, 12800)} +); \ No newline at end of file diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.4.txnqbc.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.4.txnqbc.aql new file mode 100644 index 0000000..2e19c39 --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.4.txnqbc.aql @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Test case Name : large_object_100K + * Description : ASTERIXDB-2491 (Recovery fails for objects with size larger than 32KB) + * Expected Result : Success + * Date : December 25 2018 + */ + +use dataverse recovery; + +for $x in dataset RecoveryDataset +return count($x.numbers); \ No newline at end of file diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.5.script.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.5.script.aql new file mode 100644 index 0000000..4583455 --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.5.script.aql @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +kill_cc_and_nc.sh \ No newline at end of file diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.6.script.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.6.script.aql new file mode 100644 index 0000000..7087cd3 --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.6.script.aql @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +stop_and_start.sh \ No newline at end of file diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.7.txnqar.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.7.txnqar.aql new file mode 100644 index 0000000..2e19c39 --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.7.txnqar.aql @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Test case Name : large_object_100K + * Description : ASTERIXDB-2491 (Recovery fails for objects with size larger than 32KB) + * Expected Result : Success + * Date : December 25 2018 + */ + +use dataverse recovery; + +for $x in dataset RecoveryDataset +return count($x.numbers); \ No newline at end of file diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.8.script.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.8.script.aql new file mode 100644 index 0000000..40df6fb --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.8.script.aql @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +stop_and_delete.sh \ No newline at end of file diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/create_and_start.sh b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/create_and_start.sh new file mode 100755 index 0000000..e358618 --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/create_and_start.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +$NCSERVICE_HOME/opt/local/bin/start-sample-cluster.sh 1>/dev/null 2>&1; diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/kill_cc_and_nc.sh b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/kill_cc_and_nc.sh new file mode 100755 index 0000000..b6326cc --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/kill_cc_and_nc.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +ps -ef | awk '/java.*org\.apache\.hyracks\.control\.[cn]c\.[CN]CDriver/ {print $2}' | xargs -n 1 kill -9 +ps -ef | awk '/java.*org\.apache\.hyracks\.control\.nc\.service\.NCService/ {print $2}' | xargs -n 1 kill -9 diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_delete.sh b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_delete.sh new file mode 100755 index 0000000..818d17d --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_delete.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +$NCSERVICE_HOME/opt/local/bin/stop-sample-cluster.sh; +rm -rf $NCSERVICE_HOME/opt/local/data; + diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_start.sh b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_start.sh new file mode 100755 index 0000000..9a0c506 --- /dev/null +++ b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_start.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +$NCSERVICE_HOME/opt/local/bin/stop-sample-cluster.sh; +$NCSERVICE_HOME/opt/local/bin/start-sample-cluster.sh; diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/testsuite.xml b/asterixdb/asterix-server/src/test/resources/transactionts/testsuite.xml index 38179b2..b00a14a 100644 --- a/asterixdb/asterix-server/src/test/resources/transactionts/testsuite.xml +++ b/asterixdb/asterix-server/src/test/resources/transactionts/testsuite.xml @@ -158,9 +158,15 @@ </test-case> <test-case FilePath="recover_after_abort"> - <compilation-unit name="primary_plus_multiple_secondary_indices"><!-- The only exception here is during the kill command which is in a different JVM, hence not caught --> + <compilation-unit name="primary_plus_multiple_secondary_indices">The only exception here is during the kill command which is in a different JVM, hence not caught <output-dir compare="Text">primary_plus_multiple_secondary_indices</output-dir> - <!-- <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error> --> + <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error> + </compilation-unit> + </test-case> + + <test-case FilePath="recover_after_abort"> + <compilation-unit name="large_object_100K"> + <output-dir compare="Text">large_object_100K</output-dir> </compilation-unit> </test-case> </test-group> diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/ShortPointable.java b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/ShortPointable.java index 73f3779..a684792 100644 --- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/ShortPointable.java +++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/ShortPointable.java @@ -64,6 +64,10 @@ return (short) (((bytes[start] & 0xff) << 8) + (bytes[start + 1] & 0xff)); } + public static int getUnsignedShort(byte[] bytes, int start) { + return getShort(bytes, start) & 0xFFFF; + } + public static void setShort(byte[] bytes, int start, short value) { bytes[start] = (byte) ((value >>> 8) & 0xFF); bytes[start + 1] = (byte) ((value >>> 0) & 0xFF); diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleReference.java index e82b037..90d35dc 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleReference.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleReference.java @@ -23,20 +23,30 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.util.BitOperationUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class SimpleTupleReference implements ITreeIndexTupleReference { - + private static final Logger LOGGER = LogManager.getLogger(); protected byte[] buf; protected int fieldStartIndex; protected int fieldCount; protected int tupleStartOff; protected int nullFlagsBytes; protected int fieldSlotsBytes; + private int[] fixedEndOffsets; + private int size; @Override public void resetByTupleOffset(byte[] buf, int tupleStartOff) { this.buf = buf; this.tupleStartOff = tupleStartOff; + } + + public void resetByTupleOffset(byte[] buf, int tupleStartOff, int size) { + this.buf = buf; + this.tupleStartOff = tupleStartOff; + this.size = size; } @Override @@ -50,6 +60,7 @@ nullFlagsBytes = getNullFlagsBytes(); fieldSlotsBytes = getFieldSlotsBytes(); fieldStartIndex = 0; + fixEndOffsets(); } @Override @@ -71,10 +82,9 @@ @Override public int getFieldLength(int fIdx) { if (fIdx == 0) { - return ShortPointable.getShort(buf, tupleStartOff + nullFlagsBytes); + return fixedEndOffsets[0]; } else { - return ShortPointable.getShort(buf, tupleStartOff + nullFlagsBytes + fIdx * 2) - - ShortPointable.getShort(buf, tupleStartOff + nullFlagsBytes + ((fIdx - 1) * 2)); + return fixedEndOffsets[fIdx] - fixedEndOffsets[fIdx - 1]; } } @@ -83,8 +93,7 @@ if (fIdx == 0) { return tupleStartOff + nullFlagsBytes + fieldSlotsBytes; } else { - return tupleStartOff + nullFlagsBytes + fieldSlotsBytes - + ShortPointable.getShort(buf, tupleStartOff + nullFlagsBytes + ((fIdx - 1) * 2)); + return tupleStartOff + nullFlagsBytes + fieldSlotsBytes + fixedEndOffsets[fIdx - 1]; } } @@ -98,7 +107,39 @@ @Override public int getTupleSize() { - return nullFlagsBytes + fieldSlotsBytes - + ShortPointable.getShort(buf, tupleStartOff + nullFlagsBytes + (fieldCount - 1) * 2); + return nullFlagsBytes + fieldSlotsBytes + fixedEndOffsets[fieldCount - 1]; + } + + private int getCorruptedFieldEndOffset(int fIdx) { + return ShortPointable.getUnsignedShort(buf, tupleStartOff + nullFlagsBytes + fIdx * 2); + } + + /* + * This is a hack to fix the 16-bit overflow issue [ASTERIXDB-2491] + * It won't detect double overflow or if the first endOffset overflowed. + * Hence, we need a more comprehensive fix. + */ + private void fixEndOffsets() { + if (fixedEndOffsets == null || fixedEndOffsets.length < fieldCount) { + fixedEndOffsets = new int[fieldCount]; + } + + fixedEndOffsets[0] = getCorruptedFieldEndOffset(0); + boolean isOverflowed = false; + for (int i = 1; i < fieldCount; i++) { + fixedEndOffsets[i] = getCorruptedFieldEndOffset(i); + if (isOverflowed || fixedEndOffsets[i - 1] > fixedEndOffsets[i]) { + fixedEndOffsets[i] += fixedEndOffsets[i - 1]; + isOverflowed = true; + } + } + + //it assumes that the payload (the largest field) is at the end + if (fixedEndOffsets[fieldCount - 1] != size) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Double overflow detected in log record tuple"); + } + fixedEndOffsets[fieldCount - 1] = size; + } } } diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleWriter.java index 410a0e3..e87832c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleWriter.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; +import org.apache.hyracks.data.std.primitive.ShortPointable; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.util.BitOperationUtils; @@ -33,12 +34,6 @@ public static final SimpleTupleWriter INSTANCE = new SimpleTupleWriter(); private SimpleTupleWriter() { - } - - // Write short in little endian to target byte array at given offset. - private static void writeShortL(short s, byte[] buf, int targetOff) { - buf[targetOff] = (byte) (s >> 8); - buf[targetOff + 1] = (byte) (s >> 0); } @Override @@ -83,7 +78,7 @@ System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf, runner, tuple.getFieldLength(i)); fieldEndOff += tuple.getFieldLength(i); runner += tuple.getFieldLength(i); - writeShortL((short) fieldEndOff, targetBuf, targetOff + nullFlagsBytes + i * 2); + ShortPointable.setShort(targetBuf, targetOff + nullFlagsBytes + i * 2, (short) fieldEndOff); } return runner - targetOff; } @@ -103,7 +98,7 @@ System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf, runner, tuple.getFieldLength(i)); fieldEndOff += tuple.getFieldLength(i); runner += tuple.getFieldLength(i); - writeShortL((short) fieldEndOff, targetBuf, targetOff + nullFlagsBytes + fieldCounter * 2); + ShortPointable.setShort(targetBuf, targetOff + nullFlagsBytes + fieldCounter * 2, (short) fieldEndOff); fieldCounter++; } -- To view, visit https://asterix-gerrit.ics.uci.edu/3140 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I2ab49af8ce7c4753404dbb38ac58e28f75cae8fe Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Wail Alkowaileet <wael....@gmail.com>