This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push: new 53bd906 IGNITE-14557: Improve row layout. (#100) 53bd906 is described below commit 53bd90611bc76cd4450c67404058ad85847b98c5 Author: Andrew V. Mashenkov <amashen...@users.noreply.github.com> AuthorDate: Wed May 5 19:31:58 2021 +0300 IGNITE-14557: Improve row layout. (#100) --- .../apache/ignite/internal/schema/BinaryRow.java | 8 +- .../ignite/internal/schema/ExpandableByteBuf.java | 4 +- .../org/apache/ignite/internal/schema/README.md | 16 +- .../org/apache/ignite/internal/schema/Row.java | 19 +- .../ignite/internal/schema/RowAssembler.java | 77 +-- .../marshaller/asm/AsmSerializerGenerator.java | 2 +- .../marshaller/reflection/JavaSerializer.java | 2 +- .../ignite/internal/schema/RowAssemblerTest.java | 678 +++++++++++++++++++++ .../org/apache/ignite/internal/schema/RowTest.java | 2 +- 9 files changed, 747 insertions(+), 61 deletions(-) diff --git a/modules/schema/src/main/java/org/apache/ignite/internal/schema/BinaryRow.java b/modules/schema/src/main/java/org/apache/ignite/internal/schema/BinaryRow.java index a1ef776..776504a 100644 --- a/modules/schema/src/main/java/org/apache/ignite/internal/schema/BinaryRow.java +++ b/modules/schema/src/main/java/org/apache/ignite/internal/schema/BinaryRow.java @@ -29,13 +29,13 @@ public interface BinaryRow { /** */ int SCHEMA_VERSION_OFFSET = 0; /** */ - int FLAGS_FIELD_OFFSET = SCHEMA_VERSION_OFFSET + 2; + int FLAGS_FIELD_OFFSET = SCHEMA_VERSION_OFFSET + 2 /* version length */; /** */ - int KEY_HASH_FIELD_OFFSET = FLAGS_FIELD_OFFSET + 2; + int KEY_HASH_FIELD_OFFSET = FLAGS_FIELD_OFFSET + 2 /* flags length */; /** */ - int KEY_CHUNK_OFFSET = KEY_HASH_FIELD_OFFSET + 4; + int KEY_CHUNK_OFFSET = KEY_HASH_FIELD_OFFSET + 4 /* hash length */; /** */ - int TOTAL_LEN_FIELD_SIZE = 4; + int CHUNK_LEN_FIELD_SIZE = 4; /** */ int VARLEN_TABLE_SIZE_FIELD_SIZE = 2; /** */ diff --git a/modules/schema/src/main/java/org/apache/ignite/internal/schema/ExpandableByteBuf.java b/modules/schema/src/main/java/org/apache/ignite/internal/schema/ExpandableByteBuf.java index eeae002..d74d719 100644 --- a/modules/schema/src/main/java/org/apache/ignite/internal/schema/ExpandableByteBuf.java +++ b/modules/schema/src/main/java/org/apache/ignite/internal/schema/ExpandableByteBuf.java @@ -63,7 +63,7 @@ public class ExpandableByteBuf { */ public ExpandableByteBuf(int size) { if (size <= 0) - size = 16; + size = 32; arr = new byte[size]; buf = ByteBuffer.wrap(arr); @@ -249,7 +249,7 @@ public class ExpandableByteBuf { * * @param cap Target capacity. */ - private void ensureCapacity(int cap) { + void ensureCapacity(int cap) { if (arr.length < cap) expand(cap); diff --git a/modules/schema/src/main/java/org/apache/ignite/internal/schema/README.md b/modules/schema/src/main/java/org/apache/ignite/internal/schema/README.md index 97fea08..1909628 100644 --- a/modules/schema/src/main/java/org/apache/ignite/internal/schema/README.md +++ b/modules/schema/src/main/java/org/apache/ignite/internal/schema/README.md @@ -57,14 +57,14 @@ Row structure has the following format: Each chunk section has the following structure: - ┌──────────────────────────────────────────────────┐ - │ │ - ┌─────────┬─────────────────────────┬────────┴────────┬─────────────────────────┬──────────┬────⌄─────┐ - │ Full │ Varsize Columns Offsets │ Varsize Columns │ Null-Defaults │ Fixsize │ Varsize │ - │ Size │ Table Size │ Offsets Table │ Map │ Columns │ Columns │ - ├─────────┼─────────────────────────┼─────────────────┼─────────────────────────┼──────────┼──────────┤ - │ 4 Bytes │ 2 Bytes │ Variable │ ⌈Number of columns / 8⌉ │ Variable │ Variable │ - └─────────┴─────────────────────────┴─────────────────┴─────────────────────────┴──────────┴──────────┘ + ┌────────────────────────┐ + │ │ + ┌─────────┬─────────────────────────┬─────────────────────────┬────────┴────────┬──────────┬────⌄─────┐ + │ Full │ Null-Defaults │ Varsize Columns Offsets │ Varsize Columns │ Fixsize │ Varsize │ + │ Size │ Map │ Table Size │ Offsets Table │ Columns │ Columns │ + ├─────────┼─────────────────────────┼─────────────────────────┼─────────────────┼──────────┼──────────┤ + │ 4 Bytes │ ⌈Number of columns / 8⌉ │ 2 Bytes │ Variable │ Variable │ Variable │ + └─────────┴─────────────────────────┴─────────────────────────┴─────────────────┴──────────┴──────────┘ All columns within a group are split into groups of fixed-size columns and variable-size columns. Withing the group of fixsize columns, the columns are sorted by size, then by column name. Within the group of varsize columns, the columns are sorted by column name. Inside a row default values and nulls are omitted and encoded in the null-defaults map diff --git a/modules/schema/src/main/java/org/apache/ignite/internal/schema/Row.java b/modules/schema/src/main/java/org/apache/ignite/internal/schema/Row.java index a074733..621b4b8 100644 --- a/modules/schema/src/main/java/org/apache/ignite/internal/schema/Row.java +++ b/modules/schema/src/main/java/org/apache/ignite/internal/schema/Row.java @@ -429,9 +429,9 @@ public class Row implements BinaryRow { } idx -= cols.numberOfFixsizeColumns() + numNullsBefore; - int vartableSize = readShort(baseOff + TOTAL_LEN_FIELD_SIZE); + int vartableSize = readShort(vartableChunkOffset(baseOff, cols)); - int vartableOff = vartableOffset(baseOff); + int vartableOff = vartableChunkOffset(baseOff, cols) + VARLEN_TABLE_SIZE_FIELD_SIZE; // Offset of idx-th column is from base offset. int resOff = readShort(vartableOff + VARLEN_COLUMN_OFFSET_FIELD_SIZE * idx); @@ -470,7 +470,11 @@ public class Row implements BinaryRow { off += cols.foldFixedLength(nullMapIdx, readByte(nullMapOff + nullMapIdx) | mask); - return nullMapOff + cols.nullMapSize() + off; + final int vartableChunkOffset = vartableChunkOffset(baseOff, cols); + + return vartableChunkOffset + VARLEN_TABLE_SIZE_FIELD_SIZE + + readShort(vartableChunkOffset) * VARLEN_COLUMN_OFFSET_FIELD_SIZE /* table size */ + + off; } /** @@ -478,17 +482,16 @@ public class Row implements BinaryRow { * @return Null map offset from the row start for the chunk with the given base. */ private int nullMapOffset(int baseOff) { - int varlenTblSize = readShort(baseOff + TOTAL_LEN_FIELD_SIZE); - - return vartableOffset(baseOff) + varlenTblSize * VARLEN_COLUMN_OFFSET_FIELD_SIZE; + return baseOff + CHUNK_LEN_FIELD_SIZE; } /** * @param baseOff Chunk base offset. + * @param cols Columns. * @return Offset of the varlen table from the row start for the chunk with the given base. */ - private int vartableOffset(int baseOff) { - return baseOff + TOTAL_LEN_FIELD_SIZE + VARLEN_TABLE_SIZE_FIELD_SIZE; + private int vartableChunkOffset(int baseOff, Columns cols) { + return baseOff + CHUNK_LEN_FIELD_SIZE + cols.nullMapSize(); } /** {@inheritDoc} */ diff --git a/modules/schema/src/main/java/org/apache/ignite/internal/schema/RowAssembler.java b/modules/schema/src/main/java/org/apache/ignite/internal/schema/RowAssembler.java index a42fe90..8994ab1 100644 --- a/modules/schema/src/main/java/org/apache/ignite/internal/schema/RowAssembler.java +++ b/modules/schema/src/main/java/org/apache/ignite/internal/schema/RowAssembler.java @@ -30,7 +30,7 @@ import java.util.UUID; * unnecessary data copies. The assembler provides some utility methods to calculate the resulting row size * based on the number of null columns and size calculation for strings. * - * @see #rowChunkSize(Columns, int, int, Columns, int, int) + * @see #rowSize(Columns, int, int, Columns, int, int) * @see #rowChunkSize(Columns, int, int) * @see #utf8EncodedLength(CharSequence) */ @@ -76,7 +76,7 @@ public class RowAssembler { * @return Total size of the varlen table. */ public static int varlenTableSize(int nonNullVarlenCols) { - return nonNullVarlenCols * 2; + return nonNullVarlenCols * BinaryRow.VARLEN_COLUMN_OFFSET_FIELD_SIZE; } /** @@ -108,14 +108,36 @@ public class RowAssembler { } /** + * @param keyCols Key columns. + * @param nonNullVarlenKeyCols Number of non-null varlen columns in key chunk. + * @param nonNullVarlenKeySize Size of non-null varlen columns in key chunk. + * @param valCols Value columns. + * @param nonNullVarlenValCols Number of non-null varlen columns in value chunk. + * @param nonNullVarlenValSize Size of non-null varlen columns in value chunk. + * @return Total row size. + */ + public static int rowSize( + Columns keyCols, + int nonNullVarlenKeyCols, + int nonNullVarlenKeySize, + Columns valCols, + int nonNullVarlenValCols, + int nonNullVarlenValSize + ) { + return BinaryRow.KEY_CHUNK_OFFSET /* Header size */ + + rowChunkSize(keyCols, nonNullVarlenKeyCols, nonNullVarlenKeySize) + + rowChunkSize(valCols, nonNullVarlenValCols, nonNullVarlenValSize); + } + + /** * @param cols Columns. * @param nonNullVarlenCols Number of non-null varlen columns in chunk. * @param nonNullVarlenSize Size of non-null varlen columns in chunk. * @return Row's chunk size. */ - public static int rowChunkSize(Columns cols, int nonNullVarlenCols, int nonNullVarlenSize) { - int size = BinaryRow.TOTAL_LEN_FIELD_SIZE + BinaryRow.VARLEN_TABLE_SIZE_FIELD_SIZE + - varlenTableSize(nonNullVarlenCols) + cols.nullMapSize(); + static int rowChunkSize(Columns cols, int nonNullVarlenCols, int nonNullVarlenSize) { + int size = BinaryRow.CHUNK_LEN_FIELD_SIZE + cols.nullMapSize() + + BinaryRow.VARLEN_TABLE_SIZE_FIELD_SIZE + varlenTableSize(nonNullVarlenCols); for (int i = 0; i < cols.numberOfFixsizeColumns(); i++) size += cols.column(i).type().length(); @@ -137,39 +159,18 @@ public class RowAssembler { int nonNullVarlenValCols ) { this.schema = schema; - this.nonNullVarlenValCols = nonNullVarlenValCols; - buf = new ExpandableByteBuf(size); - curCols = schema.keyColumns(); + flags = 0; + strEncoder = null; initOffsets(BinaryRow.KEY_CHUNK_OFFSET, nonNullVarlenKeyCols); - buf.putShort(0, (short)schema.version()); - buf.putShort(baseOff + BinaryRow.TOTAL_LEN_FIELD_SIZE, (short)nonNullVarlenKeyCols); - } + buf = new ExpandableByteBuf(size); - /** - * @param keyCols Key columns. - * @param nonNullVarlenKeyCols Number of non-null varlen columns in key chunk. - * @param nonNullVarlenKeySize Size of non-null varlen columns in key chunk. - * @param valCols Value columns. - * @param nonNullVarlenValCols Number of non-null varlen columns in value chunk. - * @param nonNullVarlenValSize Size of non-null varlen columns in value chunk. - * @return Total row size. - */ - public static int rowChunkSize( - Columns keyCols, - int nonNullVarlenKeyCols, - int nonNullVarlenKeySize, - Columns valCols, - int nonNullVarlenValCols, - int nonNullVarlenValSize - ) { - return BinaryRow.KEY_CHUNK_OFFSET + - rowChunkSize(keyCols, nonNullVarlenKeyCols, nonNullVarlenKeySize) + - rowChunkSize(valCols, nonNullVarlenValCols, nonNullVarlenValSize); + buf.putShort(0, (short)schema.version()); + buf.putShort(nullMapOff + curCols.nullMapSize(), (short)nonNullVarlenKeyCols); } /** @@ -375,7 +376,7 @@ public class RowAssembler { * @param off Offset to write. */ private void writeOffset(int tblEntryIdx, int off) { - buf.putShort(varlenTblOff + 2 * tblEntryIdx, (short)off); + buf.putShort(varlenTblOff + BinaryRow.VARLEN_COLUMN_OFFSET_FIELD_SIZE * tblEntryIdx, (short)off); } /** @@ -409,6 +410,8 @@ public class RowAssembler { int byteInMap = colIdx / 8; int bitInByte = colIdx % 8; + buf.ensureCapacity(nullMapOff + byteInMap + 1); + buf.put(nullMapOff + byteInMap, (byte)(buf.get(nullMapOff + byteInMap) | (1 << bitInByte))); } @@ -443,7 +446,8 @@ public class RowAssembler { buf.putShort(baseOff, (short)keyLen); if (schema.valueColumns() == curCols) { - buf.putShort(baseOff + BinaryRow.TOTAL_LEN_FIELD_SIZE, (short)nonNullVarlenValCols); + buf.putShort(nullMapOff + curCols.nullMapSize(), (short)nonNullVarlenValCols); + return; // No more columns. } @@ -463,8 +467,9 @@ public class RowAssembler { curCol = 0; curVarlenTblEntry = 0; - varlenTblOff = baseOff + BinaryRow.TOTAL_LEN_FIELD_SIZE + BinaryRow.VARLEN_TABLE_SIZE_FIELD_SIZE; - nullMapOff = varlenTblOff + varlenTableSize(nonNullVarlenCols); - curOff = nullMapOff + curCols.nullMapSize(); + nullMapOff = baseOff + BinaryRow.CHUNK_LEN_FIELD_SIZE; + varlenTblOff = nullMapOff + curCols.nullMapSize() + BinaryRow.VARLEN_TABLE_SIZE_FIELD_SIZE; + + curOff = varlenTblOff + varlenTableSize(nonNullVarlenCols); } } diff --git a/modules/schema/src/main/java/org/apache/ignite/internal/schema/marshaller/asm/AsmSerializerGenerator.java b/modules/schema/src/main/java/org/apache/ignite/internal/schema/marshaller/asm/AsmSerializerGenerator.java index fbdb013..20e9efd 100644 --- a/modules/schema/src/main/java/org/apache/ignite/internal/schema/marshaller/asm/AsmSerializerGenerator.java +++ b/modules/schema/src/main/java/org/apache/ignite/internal/schema/marshaller/asm/AsmSerializerGenerator.java @@ -301,7 +301,7 @@ public class AsmSerializerGenerator implements SerializerFactory { body.append(BytecodeExpressions.newInstance(RowAssembler.class, methodDef.getThis().getField("schema", SchemaDescriptor.class), - BytecodeExpressions.invokeStatic(RowAssembler.class, "rowChunkSize", int.class, + BytecodeExpressions.invokeStatic(RowAssembler.class, "rowSize", int.class, keyCols, varlenKeyCols, varlenKeyColsSize, valCols, varlenValueCols, varlenValueColsSize), varlenKeyCols, diff --git a/modules/schema/src/main/java/org/apache/ignite/internal/schema/marshaller/reflection/JavaSerializer.java b/modules/schema/src/main/java/org/apache/ignite/internal/schema/marshaller/reflection/JavaSerializer.java index c7a802c..4c0215e 100644 --- a/modules/schema/src/main/java/org/apache/ignite/internal/schema/marshaller/reflection/JavaSerializer.java +++ b/modules/schema/src/main/java/org/apache/ignite/internal/schema/marshaller/reflection/JavaSerializer.java @@ -85,7 +85,7 @@ public class JavaSerializer extends AbstractSerializer { ObjectStatistic keyStat = collectObjectStats(schema.keyColumns(), keyMarsh, key); ObjectStatistic valStat = collectObjectStats(schema.valueColumns(), valMarsh, val); - int size = RowAssembler.rowChunkSize( + int size = RowAssembler.rowSize( schema.keyColumns(), keyStat.nonNullFields, keyStat.nonNullFieldsSize, schema.valueColumns(), valStat.nonNullFields, valStat.nonNullFieldsSize); diff --git a/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowAssemblerTest.java b/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowAssemblerTest.java new file mode 100644 index 0000000..032ab32 --- /dev/null +++ b/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowAssemblerTest.java @@ -0,0 +1,678 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.schema; + +import java.util.Arrays; +import java.util.UUID; +import org.junit.jupiter.api.Test; + +import static org.apache.ignite.internal.schema.NativeType.BYTE; +import static org.apache.ignite.internal.schema.NativeType.BYTES; +import static org.apache.ignite.internal.schema.NativeType.INTEGER; +import static org.apache.ignite.internal.schema.NativeType.SHORT; +import static org.apache.ignite.internal.schema.NativeType.STRING; +import static org.apache.ignite.internal.schema.NativeType.UUID; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +/** + * Validate row layout for different schema configurations. + */ +public class RowAssemblerTest { + /** Uuid test value. */ + public final java.util.UUID uuidVal = new UUID(-5204230847775358097L, 4916207022290092939L); + + /** + * Validate row layout for schema of fix-len non-null key and fix-len nullable value. + */ + @Test + public void testFixedKeyFixedNullableValue() { + Column[] keyCols = new Column[] {new Column("keyIntCol", INTEGER, false)}; + Column[] valCols = new Column[] {new Column("valIntCol", INTEGER, true)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendInt(33); + asm.appendInt(-71); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, -71, -1, -1, -1}, asm.build()); + } + + { // Null value. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendInt(-33); + asm.appendNull(); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, -33, -1, -1, -1, 7, 0, 0, 0, 1, 0, 0}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendInt(-33); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, -33, -1, -1, -1}, asm.build()); + } + } + + /** + * Validate row layout for schema of fix-len non-null key and fix-len non-null value. + */ + @Test + public void testFixedKeyFixedValue() { + Column[] keyCols = new Column[] {new Column("keyShortCol", SHORT, false)}; + Column[] valCols = new Column[] {new Column("valShortCol", SHORT, false)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { // With value. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendShort((short)33); + asm.appendShort((short)71L); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 33, 0, 9, 0, 0, 0, 0, 0, 0, 71, 0}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendShort((short)-33); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, -33, -1}, asm.build()); + } + } + + /** + * Validate row layout for schema of fix-len non-null key and var-len nullable value. + */ + @Test + public void testFixedKeyVarlenNullableValue() { + Column[] keyCols = new Column[] {new Column("keyShortCol", SHORT, false)}; + Column[] valCols = new Column[] {new Column("valStrCol", STRING, true)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 0, 1); + + asm.appendShort((short)-33); + asm.appendString("val"); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, -33, -1, 12, 0, 0, 0, 0, 1, 0, 9, 0, 118, 97, 108}, asm.build()); + } + + { // Null value. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendShort((short)33); + asm.appendNull(); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 33, 0, 7, 0, 0, 0, 1, 0, 0}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendShort((short)33); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 33, 0}, asm.build()); + } + } + + /** + * Validate row layout for schema of fix-len non-null key and var-len non-null value. + */ + @Test + public void testFixedKeyVarlenValue() { + Column[] keyCols = new Column[] {new Column("keyShortCol", SHORT, false)}; + Column[] valCols = new Column[] {new Column("valStrCol", STRING, false)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 0, 1); + + asm.appendShort((short)-33); + asm.appendString("val"); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, -33, -1, 12, 0, 0, 0, 0, 1, 0, 9, 0, 118, 97, 108}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendShort((short)33); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 33, 0}, asm.build()); + } + } + + /** + * Validate row layout for schema of fix-len nullable key and fix-len non-null value. + */ + @Test + public void testFixedNullableKeyFixedValue() { + Column[] keyCols = new Column[] {new Column("keyShortCol", SHORT, true)}; + Column[] valCols = new Column[] {new Column("valByteCol", BYTE, false)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendShort((short)-33); + asm.appendByte((byte)71); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, -33, -1, 8, 0, 0, 0, 0, 0, 0, 71}, asm.build()); + } + + { // Null key. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendNull(); + asm.appendByte((byte)-71); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, -71}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendShort((short)33); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 33, 0}, asm.build()); + } + } + + /** + * Validate row layout for schema of fix-len nullable key and fix-len nullable value. + */ + @Test + public void testFixedNullableKeyFixedNullableValue() { + Column[] keyCols = new Column[] {new Column("keyShortCol", SHORT, true)}; + Column[] valCols = new Column[] {new Column("valShortCol", SHORT, true)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendShort((short)-1133); + asm.appendShort((short)-1071); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, -109, -5, 9, 0, 0, 0, 0, 0, 0, -47, -5}, asm.build()); + } + + { // Null key. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendNull(); + asm.appendShort((short)1171); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, -109, 4}, asm.build()); + } + + { // Null value. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendShort((short)1133); + asm.appendNull(); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 109, 4, 7, 0, 0, 0, 1, 0, 0}, asm.build()); + } + + { // Null both. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendNull(); + asm.appendNull(); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 7, 0, 0, 0, 1, 0, 0}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendShort((short)1133); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 109, 4}, asm.build()); + } + } + + /** + * Validate row layout for schema of fix-len nullable key and var-len nullable value. + */ + @Test + public void testFixedNullableKeyVarlenNullableValue() { + Column[] keyCols = new Column[] {new Column("keyIntCol", INTEGER, true)}; + Column[] valCols = new Column[] {new Column("valStrCol", STRING, true)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 0, 1); + + asm.appendInt(-33); + asm.appendString("val"); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, -33, -1, -1, -1, 12, 0, 0, 0, 0, 1, 0, 9, 0, 118, 97, 108}, asm.build()); + } + + { // Null key. + RowAssembler asm = new RowAssembler(schema, 0, 0, 1); + + asm.appendNull(); + asm.appendString("val"); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 118, 97, 108}, asm.build()); + } + + { // Null value. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendInt(33); + asm.appendNull(); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0}, asm.build()); + } + + { // Null both. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendNull(); + asm.appendNull(); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 7, 0, 0, 0, 1, 0, 0}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendInt(33); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0}, asm.build()); + } + } + + /** + * Validate row layout for schema of fix-len nullable key and var-len non-null value. + */ + @Test + public void testFixedNullableKeyVarlenValue() { + Column[] keyCols = new Column[] {new Column("keyByteCol", BYTE, true)}; + Column[] valCols = new Column[] {new Column("valStrCol", STRING, false)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 0, 1); + + asm.appendByte((byte)-33); + asm.appendString("val"); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, -33, 12, 0, 0, 0, 0, 1, 0, 9, 0, 118, 97, 108}, asm.build()); + } + + { // Null key. + RowAssembler asm = new RowAssembler(schema, 0, 0, 1); + + asm.appendNull(); + asm.appendString("val"); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 118, 97, 108}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendByte((byte)33); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 33}, asm.build()); + } + } + + /** + * Validate row layout for schema of var-len non-null key and fix-len nullable value. + */ + @Test + public void testVarlenKeyFixedNullableValue() { + Column[] keyCols = new Column[] {new Column("keyStrCol", STRING, false)}; + Column[] valCols = new Column[] {new Column("valUuidCol", UUID, true)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + asm.appendUuid(uuidVal); + + assertRowBytesEquals(new byte[] { + 42, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121, + 23, 0, 0, 0, 0, 0, 0, -117, -61, -31, 85, 61, -32, 57, 68, 111, 67, 56, -3, -99, -37, -58, -73}, asm.build()); + } + + { // Null value. + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + asm.appendNull(); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121, 7, 0, 0, 0, 1, 0, 0}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121}, asm.build()); + } + } + + /** + * Validate row layout for schema of var-len non-null key and fix-len non-null value. + */ + @Test + public void testVarlenKeyFixedValue() { + Column[] keyCols = new Column[] {new Column("keyStrCol", STRING, false)}; + Column[] valCols = new Column[] {new Column("valUuidCol", UUID, false)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + asm.appendUuid(uuidVal); + + assertRowBytesEquals(new byte[] { + 42, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121, + 23, 0, 0, 0, 0, 0, 0, -117, -61, -31, 85, 61, -32, 57, 68, 111, 67, 56, -3, -99, -37, -58, -73}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121}, asm.build()); + } + } + + /** + * Validate row layout for schema of var-len non-null key and var-len nullable value. + */ + @Test + public void testVarlenKeyVarlenNullableValue() { + Column[] keyCols = new Column[] {new Column("keyStrCol", STRING, false)}; + Column[] valCols = new Column[] {new Column("valBytesCol", BYTES, true)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 1, 1); + + asm.appendString("key"); + asm.appendBytes(new byte[] {-1, 1, 0, 120}); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121, 13, 0, 0, 0, 0, 1, 0, 9, 0, -1, 1, 0, 120}, asm.build()); + } + + { // Null value. + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + asm.appendNull(); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121, 7, 0, 0, 0, 1, 0, 0}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121}, asm.build()); + } + } + + /** + * Validate row layout for schema of var-len non-null key and var-len non-null value. + */ + @Test + public void testVarlenKeyVarlenValue() { + Column[] keyCols = new Column[] {new Column("keyStrCol", STRING, false)}; + Column[] valCols = new Column[] {new Column("valBytesCol", BYTES, false)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 1, 1); + + asm.appendString("key"); + asm.appendBytes(new byte[] {-1, 1, 0, 120}); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121, 13, 0, 0, 0, 0, 1, 0, 9, 0, -1, 1, 0, 120}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121}, asm.build()); + } + } + + /** + * Validate row layout for schema of var-len nullable key and fix-len nullable value. + */ + @Test + public void testVarlenNullableKeyFixedNullableValue() { + Column[] keyCols = new Column[] {new Column("keyStrCol", STRING, true)}; + Column[] valCols = new Column[] {new Column("valShortCol", SHORT, true)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + asm.appendShort((short)-71); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121, 9, 0, 0, 0, 0, 0, 0, -71, -1}, asm.build()); + } + + { // Null key. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendNull(); + asm.appendShort((short)71); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 71, 0}, asm.build()); + } + + { // Null value. + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + asm.appendNull(); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121, 7, 0, 0, 0, 1, 0, 0}, asm.build()); + } + + { // Null both. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendNull(); + asm.appendNull(); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 7, 0, 0, 0, 1, 0, 0}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121}, asm.build()); + } + } + + /** + * Validate row layout for schema of var-len nullable key and fix-len non-null value. + */ + @Test + public void testVarlenNullableKeyFixedValue() { + Column[] keyCols = new Column[] {new Column("keyStrCol", STRING, true)}; + Column[] valCols = new Column[] {new Column("valShortCol", SHORT, false)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + asm.appendShort((short)-71L); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121, 9, 0, 0, 0, 0, 0, 0, -71, -1}, asm.build()); + } + + { // Null key. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendNull(); + asm.appendShort((short)71); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 71, 0}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121}, asm.build()); + } + } + + /** + * Validate row layout for schema of var-len nullable key and var-len nullable value. + */ + @Test + public void testVarlenNullableKeyVarlenNullableValue() { + Column[] keyCols = new Column[] {new Column("keyStrCol", STRING, true)}; + Column[] valCols = new Column[] {new Column("valBytesCol", BYTES, true)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 1, 1); + + asm.appendString("key"); + asm.appendBytes(new byte[] {-1, 1, 0, 120}); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121, 13, 0, 0, 0, 0, 1, 0, 9, 0, -1, 1, 0, 120}, asm.build()); + } + + { // Null key. + RowAssembler asm = new RowAssembler(schema, 0, 0, 1); + + asm.appendNull(); + asm.appendBytes(new byte[] {-1, 1, 0, 120}); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 13, 0, 0, 0, 0, 1, 0, 9, 0, -1, 1, 0, 120}, asm.build()); + } + + { // Null value. + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + asm.appendNull(); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121, 7, 0, 0, 0, 1, 0, 0}, asm.build()); + } + + { // Null both. + RowAssembler asm = new RowAssembler(schema, 0, 0, 0); + + asm.appendNull(); + asm.appendNull(); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 7, 0, 0, 0, 1, 0, 0}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121}, asm.build()); + } + } + + /** + * Validate row layout for schema of var-len nullable key and var-len non-null value. + */ + @Test + public void testVarlenNullableKeyVarlenValue() { + Column[] keyCols = new Column[] {new Column("keyStrCol", STRING, true)}; + Column[] valCols = new Column[] {new Column("valBytesCol", BYTES, false)}; + + SchemaDescriptor schema = new SchemaDescriptor(42, keyCols, valCols); + + { + RowAssembler asm = new RowAssembler(schema, 0, 1, 1); + + asm.appendString("key"); + asm.appendBytes(new byte[] {-1, 1, 0, 120}); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121, 13, 0, 0, 0, 0, 1, 0, 9, 0, -1, 1, 0, 120}, asm.build()); + } + + { // Null key. + RowAssembler asm = new RowAssembler(schema, 0, 0, 1); + + asm.appendNull(); + asm.appendBytes(new byte[] {-1, 1, 0, 120}); + + assertRowBytesEquals(new byte[] {42, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 13, 0, 0, 0, 0, 1, 0, 9, 0, -1, 1, 0, 120}, asm.build()); + } + + { // No value. + RowAssembler asm = new RowAssembler(schema, 0, 1, 0); + + asm.appendString("key"); + + assertRowBytesEquals(new byte[] {42, 0, 2, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 9, 0, 107, 101, 121,}, asm.build()); + } + } + + /** + * @param expected Expected row bytes. + * @param actual Actual row bytes. + */ + private void assertRowBytesEquals(byte[] expected, byte[] actual) { + assertArrayEquals(expected, actual, Arrays.toString(actual)); + } +} diff --git a/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowTest.java b/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowTest.java index e50598d..cd36776 100644 --- a/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowTest.java +++ b/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowTest.java @@ -235,7 +235,7 @@ public class RowTest { } } - int size = RowAssembler.rowChunkSize( + int size = RowAssembler.rowSize( schema.keyColumns(), nonNullVarLenKeyCols, nonNullVarLenKeySize, schema.valueColumns(), nonNullVarLenValCols, nonNullVarLenValSize);