This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a commit to branch ignite-15744 in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit d24539bf17c5e8dba528f529d09c8fd4d347502b Author: Andrew Mashenkov <andrey.mashen...@gmail.com> AuthorDate: Fri Oct 22 14:15:49 2021 +0300 Fix vartable size overflow. --- .../ignite/internal/schema/row/VarTableFormat.java | 26 ++++----- .../schema/RowAssemblerAdvancedSchemaTest.java | 22 ++++---- .../org/apache/ignite/internal/schema/RowTest.java | 65 +++++++++++++--------- 3 files changed, 62 insertions(+), 51 deletions(-) diff --git a/modules/schema/src/main/java/org/apache/ignite/internal/schema/row/VarTableFormat.java b/modules/schema/src/main/java/org/apache/ignite/internal/schema/row/VarTableFormat.java index 1dabf87..e1dde85 100644 --- a/modules/schema/src/main/java/org/apache/ignite/internal/schema/row/VarTableFormat.java +++ b/modules/schema/src/main/java/org/apache/ignite/internal/schema/row/VarTableFormat.java @@ -95,13 +95,12 @@ abstract class VarTableFormat { private final byte flags; /** - * @param vartblSizeFieldSize Size of vartalble size field (in bytes). * @param vartblEntrySize Size of vartable entry (in bytes). * @param flags Format specific flags. */ - VarTableFormat(int vartblSizeFieldSize, int vartblEntrySize, byte flags) { + VarTableFormat(int vartblEntrySize, byte flags) { this.vartblEntrySize = vartblEntrySize; - this.vartblSizeFieldSize = vartblSizeFieldSize; + this.vartblSizeFieldSize = Short.BYTES; this.flags = flags; } @@ -149,7 +148,9 @@ abstract class VarTableFormat { * @param vartblOff Vartable offset. * @return Number of entries in the vartable. */ - abstract int readVartableSize(BinaryRow row, int vartblOff); + int readVartableSize(BinaryRow row, int vartblOff) { + return Short.toUnsignedInt(row.readShort(vartblOff)); + } /** * Convert vartable inplace to the current format. @@ -169,7 +170,7 @@ abstract class VarTableFormat { * Creates chunk format. */ TinyFormat() { - super(Byte.BYTES, Byte.BYTES, (byte)1); + super(Byte.BYTES, (byte)1); } /** {@inheritDoc} */ @@ -178,17 +179,12 @@ abstract class VarTableFormat { } /** {@inheritDoc} */ - @Override int readVartableSize(BinaryRow row, int vartblOff) { - return Byte.toUnsignedInt(row.readByte(vartblOff)); - } - - /** {@inheritDoc} */ @Override public int compactVarTable(ExpandableByteBuf buf, int vartblOff, int entres) { - assert entres > 0; + assert entres > 0 && entres < 0xFFFF; - buf.put(vartblOff, (byte)entres); + buf.putShort(vartblOff, (short)entres); - int dstOff = vartblOff + 1; + int dstOff = vartblOff + 2; int srcOff = vartblOff + 2; for (int i = 0; i < entres; i++, srcOff += Integer.BYTES, dstOff++) @@ -208,7 +204,7 @@ abstract class VarTableFormat { * Creates chunk format. */ MediumFormat() { - super(Short.BYTES, Short.BYTES, (byte)2); + super(Short.BYTES, (byte)2); } /** {@inheritDoc} */ @@ -245,7 +241,7 @@ abstract class VarTableFormat { * Creates chunk format. */ LargeFormat() { - super(Short.BYTES, Integer.BYTES, (byte)0); + super(Integer.BYTES, (byte)0); } /** {@inheritDoc} */ diff --git a/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowAssemblerAdvancedSchemaTest.java b/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowAssemblerAdvancedSchemaTest.java index 218a029..8d2fe15 100644 --- a/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowAssemblerAdvancedSchemaTest.java +++ b/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowAssemblerAdvancedSchemaTest.java @@ -125,8 +125,8 @@ public class RowAssemblerAdvancedSchemaTest { assertRowBytesEquals( new byte[]{ 42, 0, 0, 17, -12, -36, 109, -81, - 15, 0, 0, 0, 4, 1, 3, 33, -77, 120, 97, 115, 99, 105, 105, - 19, 0, 0, 0, 4, 1, 2, 33, -77, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97}, + 16, 0, 0, 0, 4, 1, 0, 3, 33, -77, 120, 97, 115, 99, 105, 105, + 20, 0, 0, 0, 4, 1, 0, 2, 33, -77, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97}, new RowAssembler(schema, 2, 2) .appendBytes(new byte[]{33, -77, 120}) .appendString("ascii") @@ -140,8 +140,8 @@ public class RowAssemblerAdvancedSchemaTest { assertRowBytesEquals( new byte[]{ 42, 0, 0, 17, 59, -11, 127, 92, - 22, 0, 0, 0, 1, 1, 5, 97, 115, 99, 105, 105, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, - 22, 0, 0, 0, 1, 1, 10, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, 97, 115, 99, 105, 105}, + 23, 0, 0, 0, 1, 1, 0, 5, 97, 115, 99, 105, 105, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, + 23, 0, 0, 0, 1, 1, 0, 10, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, 97, 115, 99, 105, 105}, new RowAssembler(schema, 2, 2) .appendNull() .appendString("ascii") @@ -155,8 +155,8 @@ public class RowAssemblerAdvancedSchemaTest { assertRowBytesEquals( new byte[]{ 42, 0, 0, 17, -111, 92, -37, -77, - 20, 0, 0, 0, 2, 1, 3, 33, -77, 120, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, - 14, 0, 0, 0, 2, 1, 2, 33, -77, 97, 115, 99, 105, 105}, + 21, 0, 0, 0, 2, 1, 0, 3, 33, -77, 120, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, + 15, 0, 0, 0, 2, 1, 0, 2, 33, -77, 97, 115, 99, 105, 105}, new RowAssembler(schema, 2, 2) .appendBytes(new byte[]{33, -77, 120}) .appendNull() @@ -182,7 +182,7 @@ public class RowAssemblerAdvancedSchemaTest { assertRowBytesEquals( new byte[]{ 42, 0, 1, 1, -118, -28, 81, 103, - 26, 0, 0, 0, 0, 2, 3, 13, 33, -77, 120, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, 97, 115, 99, 105, 105}, + 27, 0, 0, 0, 0, 2, 0, 3, 13, 33, -77, 120, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, 97, 115, 99, 105, 105}, new RowAssembler(schema, 3, 0) .appendBytes(new byte[]{33, -77, 120}) .appendString("我愛Java") @@ -213,7 +213,7 @@ public class RowAssemblerAdvancedSchemaTest { assertRowBytesEquals( new byte[]{ 42, 0, 0, 24, -85, 82, 5, 0, 8, 0, 0, 0, 12, 11, 22, 0, - 14, 0, 0, 0, 3, 1, 2, 77, -88, 97, 115, 99, 105, 105}, + 15, 0, 0, 0, 3, 1, 0, 2, 77, -88, 97, 115, 99, 105, 105}, new RowAssembler(schema, 0, 2) .appendByte((byte)11) .appendShort((short)22) @@ -229,8 +229,8 @@ public class RowAssemblerAdvancedSchemaTest { assertRowBytesEquals( new byte[]{ 42, 0, 0, 17, -18, -9, 119, -80, - 21, 0, 0, 0, 1, 1, 4, 22, 0, 33, -44, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, - 15, 0, 0, 0, 2, 1, 3, 55, 77, -88, 97, 115, 99, 105, 105}, + 22, 0, 0, 0, 1, 1, 0, 4, 22, 0, 33, -44, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, + 16, 0, 0, 0, 2, 1, 0, 3, 55, 77, -88, 97, 115, 99, 105, 105}, new RowAssembler(schema, 2, 2) .appendNull() .appendShort((short)22) @@ -280,7 +280,7 @@ public class RowAssemblerAdvancedSchemaTest { assertRowBytesEquals( new byte[]{ 42, 0, 0, -127, 3, -8, 124, -80, - 22, 0, 0, 0, 0, 1, 5, 11, 22, 0, 33, -44, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, + 23, 0, 0, 0, 0, 1, 0, 5, 11, 22, 0, 33, -44, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, 5, 0, 0, 0, 15}, new RowAssembler(schema, 2, 0) .appendByte((byte)11) diff --git a/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowTest.java b/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowTest.java index eea1ec0..006abe2 100644 --- a/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowTest.java +++ b/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowTest.java @@ -74,7 +74,7 @@ public class RowTest { } /** - * Check row serialization for schema with nullable fix-sized columns only. + * Check row serialization for a schema with nullable fix-sized columns only. */ @Test public void nullableFixSizedColumns() { @@ -114,7 +114,7 @@ public class RowTest { } /** - * Check row serialization for schema with non-nullable fix-sized columns only. + * Check row serialization for a schema with non-nullable fix-sized columns only. */ @Test public void fixSizedColumns() { @@ -154,7 +154,7 @@ public class RowTest { } /** - * Check row serialization for schema with various columns. + * Check row serialization for a schema with various columns. */ @Test public void mixedColumns() { @@ -186,7 +186,7 @@ public class RowTest { } /** - * Check row serialization for schema with various columns. + * Check row serialization for a schema with various columns. */ @Test public void temporalColumns() { @@ -212,7 +212,7 @@ public class RowTest { } /** - * Check row serialization for schema with non-nullable varlen columns only. + * Check row serialization for a schema with non-nullable varlen columns only. */ @Test public void varlenColumns() { @@ -234,7 +234,7 @@ public class RowTest { } /** - * Check row serialization for schema with nullable varlen columns only. + * Check row serialization for a schema with nullable varlen columns only. */ @Test public void nullableVarlenColumns() { @@ -254,16 +254,16 @@ public class RowTest { } /** - * Check row serialization for schema with large varlen columns (64Kb+). + * Check row serialization for a schema with large varlen columns (64Kb+). */ @Test public void largeVarlenColumns() { - Column[] keyCols = new Column[] { + Column[] keyCols = new Column[]{ new Column("keyBytesCol", BYTES, false), new Column("keyStringCol", STRING, false), }; - Column[] valCols = new Column[] { + Column[] valCols = new Column[]{ new Column("valBytesCol", BYTES, true), new Column("valStringCol", STRING, true), }; @@ -290,6 +290,29 @@ public class RowTest { } /** + * Check row serialization for a schema with many empty varlen columns (255+). + */ + @Test + public void bigVartable() { + Column[] keyCols = new Column[]{ + new Column("id", INT64, false), + }; + + int columnCount = 260; + + Column[] valCols = IntStream.range(1, columnCount) + .mapToObj(i -> new Column("val" + i, STRING, true)) + .toArray(Column[]::new); + + SchemaDescriptor sch = new SchemaDescriptor(1, keyCols, valCols); + + Object[] checkArr = IntStream.range(0, columnCount) + .mapToObj(i -> i == 0 ? 42L : (i > columnCount - 5) ? "str" : "").toArray(); + + checkValues(sch, checkArr); + } + + /** * Check row serialization for 256+ fixsized columns. */ @Test @@ -469,43 +492,35 @@ public class RowTest { if (schema.isKeyColumn(i)) { nonNullVarLenKeyCols++; nonNullVarLenKeySize += val.length; - } - else { + } else { nonNullVarLenValCols++; nonNullVarLenValSize += val.length; } - } - else if (type == NativeTypeSpec.STRING) { + } else if (type == NativeTypeSpec.STRING) { if (schema.isKeyColumn(i)) { nonNullVarLenKeyCols++; nonNullVarLenKeySize += RowAssembler.utf8EncodedLength((CharSequence)vals[i]); - } - else { + } else { nonNullVarLenValCols++; nonNullVarLenValSize += RowAssembler.utf8EncodedLength((CharSequence)vals[i]); } - } - else if (type == NativeTypeSpec.NUMBER) { + } else if (type == NativeTypeSpec.NUMBER) { if (schema.isKeyColumn(i)) { nonNullVarLenKeyCols++; nonNullVarLenKeySize += RowAssembler.sizeInBytes((BigInteger)vals[i]); - } - else { + } else { nonNullVarLenValCols++; nonNullVarLenValSize += RowAssembler.sizeInBytes((BigInteger)vals[i]); } - } - else if (type == NativeTypeSpec.DECIMAL) { + } else if (type == NativeTypeSpec.DECIMAL) { if (schema.isKeyColumn(i)) { nonNullVarLenKeyCols++; nonNullVarLenKeySize += RowAssembler.sizeInBytes((BigDecimal)vals[i]); - } - else { + } else { nonNullVarLenValCols++; nonNullVarLenValSize += RowAssembler.sizeInBytes((BigDecimal)vals[i]); } - } - else + } else throw new IllegalStateException("Unsupported variable-length type: " + type); } }