This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch new_vector in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 265538d39cee80e5c1875ff1cc7de480189e706a Author: JackieTien97 <[email protected]> AuthorDate: Wed Nov 10 11:30:03 2021 +0800 fix aligned write --- .../tsfile/write/chunk/AlignedChunkWriterImpl.java | 11 +++-- .../org/apache/iotdb/tsfile/write/WriteTest.java | 23 ++++++----- .../write/writer/AlignedChunkWriterImplTest.java | 48 +++++++++++----------- .../write/writer/VectorMeasurementSchemaStub.java | 17 ++++---- 4 files changed, 48 insertions(+), 51 deletions(-) diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java index 8ad44cd..b3dbf33 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/AlignedChunkWriterImpl.java @@ -18,17 +18,16 @@ */ package org.apache.iotdb.tsfile.write.chunk; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.apache.iotdb.tsfile.encoding.encoder.Encoder; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.Binary; -import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; +import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema; import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - public class AlignedChunkWriterImpl implements IChunkWriter { private final TimeChunkWriter timeChunkWriter; @@ -36,7 +35,7 @@ public class AlignedChunkWriterImpl implements IChunkWriter { private int valueIndex; /** @param schema schema of this measurement */ - public AlignedChunkWriterImpl(IMeasurementSchema schema) { + public AlignedChunkWriterImpl(VectorMeasurementSchema schema) { timeChunkWriter = new TimeChunkWriter( schema.getMeasurementId(), diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java index d01836a..b98418b 100755 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java @@ -18,6 +18,15 @@ */ package org.apache.iotdb.tsfile.write; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Random; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant; @@ -33,23 +42,12 @@ import org.apache.iotdb.tsfile.utils.StringContainer; import org.apache.iotdb.tsfile.write.record.TSRecord; import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema; - import org.junit.After; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Random; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - /** test writing processing correction combining writing process and reading process. */ public class WriteTest { @@ -241,6 +239,9 @@ public class WriteTest { strings = getNextRecord(lineCount, stageState); for (String str : strings) { TSRecord record = RecordUtils.parseSimpleTupleRecord(str, schema); + if (record.dataPointList.isEmpty()) { + continue; + } tsFileWriter.write(record); } lineCount++; diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/AlignedChunkWriterImplTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/AlignedChunkWriterImplTest.java index 495ddbe..ac05405 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/AlignedChunkWriterImplTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/AlignedChunkWriterImplTest.java @@ -18,6 +18,11 @@ */ package org.apache.iotdb.tsfile.write.writer; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.nio.ByteBuffer; import org.apache.iotdb.tsfile.file.MetaMarker; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -26,15 +31,8 @@ import org.apache.iotdb.tsfile.utils.PublicBAOS; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.iotdb.tsfile.write.chunk.AlignedChunkWriterImpl; - import org.junit.Test; -import java.io.IOException; -import java.nio.ByteBuffer; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - public class AlignedChunkWriterImplTest { @Test @@ -50,11 +48,11 @@ public class AlignedChunkWriterImplTest { } chunkWriter.sealCurrentPage(); - // time chunk: 17 + 4 + 160; - // value chunk 1: 19 + 2 + 4 + 3 + 80; - // value chunk 2: 19 + 2 + 4 + 3 + 20; - // value chunk 3: 20 + 4 + 7 + 20 * 8; - assertEquals(528, chunkWriter.getSerializedChunkSize()); + // time chunk: 7(ChunkHeader Size) + 4(PageHeader Size: uncompressedSize + compressedSize) + 160(dataSize); + // value chunk 1: 8(ChunkHeader Size) + 2(PageHeader Size: uncompressedSize + compressedSize) + 4(bitmap length) + 3(bitmap data) + 80(data size); + // value chunk 2: 8 + 2 + 4 + 3 + 20; + // value chunk 3: 9 + 4 + 4 + 3 + 20 * 8; + assertEquals(485, chunkWriter.getSerializedChunkSize()); try { TestTsFileOutput testTsFileOutput = new TestTsFileOutput(); @@ -65,7 +63,7 @@ public class AlignedChunkWriterImplTest { // time chunk assertEquals( (byte) (0x80 | MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER), ReadWriteIOUtils.readByte(buffer)); - assertEquals("vectorName", ReadWriteIOUtils.readVarIntString(buffer)); + assertEquals("", ReadWriteIOUtils.readVarIntString(buffer)); assertEquals(164, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer)); assertEquals(TSDataType.VECTOR.serialize(), ReadWriteIOUtils.readByte(buffer)); assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer)); @@ -74,7 +72,7 @@ public class AlignedChunkWriterImplTest { // value chunk 1 assertEquals(0x40 | MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER, ReadWriteIOUtils.readByte(buffer)); - assertEquals("vectorName.s1", ReadWriteIOUtils.readVarIntString(buffer)); + assertEquals("s1", ReadWriteIOUtils.readVarIntString(buffer)); assertEquals(89, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer)); assertEquals(TSDataType.FLOAT.serialize(), ReadWriteIOUtils.readByte(buffer)); assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer)); @@ -83,7 +81,7 @@ public class AlignedChunkWriterImplTest { // value chunk 2 assertEquals(0x40 | MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER, ReadWriteIOUtils.readByte(buffer)); - assertEquals("vectorName.s2", ReadWriteIOUtils.readVarIntString(buffer)); + assertEquals("s2", ReadWriteIOUtils.readVarIntString(buffer)); assertEquals(29, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer)); assertEquals(TSDataType.INT32.serialize(), ReadWriteIOUtils.readByte(buffer)); assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer)); @@ -92,7 +90,7 @@ public class AlignedChunkWriterImplTest { // value chunk 2 assertEquals(0x40 | MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER, ReadWriteIOUtils.readByte(buffer)); - assertEquals("vectorName.s3", ReadWriteIOUtils.readVarIntString(buffer)); + assertEquals("s3", ReadWriteIOUtils.readVarIntString(buffer)); assertEquals(171, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer)); assertEquals(TSDataType.DOUBLE.serialize(), ReadWriteIOUtils.readByte(buffer)); assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer)); @@ -124,11 +122,11 @@ public class AlignedChunkWriterImplTest { } chunkWriter.sealCurrentPage(); - // time chunk: 17 + (4 + 17 + 160) * 2 - // value chunk 1: 20 + (2 + 41 + 4 + 3 + 80) * 2 - // value chunk 2: 20 + (2 + 41 + 4 + 3 + 20) * 2 - // value chunk 3: 20 + (4 + 57 + 4 + 3 + 160) * 2 - assertEquals(1295, chunkWriter.getSerializedChunkSize()); + // time chunk: 7 + (4 + 17 + 160) * 2 + // value chunk 1: 9 + (2 + 41 + 4 + 3 + 80) * 2 + // value chunk 2: 9 + (2 + 41 + 4 + 3 + 20) * 2 + // value chunk 3: 9 + (4 + 57 + 4 + 3 + 160) * 2 + assertEquals(1252, chunkWriter.getSerializedChunkSize()); try { TestTsFileOutput testTsFileOutput = new TestTsFileOutput(); @@ -138,7 +136,7 @@ public class AlignedChunkWriterImplTest { ByteBuffer buffer = ByteBuffer.wrap(publicBAOS.getBuf(), 0, publicBAOS.size()); // time chunk assertEquals((byte) (0x80 | MetaMarker.CHUNK_HEADER), ReadWriteIOUtils.readByte(buffer)); - assertEquals("vectorName", ReadWriteIOUtils.readVarIntString(buffer)); + assertEquals("", ReadWriteIOUtils.readVarIntString(buffer)); assertEquals(362, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer)); assertEquals(TSDataType.VECTOR.serialize(), ReadWriteIOUtils.readByte(buffer)); assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer)); @@ -147,7 +145,7 @@ public class AlignedChunkWriterImplTest { // value chunk 1 assertEquals(0x40 | MetaMarker.CHUNK_HEADER, ReadWriteIOUtils.readByte(buffer)); - assertEquals("vectorName.s1", ReadWriteIOUtils.readVarIntString(buffer)); + assertEquals("s1", ReadWriteIOUtils.readVarIntString(buffer)); assertEquals(260, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer)); assertEquals(TSDataType.FLOAT.serialize(), ReadWriteIOUtils.readByte(buffer)); assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer)); @@ -156,7 +154,7 @@ public class AlignedChunkWriterImplTest { // value chunk 2 assertEquals(0x40 | MetaMarker.CHUNK_HEADER, ReadWriteIOUtils.readByte(buffer)); - assertEquals("vectorName.s2", ReadWriteIOUtils.readVarIntString(buffer)); + assertEquals("s2", ReadWriteIOUtils.readVarIntString(buffer)); assertEquals(140, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer)); assertEquals(TSDataType.INT32.serialize(), ReadWriteIOUtils.readByte(buffer)); assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer)); @@ -165,7 +163,7 @@ public class AlignedChunkWriterImplTest { // value chunk 2 assertEquals(0x40 | MetaMarker.CHUNK_HEADER, ReadWriteIOUtils.readByte(buffer)); - assertEquals("vectorName.s3", ReadWriteIOUtils.readVarIntString(buffer)); + assertEquals("s3", ReadWriteIOUtils.readVarIntString(buffer)); assertEquals(456, ReadWriteForEncodingUtils.readUnsignedVarInt(buffer)); assertEquals(TSDataType.DOUBLE.serialize(), ReadWriteIOUtils.readByte(buffer)); assertEquals(CompressionType.UNCOMPRESSED.serialize(), ReadWriteIOUtils.readByte(buffer)); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/VectorMeasurementSchemaStub.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/VectorMeasurementSchemaStub.java index 2128a46..e0b7ba9e 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/VectorMeasurementSchemaStub.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/VectorMeasurementSchemaStub.java @@ -18,25 +18,24 @@ */ package org.apache.iotdb.tsfile.write.writer; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.Map; import org.apache.iotdb.tsfile.encoding.encoder.Encoder; import org.apache.iotdb.tsfile.encoding.encoder.PlainEncoder; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; -import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; - -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema; -public class VectorMeasurementSchemaStub implements IMeasurementSchema { +public class VectorMeasurementSchemaStub extends VectorMeasurementSchema { @Override public String getMeasurementId() { - return "vectorName"; + return ""; } @Override
