This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 420f5c01 fix the bug that chunk metadata mask is lost when
deserializing from temporary .meta file (#696)
420f5c01 is described below
commit 420f5c0190f6afee80cc53cb4e1078f9b1dc49c5
Author: shuwenwei <[email protected]>
AuthorDate: Thu Jan 8 08:28:05 2026 +0800
fix the bug that chunk metadata mask is lost when deserializing from
temporary .meta file (#696)
---
.../apache/tsfile/write/writer/TsFileIOWriter.java | 2 +
.../write/writer/tsmiterator/DiskTSMIterator.java | 5 +-
.../TsFileIOWriterFlushTempChunkMetadataTest.java | 154 +++++++++++++++++++++
3 files changed, 160 insertions(+), 1 deletion(-)
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
index 96cc383e..377ef90f 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java
@@ -822,6 +822,8 @@ public class TsFileIOWriter implements AutoCloseable {
writtenSize +=
ReadWriteIOUtils.write(
iChunkMetadataList.get(0).getDataType(),
tempOutput.wrapAsStream());
+ writtenSize +=
+ ReadWriteIOUtils.write(iChunkMetadataList.get(0).getMask(),
tempOutput.wrapAsStream());
}
PublicBAOS buffer = new PublicBAOS();
int totalSize = 0;
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/tsmiterator/DiskTSMIterator.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/tsmiterator/DiskTSMIterator.java
index 44c596b1..19064ee5 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/tsmiterator/DiskTSMIterator.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/tsmiterator/DiskTSMIterator.java
@@ -105,6 +105,7 @@ public class DiskTSMIterator extends TSMIterator {
String measurementUid =
ReadWriteIOUtils.readVarIntString(input.wrapAsInputStream());
byte dataTypeInByte = ReadWriteIOUtils.readByte(input.wrapAsInputStream());
TSDataType dataType = TSDataType.getTsDataType(dataTypeInByte);
+ byte mask = ReadWriteIOUtils.readByte(input.wrapAsInputStream());
int chunkBufferSize = ReadWriteIOUtils.readInt(input.wrapAsInputStream());
ByteBuffer chunkBuffer = ByteBuffer.allocate(chunkBufferSize);
int readSize = ReadWriteIOUtils.readAsPossible(input, chunkBuffer);
@@ -118,7 +119,9 @@ public class DiskTSMIterator extends TSMIterator {
// deserialize chunk metadata from chunk buffer
List<IChunkMetadata> chunkMetadataList = new ArrayList<>();
while (chunkBuffer.hasRemaining()) {
- chunkMetadataList.add(ChunkMetadata.deserializeFrom(chunkBuffer,
dataType));
+ ChunkMetadata chunkMetadata = ChunkMetadata.deserializeFrom(chunkBuffer,
dataType);
+ chunkMetadata.setMask(mask);
+ chunkMetadataList.add(chunkMetadata);
}
updateCurrentPos();
return new Pair<>(
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileIOWriterFlushTempChunkMetadataTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileIOWriterFlushTempChunkMetadataTest.java
new file mode 100644
index 00000000..98ebe2e6
--- /dev/null
+++
b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileIOWriterFlushTempChunkMetadataTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read;
+
+import org.apache.tsfile.constant.TestConstant;
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.file.metadata.StringArrayDeviceID;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.writer.TsFileIOWriter;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Arrays;
+
+public class TsFileIOWriterFlushTempChunkMetadataTest {
+
+ private static final String FILE_PATH1 =
+
TestConstant.BASE_OUTPUT_PATH.concat("TsFileIOWriterFlushTempChunkMetadataTest1.tsfile");
+ private static final String FILE_PATH2 =
+
TestConstant.BASE_OUTPUT_PATH.concat("TsFileIOWriterFlushTempChunkMetadataTest2.tsfile");
+
+ @After
+ public void teardown() {
+ new File(FILE_PATH1).delete();
+ new File(FILE_PATH2).delete();
+ }
+
+ @Test
+ public void testAligned() throws IOException, WriteProcessException {
+ TableSchema tableSchema =
+ new TableSchema(
+ "t1",
+ Arrays.asList(
+ new MeasurementSchema("device", TSDataType.STRING),
+ new MeasurementSchema("s1", TSDataType.INT32),
+ new MeasurementSchema("s2", TSDataType.INT32),
+ new MeasurementSchema("s3", TSDataType.INT32)),
+ Arrays.asList(
+ ColumnCategory.TAG,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD));
+ Tablet tablet1 =
+ new Tablet(
+ tableSchema.getTableName(),
+ Arrays.asList("device", "s1", "s2", "s3"),
+ Arrays.asList(TSDataType.STRING, TSDataType.INT32,
TSDataType.INT32, TSDataType.INT32),
+ Arrays.asList(
+ ColumnCategory.TAG,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD));
+ for (int i = 0; i < 1000; i++) {
+ tablet1.addTimestamp(i, i);
+ tablet1.addValue("device", i, "d" + i);
+ tablet1.addValue("s1", i, 0);
+ tablet1.addValue("s2", i, 0);
+ tablet1.addValue("s3", i, 0);
+ }
+ try (TsFileIOWriter writer = new TsFileIOWriter(new File(FILE_PATH1), 1)) {
+ TsFileWriter tsFileWriter = new TsFileWriter(writer);
+ tsFileWriter.registerTableSchema(tableSchema);
+ tsFileWriter.writeTable(tablet1);
+ tsFileWriter.flush();
+ writer.checkMetadataSizeAndMayFlush();
+ tsFileWriter.close();
+ }
+ try (TsFileIOWriter writer = new TsFileIOWriter(new File(FILE_PATH2))) {
+ TsFileWriter tsFileWriter = new TsFileWriter(writer);
+ tsFileWriter.registerTableSchema(tableSchema);
+ tsFileWriter.writeTable(tablet1);
+ tsFileWriter.flush();
+ tsFileWriter.close();
+ }
+ byte[] file1Contents = Files.readAllBytes(new File(FILE_PATH1).toPath());
+ byte[] file2Contents = Files.readAllBytes(new File(FILE_PATH2).toPath());
+ Assert.assertArrayEquals(file1Contents, file2Contents);
+ }
+
+ @Test
+ public void testNonAligned() throws IOException, WriteProcessException {
+ try (TsFileIOWriter writer = new TsFileIOWriter(new File(FILE_PATH1), 1)) {
+ TsFileWriter tsFileWriter = new TsFileWriter(writer);
+ writeNonAlignedData(tsFileWriter, writer, true);
+ tsFileWriter.close();
+ }
+ try (TsFileIOWriter writer = new TsFileIOWriter(new File(FILE_PATH2))) {
+ TsFileWriter tsFileWriter = new TsFileWriter(writer);
+ writeNonAlignedData(tsFileWriter, writer, false);
+ tsFileWriter.close();
+ }
+ byte[] file1Contents = Files.readAllBytes(new File(FILE_PATH1).toPath());
+ byte[] file2Contents = Files.readAllBytes(new File(FILE_PATH2).toPath());
+ Assert.assertArrayEquals(file1Contents, file2Contents);
+ }
+
+ private void writeNonAlignedData(
+ TsFileWriter tsFileWriter,
+ TsFileIOWriter tsFileIOWriter,
+ boolean flushChunkMetadataToTempFile)
+ throws WriteProcessException, IOException {
+ for (int i = 0; i < 10; i++) {
+ Tablet tablet =
+ new Tablet(
+ new StringArrayDeviceID("root.test.d" + i),
+ Arrays.asList("s1", "s2", "s3"),
+ Arrays.asList(TSDataType.INT32, TSDataType.INT32,
TSDataType.INT32));
+ for (int j = 0; j < 1000; j++) {
+ tablet.addTimestamp(j, j);
+ tablet.addValue("s1", j, 0);
+ tablet.addValue("s2", j, 0);
+ tablet.addValue("s3", j, 0);
+ }
+ tsFileWriter.registerTimeseries(
+ "root.test.d" + i, new MeasurementSchema("s1", TSDataType.INT32));
+ tsFileWriter.registerTimeseries(
+ "root.test.d" + i, new MeasurementSchema("s2", TSDataType.INT32));
+ tsFileWriter.registerTimeseries(
+ "root.test.d" + i, new MeasurementSchema("s3", TSDataType.INT32));
+ tsFileWriter.writeTree(tablet);
+ tsFileWriter.flush();
+ if (flushChunkMetadataToTempFile) {
+ tsFileIOWriter.checkMetadataSizeAndMayFlush();
+ }
+ }
+ }
+}
