This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 6b3b6a540a4 Keep fieldless TableSchema during compaction (#18997)
6b3b6a540a4 is described below
commit 6b3b6a540a4ce57ea3ed550ecd2c930e0511194a
Author: Zhenyu Luo <[email protected]>
AuthorDate: Mon Oct 26 09:33:43 2026 +0800
Keep fieldless TableSchema during compaction (#18997)
A table-model TsFile may contain only TAG/TIME columns. Do not drop such a
TableSchema during compaction: TIME still exists, and the schema is needed as
the table-model marker. Track tables that wrote data in the target and only
remove no-FIELD, no-TIME schemas of tables absent from that target.
---
.../compaction/io/CompactionTsFileWriter.java | 17 +++++++----
.../CompactionTableModelTestFileWriter.java | 33 ++++++++++++++++++++++
.../utils/CompactionTestFileWriterTest.java | 23 +++++++++++++++
3 files changed, 68 insertions(+), 5 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java
index 1f822e3b934..45adb94cf8f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java
@@ -47,6 +47,7 @@ import java.io.Serializable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
public class CompactionTsFileWriter extends TsFileIOWriter {
CompactionType type;
@@ -181,15 +182,21 @@ public class CompactionTsFileWriter extends
TsFileIOWriter {
}
private void removeUnusedTableSchema() {
- Map<String, TableSchema> tableSchemaMap = getSchema().getTableSchemaMap();
- Iterator<Map.Entry<String, TableSchema>> iterator =
tableSchemaMap.entrySet().iterator();
+ final Map<String, TableSchema> tableSchemaMap =
getSchema().getTableSchemaMap();
+ final Iterator<Map.Entry<String, TableSchema>> iterator =
tableSchemaMap.entrySet().iterator();
while (iterator.hasNext()) {
- Map.Entry<String, TableSchema> entry = iterator.next();
- List<ColumnCategory> columnTypes = entry.getValue().getColumnTypes();
- if (columnTypes.contains(ColumnCategory.FIELD)) {
+ final Map.Entry<String, TableSchema> entry = iterator.next();
+ final List<ColumnCategory> columnTypes =
entry.getValue().getColumnTypes();
+ if (hasFieldOrTimeColumn(columnTypes)) {
continue;
}
iterator.remove();
}
}
+
+ private boolean hasFieldOrTimeColumn(final List<ColumnCategory> columnTypes)
{
+ return Objects.nonNull(columnTypes)
+ && (columnTypes.contains(ColumnCategory.FIELD)
+ || columnTypes.contains(ColumnCategory.TIME));
+ }
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionTableModelTestFileWriter.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionTableModelTestFileWriter.java
index f586f6a5473..39f0b909856 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionTableModelTestFileWriter.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionTableModelTestFileWriter.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.tablemodel;
+import org.apache.iotdb.commons.utils.TestOnly;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionTableSchema;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionTestFileWriter;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -30,12 +31,15 @@ import org.apache.tsfile.file.metadata.StringArrayDeviceID;
import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.apache.tsfile.write.schema.Schema;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
public class CompactionTableModelTestFileWriter extends
CompactionTestFileWriter {
@@ -61,6 +65,22 @@ public class CompactionTableModelTestFileWriter extends
CompactionTestFileWriter
schema.registerTableSchema(tableSchema);
}
+ public void registerTableSchemaWithTagAndTime(String tableName, List<String>
idColumnNames) {
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
+ List<ColumnCategory> columnTypes = new ArrayList<>();
+ for (String idColumnName : idColumnNames) {
+ measurementSchemas.add(
+ new MeasurementSchema(
+ idColumnName, TSDataType.STRING, TSEncoding.PLAIN,
CompressionType.UNCOMPRESSED));
+ columnTypes.add(ColumnCategory.TAG);
+ }
+ measurementSchemas.add(
+ new MeasurementSchema(
+ "time", TSDataType.INT64, TSEncoding.PLAIN,
CompressionType.UNCOMPRESSED));
+ columnTypes.add(ColumnCategory.TIME);
+ schema.registerTableSchema(new TableSchema(tableName, measurementSchemas,
columnTypes));
+ }
+
public void registerTableSchemaWithTagField(String tableName, List<String>
columnNames) {
List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
List<ColumnCategory> columnTypes = new ArrayList<>();
@@ -75,6 +95,19 @@ public class CompactionTableModelTestFileWriter extends
CompactionTestFileWriter
schema.registerTableSchema(new TableSchema(tableName, measurementSchemas,
columnTypes));
}
+ @TestOnly
+ public void generateTimeOnlyDataToCurrentDevice(TimeRange[] timeRanges)
throws IOException {
+ for (TimeRange timeRange : timeRanges) {
+ AlignedChunkWriterImpl chunkWriter = new
AlignedChunkWriterImpl(Collections.emptyList());
+ currentDeviceStartTime = Math.min(timeRange.getMin(),
currentDeviceStartTime);
+ currentDeviceEndTime = Math.max(timeRange.getMax(),
currentDeviceEndTime);
+ for (long time = timeRange.getMin(); time <= timeRange.getMax(); time++)
{
+ chunkWriter.getTimeChunkWriter().write(time);
+ }
+ chunkWriter.writeToFileWriter(fileWriter);
+ }
+ }
+
public IDeviceID startChunkGroup(String tableName, List<String>
idColumnFields)
throws IOException {
int idx = 0;
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTestFileWriterTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTestFileWriterTest.java
index 70431d61f07..05f345b7bbc 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTestFileWriterTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTestFileWriterTest.java
@@ -22,11 +22,14 @@ package
org.apache.iotdb.db.storageengine.dataregion.compaction.utils;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.db.exception.StorageEngineException;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest;
+import
org.apache.iotdb.db.storageengine.dataregion.compaction.tablemodel.CompactionTableModelTestFileWriter;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.tsfile.enums.ColumnCategory;
import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.TsFileSequenceReader;
import org.apache.tsfile.read.common.TimeRange;
import org.junit.After;
import org.junit.Assert;
@@ -35,6 +38,7 @@ import org.junit.Test;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
public class CompactionTestFileWriterTest extends AbstractCompactionTest {
@Before
@@ -196,4 +200,23 @@ public class CompactionTestFileWriterTest extends
AbstractCompactionTest {
Assert.fail();
}
}
+
+ @Test
+ public void testFieldlessTableSchemaWithTimeDataIsRetained() throws
IOException {
+ final TsFileResource resource = createEmptyFileAndResource(true);
+ try (final CompactionTableModelTestFileWriter writer =
+ new CompactionTableModelTestFileWriter(resource)) {
+ writer.registerTableSchemaWithTagAndTime("table1",
Collections.singletonList("tag1"));
+ writer.startChunkGroup("table1",
Collections.singletonList("tag1_value"));
+ writer.generateTimeOnlyDataToCurrentDevice(new TimeRange[] {new
TimeRange(1, 3)});
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ try (final TsFileSequenceReader reader =
+ new TsFileSequenceReader(resource.getTsFile().getAbsolutePath())) {
+ Assert.assertTrue(reader.getTableSchemaMap().containsKey("table1"));
+ Assert.assertFalse(
+
reader.getTableSchemaMap().get("table1").getColumnTypes().contains(ColumnCategory.FIELD));
+ }
+ }
}
