This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch
fetchSchemaWhenDataTypeNotMatchedInCompaction
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to
refs/heads/fetchSchemaWhenDataTypeNotMatchedInCompaction by this push:
new 5b635a6b791 fix bugs
5b635a6b791 is described below
commit 5b635a6b79190e49de10b9382e11ce85780fb2ef
Author: shuwenwei <[email protected]>
AuthorDate: Mon Mar 16 10:01:24 2026 +0800
fix bugs
---
.../execute/utils/MultiTsFileDeviceIterator.java | 3 +
.../ReadChunkAlignedSeriesCompactionExecutor.java | 6 +-
.../AbstractCompactionAlterDataTypeTest.java | 1 +
.../alterDataType/CompactionDataTypeAlterTest.java | 14 ++--
...ctionDataTypeNotMatchAlterableDataTypeTest.java | 93 ++++++++--------------
.../CompactionDataTypeNotMatchTest.java | 27 ++++---
6 files changed, 67 insertions(+), 77 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
index f2c0a49575d..11225bccbef 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -359,6 +359,9 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
measurementSchema =
reader.getMeasurementSchema(timeseriesMetadata.getChunkMetadataList());
}
+ if (measurementSchema == null) {
+ continue;
+ }
schemaMap.put(timeseriesMetadata.getMeasurementId(),
measurementSchema);
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
index 220e45dfb36..f33f2c276a4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
@@ -170,8 +170,10 @@ public class ReadChunkAlignedSeriesCompactionExecutor {
||
measurementSchemaMap.containsKey(chunkMetadata.getMeasurementUid())) {
continue;
}
- TsTableColumnSchema schemaInTsTable =
- tsTable.getColumnSchema(chunkMetadata.getMeasurementUid());
+ TsTableColumnSchema schemaInTsTable = null;
+ if (tsTable != null) {
+ schemaInTsTable =
tsTable.getColumnSchema(chunkMetadata.getMeasurementUid());
+ }
IMeasurementSchema measurementSchema;
if (schemaInTsTable == null) {
ChunkHeader chunkHeader =
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/AbstractCompactionAlterDataTypeTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/AbstractCompactionAlterDataTypeTest.java
index d97b66b68b7..6d21b70ebec 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/AbstractCompactionAlterDataTypeTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/AbstractCompactionAlterDataTypeTest.java
@@ -67,6 +67,7 @@ public class AbstractCompactionAlterDataTypeTest extends
AbstractCompactionTest
@After
public void tearDown() throws IOException, StorageEngineException {
+ CompactionUtils.setSchemaFetcher(null);
super.tearDown();
Thread.currentThread().setName(oldThreadName);
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTest.java
index d20e0307a32..2826b8a70e7 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTest.java
@@ -23,9 +23,7 @@ import
org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.db.exception.StorageEngineException;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import
org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
@@ -69,12 +67,12 @@ public class CompactionDataTypeAlterTest extends
AbstractCompactionAlterDataType
public static Collection<Object[]> data() {
return Arrays.asList(
new Object[][] {
- {true, "read_chunk"},
- {false, "read_chunk"},
- {true, "fast"},
- {false, "fast"},
- {true, "read_point"},
- {false, "read_point"},
+ {true, "read_chunk"},
+ {false, "read_chunk"},
+ {true, "fast"},
+ {false, "fast"},
+ {true, "read_point"},
+ {false, "read_point"},
});
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchAlterableDataTypeTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchAlterableDataTypeTest.java
index c7782a411c0..8a7a704dfbd 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchAlterableDataTypeTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchAlterableDataTypeTest.java
@@ -19,11 +19,10 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.alterDataType;
+import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.db.exception.StorageEngineException;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
@@ -43,15 +42,22 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
@SuppressWarnings("OptionalGetWithoutIsPresent")
+@RunWith(Parameterized.class)
public class CompactionDataTypeNotMatchAlterableDataTypeTest
extends AbstractCompactionAlterDataTypeTest {
+ private String performerType;
+
@Before
public void setUp()
throws IOException, WriteProcessException, MetadataException,
InterruptedException {
@@ -63,39 +69,25 @@ public class CompactionDataTypeNotMatchAlterableDataTypeTest
super.tearDown();
}
- @Test
- public void testCompactNonAlignedSeriesWithReadChunkCompactionPerformer()
- throws IOException, WriteProcessException {
- generateDataTypeNotMatchFilesWithNonAlignedSeries();
- InnerSpaceCompactionTask task =
- new InnerSpaceCompactionTask(
- 0, tsFileManager, seqResources, true, new
ReadChunkCompactionPerformer(), 0);
- Assert.assertTrue(task.start());
-
TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0));
- Assert.assertEquals(
- 1, ((long)
tsFileManager.getTsFileList(true).get(0).getStartTime(device).get()));
+ @Parameterized.Parameters(name = "type={0}")
+ public static Collection<Object[]> data() {
+ return Arrays.asList(
+ new Object[][] {
+ {"read_chunk"}, {"fast"}, {"read_point"},
+ });
}
- @Test
- public void testCompactNonAlignedSeriesWithFastCompactionPerformer()
- throws IOException, WriteProcessException {
- generateDataTypeNotMatchFilesWithNonAlignedSeries();
- InnerSpaceCompactionTask task =
- new InnerSpaceCompactionTask(
- 0, tsFileManager, seqResources, true, new
FastCompactionPerformer(false), 0);
- Assert.assertTrue(task.start());
-
TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0));
- Assert.assertEquals(
- 1, ((long)
tsFileManager.getTsFileList(true).get(0).getStartTime(device).get()));
+ public CompactionDataTypeNotMatchAlterableDataTypeTest(String type) {
+ this.performerType = type;
}
@Test
- public void testCompactNonAlignedSeriesWithReadPointCompactionPerformer()
- throws IOException, WriteProcessException {
+ public void testCompactNonAlignedSeries()
+ throws IOException, WriteProcessException, IllegalPathException {
generateDataTypeNotMatchFilesWithNonAlignedSeries();
InnerSpaceCompactionTask task =
new InnerSpaceCompactionTask(
- 0, tsFileManager, seqResources, true, new
ReadPointCompactionPerformer(), 0);
+ 0, tsFileManager, seqResources, true, getPerformer(performerType),
0);
Assert.assertTrue(task.start());
TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0));
Assert.assertEquals(
@@ -104,37 +96,11 @@ public class
CompactionDataTypeNotMatchAlterableDataTypeTest
@Test
public void testCompactAlignedSeriesWithReadChunkCompactionPerformer()
- throws IOException, WriteProcessException {
+ throws IOException, WriteProcessException, IllegalPathException {
generateDataTypeNotMatchFilesWithAlignedSeries();
InnerSpaceCompactionTask task =
new InnerSpaceCompactionTask(
- 0, tsFileManager, seqResources, true, new
ReadChunkCompactionPerformer(), 0);
- Assert.assertTrue(task.start());
-
TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0));
- Assert.assertEquals(
- 1, ((long)
tsFileManager.getTsFileList(true).get(0).getStartTime(device).get()));
- }
-
- @Test
- public void testCompactAlignedSeriesWithFastCompactionPerformer()
- throws IOException, WriteProcessException {
- generateDataTypeNotMatchFilesWithAlignedSeries();
- InnerSpaceCompactionTask task =
- new InnerSpaceCompactionTask(
- 0, tsFileManager, seqResources, true, new
FastCompactionPerformer(false), 0);
- Assert.assertTrue(task.start());
-
TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0));
- Assert.assertEquals(
- 1, ((long)
tsFileManager.getTsFileList(true).get(0).getStartTime(device).get()));
- }
-
- @Test
- public void testCompactAlignedSeriesWithReadPointCompactionPerformer()
- throws IOException, WriteProcessException {
- generateDataTypeNotMatchFilesWithAlignedSeries();
- InnerSpaceCompactionTask task =
- new InnerSpaceCompactionTask(
- 0, tsFileManager, seqResources, true, new
ReadPointCompactionPerformer(), 0);
+ 0, tsFileManager, seqResources, true, getPerformer(performerType),
0);
Assert.assertTrue(task.start());
TsFileResourceUtils.validateTsFileDataCorrectness(tsFileManager.getTsFileList(true).get(0));
Assert.assertEquals(
@@ -142,7 +108,7 @@ public class CompactionDataTypeNotMatchAlterableDataTypeTest
}
private void generateDataTypeNotMatchFilesWithNonAlignedSeries()
- throws IOException, WriteProcessException {
+ throws IOException, WriteProcessException, IllegalPathException {
MeasurementSchema measurementSchema1 = new MeasurementSchema("s1",
TSDataType.INT32);
TsFileResource resource1 = generateInt32NonAlignedSeriesFile(new
TimeRange(1, 1), true);
seqResources.add(resource1);
@@ -150,10 +116,14 @@ public class
CompactionDataTypeNotMatchAlterableDataTypeTest
MeasurementSchema measurementSchema2 = new MeasurementSchema("s1",
TSDataType.FLOAT);
TsFileResource resource2 = generateFloatNonAlignedSeriesFile(new
TimeRange(2, 2), true);
seqResources.add(resource2);
+
+ schemaFetcher
+ .getSchemaTree()
+ .appendSingleMeasurementPath(new MeasurementPath(device, "s1",
measurementSchema2));
}
private void generateDataTypeNotMatchFilesWithAlignedSeries()
- throws IOException, WriteProcessException {
+ throws IOException, WriteProcessException, IllegalPathException {
List<IMeasurementSchema> measurementSchemas1 = new ArrayList<>();
measurementSchemas1.add(new MeasurementSchema("s1", TSDataType.INT32));
measurementSchemas1.add(new MeasurementSchema("s2", TSDataType.INT32));
@@ -178,5 +148,12 @@ public class
CompactionDataTypeNotMatchAlterableDataTypeTest
resource2.updateEndTime(device, 2);
resource2.serialize();
seqResources.add(resource2);
+
+ MeasurementPath s1Path = new MeasurementPath(device, "s1",
measurementSchemas2.get(0));
+ s1Path.setUnderAlignedEntity(true);
+ MeasurementPath s2Path = new MeasurementPath(device, "s2",
measurementSchemas2.get(1));
+ s2Path.setUnderAlignedEntity(true);
+ schemaFetcher.getSchemaTree().appendSingleMeasurementPath(s1Path);
+ schemaFetcher.getSchemaTree().appendSingleMeasurementPath(s2Path);
}
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchTest.java
index 3b4dcacb8fa..fdaeebbd612 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeNotMatchTest.java
@@ -19,11 +19,10 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.alterDataType;
+import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.db.exception.StorageEngineException;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
@@ -74,7 +73,7 @@ public class CompactionDataTypeNotMatchTest extends
AbstractCompactionAlterDataT
public static Collection<Object[]> data() {
return Arrays.asList(
new Object[][] {
- {"read_chunk"}, {"fast"}, {"read_point"},
+ {"read_chunk"}, {"fast"}, {"read_point"},
});
}
@@ -84,7 +83,7 @@ public class CompactionDataTypeNotMatchTest extends
AbstractCompactionAlterDataT
@Test
public void testCompactNonAlignedSeries()
- throws IOException, WriteProcessException {
+ throws IOException, WriteProcessException, IllegalPathException {
generateDataTypeNotMatchFilesWithNonAlignedSeries();
InnerSpaceCompactionTask task =
new InnerSpaceCompactionTask(
@@ -97,7 +96,7 @@ public class CompactionDataTypeNotMatchTest extends
AbstractCompactionAlterDataT
@Test
public void testCompactAlignedSeries()
- throws IOException, WriteProcessException {
+ throws IOException, WriteProcessException, IllegalPathException {
generateDataTypeNotMatchFilesWithAlignedSeries();
InnerSpaceCompactionTask task =
new InnerSpaceCompactionTask(
@@ -108,9 +107,8 @@ public class CompactionDataTypeNotMatchTest extends
AbstractCompactionAlterDataT
2, ((long)
tsFileManager.getTsFileList(true).get(0).getStartTime(device).get()));
}
-
private void generateDataTypeNotMatchFilesWithNonAlignedSeries()
- throws IOException, WriteProcessException {
+ throws IOException, WriteProcessException, IllegalPathException {
MeasurementSchema measurementSchema1 = new MeasurementSchema("s1",
TSDataType.BOOLEAN);
TsFileResource resource1 = createEmptyFileAndResource(true);
resource1.setStatusForTest(TsFileResourceStatus.COMPACTING);
@@ -129,10 +127,14 @@ public class CompactionDataTypeNotMatchTest extends
AbstractCompactionAlterDataT
MeasurementSchema measurementSchema2 = new MeasurementSchema("s1",
TSDataType.INT32);
TsFileResource resource2 = generateInt32NonAlignedSeriesFile(new
TimeRange(2, 2), true);
seqResources.add(resource2);
+
+ schemaFetcher
+ .getSchemaTree()
+ .appendSingleMeasurementPath(new MeasurementPath(device, "s1",
measurementSchema2));
}
private void generateDataTypeNotMatchFilesWithAlignedSeries()
- throws IOException, WriteProcessException {
+ throws IOException, WriteProcessException, IllegalPathException {
List<IMeasurementSchema> measurementSchemas1 = new ArrayList<>();
measurementSchemas1.add(new MeasurementSchema("s1", TSDataType.INT32));
measurementSchemas1.add(new MeasurementSchema("s2", TSDataType.INT32));
@@ -157,5 +159,12 @@ public class CompactionDataTypeNotMatchTest extends
AbstractCompactionAlterDataT
resource2.updateEndTime(device, 2);
resource2.serialize();
seqResources.add(resource2);
+
+ MeasurementPath s1Path = new MeasurementPath(device, "s1",
measurementSchemas2.get(0));
+ s1Path.setUnderAlignedEntity(true);
+ MeasurementPath s2Path = new MeasurementPath(device, "s2",
measurementSchemas2.get(1));
+ s2Path.setUnderAlignedEntity(true);
+ schemaFetcher.getSchemaTree().appendSingleMeasurementPath(s1Path);
+ schemaFetcher.getSchemaTree().appendSingleMeasurementPath(s2Path);
}
}
