This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch AlignedQueryBug in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a5fd903282ce80c27a9731bc9d7f55fe7e6f8c4f Author: JackieTien97 <[email protected]> AuthorDate: Sun Jan 26 09:29:55 2025 +0800 TryFix --- .../execution/operator/source/FileLoaderUtils.java | 4 +-- .../schemaregion/utils/ResourceByPathUtils.java | 31 ++++++++++++++++------ .../memtable/AlignedReadOnlyMemChunk.java | 10 ++++--- .../read/reader/chunk/DiskAlignedChunkLoader.java | 3 ++- .../metadata/DiskAlignedChunkMetadataLoader.java | 4 +-- .../apache/iotdb/db/utils/ModificationUtils.java | 9 ++++--- pom.xml | 2 +- 7 files changed, 42 insertions(+), 21 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index cd00aa5e419..213e4990a5f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -41,7 +41,7 @@ import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; -import org.apache.tsfile.file.metadata.TableDeviceMetadata; +import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.controller.IChunkLoader; import org.apache.tsfile.read.filter.basic.Filter; @@ -395,7 +395,7 @@ public class FileLoaderUtils { AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata = ignoreAllNullRows ? new AlignedTimeSeriesMetadata(timeColumnMetadata, valueColumnMetadataList) - : new TableDeviceMetadata(timeColumnMetadata, valueColumnMetadataList); + : new TableDeviceTimeSeriesMetadata(timeColumnMetadata, valueColumnMetadataList); alignedTimeSeriesMetadata.setChunkMetadataLoader( new DiskAlignedChunkMetadataLoader( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index dbd32b47276..12393fb2a20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -37,12 +37,16 @@ import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; +import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; +import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -108,7 +112,7 @@ class AlignedResourceByPathUtils extends ResourceByPathUtils { * have chunkMetadata, but query will use these, so we need to generate it for them. */ @Override - public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( + public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata( List<ReadOnlyMemChunk> readOnlyMemChunk, List<IChunkMetadata> chunkMetadataList) { TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata(); timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1); @@ -131,8 +135,11 @@ class AlignedResourceByPathUtils extends ResourceByPathUtils { boolean[] exist = new boolean[alignedFullPath.getSchemaList().size()]; boolean modified = false; + boolean isTable = false; for (IChunkMetadata chunkMetadata : chunkMetadataList) { - AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadata; + isTable = isTable || (alignedChunkMetadata instanceof TableDeviceChunkMetadata); modified = (modified || alignedChunkMetadata.isModified()); timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { @@ -149,8 +156,9 @@ class AlignedResourceByPathUtils extends ResourceByPathUtils { for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { if (!memChunk.isEmpty()) { - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) memChunk.getChunkMetaData(); + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) memChunk.getChunkMetaData(); + isTable = isTable || (alignedChunkMetadata instanceof TableDeviceChunkMetadata); timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { @@ -174,7 +182,9 @@ class AlignedResourceByPathUtils extends ResourceByPathUtils { } } - return new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); + return isTable + ? new TableDeviceTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList) + : new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); } @Override @@ -268,7 +278,7 @@ class AlignedResourceByPathUtils extends ResourceByPathUtils { QueryContext context, long timeLowerBound) { - List<AlignedChunkMetadata> chunkMetadataList = new ArrayList<>(); + List<AbstractAlignedChunkMetadata> chunkMetadataList = new ArrayList<>(); List<ChunkMetadata> timeChunkMetadataList = writer.getVisibleMetadataList( alignedFullPath.getDeviceId(), AlignedFullPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR); @@ -285,7 +295,10 @@ class AlignedResourceByPathUtils extends ResourceByPathUtils { // only need time column if (alignedFullPath.getMeasurementList().isEmpty()) { chunkMetadataList.add( - new AlignedChunkMetadata(timeChunkMetadataList.get(i), Collections.emptyList())); + context.isIgnoreAllNullRows() + ? new AlignedChunkMetadata(timeChunkMetadataList.get(i), Collections.emptyList()) + : new TableDeviceChunkMetadata( + timeChunkMetadataList.get(i), Collections.emptyList())); } else { List<IChunkMetadata> valueChunkMetadata = new ArrayList<>(); // if all the sub sensors doesn't exist, it will be false @@ -298,7 +311,9 @@ class AlignedResourceByPathUtils extends ResourceByPathUtils { } if (!context.isIgnoreAllNullRows() || exits) { chunkMetadataList.add( - new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata)); + context.isIgnoreAllNullRows() + ? new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata) + : new TableDeviceChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index 569db7347ab..a3eb66b018d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -30,6 +30,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; +import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.TimeRange; @@ -75,10 +76,11 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk { timeColumnDeletion, valueColumnsDeletionList, context.isIgnoreAllNullRows()); - initAlignedChunkMetaFromTsBlock(); + initAlignedChunkMetaFromTsBlock(context.isIgnoreAllNullRows()); } - private void initAlignedChunkMetaFromTsBlock() throws QueryProcessException { + private void initAlignedChunkMetaFromTsBlock(boolean ignoreAllNullRows) + throws QueryProcessException { // Time chunk Statistics timeStatistics = Statistics.getStatsByType(TSDataType.VECTOR); IChunkMetadata timeChunkMetadata = @@ -160,7 +162,9 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk { } } IChunkMetadata alignedChunkMetadata = - new AlignedChunkMetadata(timeChunkMetadata, valueChunkMetadataList); + ignoreAllNullRows + ? new AlignedChunkMetadata(timeChunkMetadata, valueChunkMetadataList) + : new TableDeviceChunkMetadata(timeChunkMetadata, valueChunkMetadataList); alignedChunkMetadata.setChunkLoader(new MemAlignedChunkLoader(context, this)); alignedChunkMetadata.setVersion(Long.MAX_VALUE); cachedMetaData = alignedChunkMetadata; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java index adef246ec8d..b0cfd3c0362 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.storageengine.buffer.ChunkCache; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -76,7 +77,7 @@ public class DiskAlignedChunkLoader implements IChunkLoader { throws IOException { long t1 = System.nanoTime(); try { - AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetaData; + AbstractAlignedChunkMetadata alignedChunkMetadata = (AbstractAlignedChunkMetadata) chunkMetaData; ChunkMetadata timeChunkMetadata = (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata(); Chunk timeChunk = ChunkCache.getInstance() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java index afc57079c88..f243b51ee1a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java @@ -26,8 +26,8 @@ import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskAligne import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; import org.apache.tsfile.read.controller.IChunkMetadataLoader; @@ -83,7 +83,7 @@ public class DiskAlignedChunkMetadataLoader implements IChunkMetadataLoader { public List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata) { final long t1 = System.nanoTime(); try { - List<AlignedChunkMetadata> alignedChunkMetadataList = + List<AbstractAlignedChunkMetadata> alignedChunkMetadataList = ((AbstractAlignedTimeSeriesMetadata) timeSeriesMetadata).getCopiedChunkMetadataList(); // when alignedChunkMetadataList.size() == 1, it means that the chunk statistics is same as diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java index 59966207648..f9a6d28e2b9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -101,7 +102,7 @@ public class ModificationUtils { } private static void modifyValueColumns( - AlignedChunkMetadata metaData, List<List<ModEntry>> valueColumnsModifications) { + AbstractAlignedChunkMetadata metaData, List<List<ModEntry>> valueColumnsModifications) { List<IChunkMetadata> valueChunkMetadataList = metaData.getValueChunkMetadataList(); // deal with each sub sensor for (int j = 0; j < valueChunkMetadataList.size(); j++) { @@ -116,7 +117,7 @@ public class ModificationUtils { } private static boolean areAllValueColumnsDeleted( - AlignedChunkMetadata alignedChunkMetadata, boolean modified) { + AbstractAlignedChunkMetadata alignedChunkMetadata, boolean modified) { // the whole aligned path need to be removed, only set to be true if all the sub sensors // are deleted and ignoreAllNullRows is true @@ -156,11 +157,11 @@ public class ModificationUtils { } public static void modifyAlignedChunkMetaData( - List<AlignedChunkMetadata> chunkMetaData, + List<? extends AbstractAlignedChunkMetadata> chunkMetaData, List<ModEntry> timeColumnModifications, List<List<ModEntry>> valueColumnsModifications, boolean ignoreAllNullRows) { - for (AlignedChunkMetadata metaData : chunkMetaData) { + for (AbstractAlignedChunkMetadata metaData : chunkMetaData) { IChunkMetadata timeColumnChunkMetadata = metaData.getTimeChunkMetadata(); for (ModEntry modification : timeColumnModifications) { diff --git a/pom.xml b/pom.xml index 40164fc32ed..2e6bdf4efae 100644 --- a/pom.xml +++ b/pom.xml @@ -167,7 +167,7 @@ <thrift.version>0.14.1</thrift.version> <xz.version>1.9</xz.version> <zstd-jni.version>1.5.6-3</zstd-jni.version> - <tsfile.version>2.0.0-250118-SNAPSHOT</tsfile.version> + <tsfile.version>2.1.0-SNAPSHOT</tsfile.version> </properties> <!-- if we claim dependencies in dependencyManagement, then we do not claim
