This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch optimizeMemtableScan in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 05d578231cab987c4cf0d92631390c46efb7c326 Merge: de65501fd57 72a11f37168 Author: shuwenwei <[email protected]> AuthorDate: Mon Aug 25 10:30:12 2025 +0800 Merge branch 'iotdb_master' into optimizeMemtableScan LICENSE-binary | 59 +-- NOTICE | 2 +- NOTICE-binary | 2 +- .../org/apache/iotdb/it/utils/TsFileGenerator.java | 28 ++ .../iotdb/it/utils/TsFileTableGenerator.java | 17 +- .../IoTDBRegionGroupExpandAndShrinkForIoTV1IT.java | 329 ++++++++++++++- .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 155 +------ .../org/apache/iotdb/db/it/IoTDBRestartIT.java | 59 +++ .../treemodel/auto/basic/IoTDBPipeDataSinkIT.java | 13 +- .../relational/it/db/it/IoTDBLoadTsFileIT.java | 299 ++++++++++++++ .../it/insertquery/IoTDBInsertQueryIT.java | 12 +- .../it/query/recent/IoTDBUnionTableIT.java | 127 ++++++ .../iotdb/session/it/IoTDBSessionSimpleIT.java | 180 ++++++++- .../org/apache/iotdb/tool/common/Constants.java | 2 + .../iotdb/tool/schema/ExportSchemaTable.java | 44 +- iotdb-core/ainode/ainode/core/constant.py | 6 - .../{scheduler => dispatcher}/__init__.py | 0 .../dispatcher/abstract_dispatcher.py} | 31 +- .../core/inference/dispatcher/basic_dispatcher.py | 60 +++ .../ainode/core/inference/inference_request.py | 2 + .../core/inference/inference_request_pool.py | 19 +- .../ainode/core/inference/pool_controller.py | 188 +++++++++ .../ainode/ainode/core/inference/pool_group.py | 93 +++++ .../{scheduler => pool_scheduler}/__init__.py | 0 .../pool_scheduler/abstract_pool_scheduler.py | 56 +++ .../pool_scheduler/basic_pool_scheduler.py | 59 +++ .../{scheduler => request_scheduler}/__init__.py | 0 .../abstract_request_scheduler.py} | 2 +- .../basic_request_scheduler.py} | 9 +- iotdb-core/ainode/ainode/core/log.py | 13 + .../ainode/core/manager/inference_manager.py | 134 ++---- iotdb-core/ainode/ainode/core/manager/utils.py | 8 +- iotdb-core/ainode/ainode/core/util/decorator.py | 15 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 4 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 10 + .../confignode/conf/ConfigNodeDescriptor.java | 6 + .../exception/NoAvailableRegionGroupException.java | 14 +- .../iotdb/confignode/manager/ConfigManager.java | 4 +- .../iotdb/confignode/manager/ProcedureManager.java | 63 ++- .../manager/consensus/ConsensusManager.java | 5 + .../manager/partition/PartitionManager.java | 4 +- .../persistence/schema/TemplateTable.java | 2 +- .../apache/iotdb/consensus/config/RatisConfig.java | 27 +- .../exception/RatisRequestFailedException.java | 9 + .../iotdb/consensus/ratis/RatisConsensus.java | 23 +- iotdb-core/datanode/pom.xml | 4 - .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 21 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 27 ++ .../db/consensus/DataRegionConsensusImpl.java | 4 + .../db/consensus/SchemaRegionConsensusImpl.java | 5 + .../tsfile/parser/TsFileInsertionEventParser.java | 11 +- .../query/TsFileInsertionEventQueryParser.java | 215 +++++----- ...ileInsertionEventQueryParserTabletIterator.java | 14 +- .../scan/TsFileInsertionEventScanParser.java | 184 +++++---- .../table/TsFileInsertionEventTableParser.java | 256 ++++++------ ...ileInsertionEventTableParserTabletIterator.java | 79 ++-- .../processor/aggregate/AggregateProcessor.java | 5 +- .../pipeconsensus/PipeConsensusReceiver.java | 114 +++--- ...istoricalDataRegionTsFileAndDeletionSource.java | 11 +- .../realtime/PipeRealtimeDataRegionSource.java | 14 +- .../realtime/assigner/PipeDataRegionAssigner.java | 16 +- .../db/pipe/source/mqtt/MQTTPublishHandler.java | 21 +- .../iotdb/db/pipe/source/mqtt/MQTTSource.java | 102 ++++- .../protocol/rest/v2/impl/RestApiServiceImpl.java | 43 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 13 +- .../db/queryengine/common/MPPQueryContext.java | 7 +- .../queryengine/execution/driver/DataDriver.java | 44 +- .../fragment/FakedFragmentInstanceContext.java | 5 +- .../fragment/FragmentInstanceContext.java | 244 +++++++---- .../fragment/FragmentInstanceExecution.java | 8 +- .../execution/fragment/QueryContext.java | 5 +- .../queryengine/metric/QueryResourceMetricSet.java | 26 +- .../iotdb/db/queryengine/plan/Coordinator.java | 69 +++- .../queryengine/plan/analyze/PredicateUtils.java | 11 +- .../analyze/load/LoadTsFileTableSchemaCache.java | 19 +- .../analyze/load/LoadTsFileTreeSchemaCache.java | 24 +- .../load/TreeSchemaAutoCreatorAndVerifier.java | 2 +- .../analyze/schema/AutoCreateSchemaExecutor.java | 13 +- .../plan/analyze/schema/TemplateSchemaFetcher.java | 2 +- .../config/executor/ClusterConfigTaskExecutor.java | 4 +- .../config/metadata/region/ExtendRegionTask.java | 2 +- .../config/metadata/region/RemoveRegionTask.java | 2 +- .../db/queryengine/plan/parser/ASTVisitor.java | 19 +- .../plan/planner/LocalExecutionPlanner.java | 7 +- .../plan/planner/TableOperatorGenerator.java | 22 +- .../plan/AbstractFragmentParallelPlanner.java | 20 +- .../plan/planner/plan/node/PlanGraphPrinter.java | 9 + .../plan/planner/plan/node/PlanNodeType.java | 4 + .../plan/planner/plan/node/PlanVisitor.java | 5 + .../relational/analyzer/StatementAnalyzer.java | 6 +- .../fetcher/TableHeaderSchemaValidator.java | 4 +- .../plan/relational/planner/QueryPlanner.java | 34 ++ .../plan/relational/planner/RelationPlanner.java | 92 ++++- .../relational/planner/TableLogicalPlanner.java | 5 +- .../distribute/TableDistributedPlanGenerator.java | 7 + .../iterative/rule/PruneDistinctAggregation.java | 6 + .../planner/iterative/rule/PruneUnionColumns.java | 80 ++++ .../iterative/rule/PruneUnionSourceColumns.java | 52 +++ .../plan/relational/planner/node/Patterns.java | 4 + .../relational/planner/node/SetOperationNode.java | 140 +++++++ .../plan/relational/planner/node/UnionNode.java | 97 +++++ .../optimizations/LogicalOptimizeFactory.java | 6 +- .../optimizations/PushPredicateIntoTableScan.java | 32 +- .../TransformAggregationToStreamable.java | 6 + .../optimizations/TransformSortToStreamSort.java | 7 + .../optimizations/UnaliasSymbolReferences.java | 54 +++ .../plan/relational/sql/ast/ExtendRegion.java | 16 +- .../plan/relational/sql/ast/RemoveRegion.java | 16 +- .../relational/sql/ast/WrappedInsertStatement.java | 2 +- .../plan/relational/sql/parser/AstBuilder.java | 10 +- .../plan/relational/type/CompatibleResolver.java | 108 +++++ .../metadata/region/ExtendRegionStatement.java | 10 +- .../metadata/region/RemoveRegionStatement.java | 10 +- .../FragmentInstanceStatisticsDrawer.java | 11 + .../metrics/IoTDBInternalLocalReporter.java | 3 +- .../db/storageengine/dataregion/DataRegion.java | 450 ++++++++++++++------- .../dataregion/IDataRegionForQuery.java | 29 +- .../dataregion/VirtualDataRegion.java | 36 +- .../performer/impl/FastCompactionPerformer.java | 5 +- .../compaction/execute/utils/CompactionUtils.java | 36 ++ .../execute/utils/MultiTsFileDeviceIterator.java | 45 +-- .../FirstBatchCompactionAlignedChunkWriter.java | 3 +- .../executor/fast/SeriesCompactionExecutor.java | 19 +- .../selector/impl/SettleSelectorImpl.java | 23 +- .../memtable/AbstractWritableMemChunk.java | 11 + .../dataregion/memtable/TsFileProcessor.java | 65 ++- .../dataregion/tsfile/TsFileManager.java | 43 +- .../db/storageengine/load/LoadTsFileManager.java | 22 +- .../memory/LoadTsFileDataCacheMemoryBlock.java | 8 +- .../load/memory/LoadTsFileMemoryManager.java | 15 +- .../apache/iotdb/db/utils/ModificationUtils.java | 59 +-- .../db/queryengine/execution/DataDriverTest.java | 15 +- .../plan/relational/analyzer/UnionTest.java | 65 +++ .../InformationSchemaTableScanMatcher.java | 2 +- .../planner/assertions/PlanMatchPattern.java | 5 + .../tablemodel/CompactionWithAllNullRowsTest.java | 73 +++- iotdb-core/metrics/interface/pom.xml | 1 - .../conf/iotdb-system.properties.template | 16 +- .../commons/auth/authorizer/OpenIdAuthorizer.java | 10 +- .../apache/iotdb/commons/conf/CommonConfig.java | 40 +- .../iotdb/commons/enums/ReadConsistencyLevel.java} | 15 +- .../apache/iotdb/commons/path/PatternTreeMap.java | 4 + .../iotdb/commons/pipe/config/PipeConfig.java | 8 +- .../iotdb/commons/pipe/config/PipeDescriptor.java | 8 +- .../pipe/config/constant/PipeSourceConstant.java | 3 +- .../commons/pipe/event/ProgressReportEvent.java | 33 +- .../pipe/source/IoTDBNonDataRegionSource.java | 11 +- .../db/relational/grammar/sql/RelationalSql.g4 | 6 +- .../src/main/thrift/confignode.thrift | 6 +- .../src/main/thrift/datanode.thrift | 1 + pom.xml | 12 +- 151 files changed, 4534 insertions(+), 1413 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java index 165a218a9b5,95d90dfa63d..351e64650a7 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java @@@ -20,7 -20,9 +20,10 @@@ package org.apache.iotdb.db.utils; import org.apache.iotdb.commons.exception.IllegalPathException; + import org.apache.iotdb.commons.path.AlignedPath; + import org.apache.iotdb.commons.path.PatternTreeMap; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; + import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionPathUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.impl.SettleSelectorImpl; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
