This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch update_last_cache_in_load in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2ef2f2ba53e82ba51f1d7268c2ffbd43f56519e8 Merge: 566c2b8984d f6b2d07a61b Author: Tian Jiang <[email protected]> AuthorDate: Thu May 29 11:43:38 2025 +0800 Merge branch 'master' into update_last_cache_in_load # Conflicts: # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java LICENSE | 15 + .../server/CustomizedJsonPayloadFormatter.java | 9 +- .../iotdb/it/env/cluster/node/DataNodeWrapper.java | 1 + .../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 22 +- .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 4 +- .../iotdb/db/it/IoTDBSetConfigurationIT.java | 12 +- .../manual/basic/IoTDBPipePermissionIT.java | 43 ++ .../db/it/IoTDBMultiTAGsWithAttributesTableIT.java | 29 +- .../it/query/recent/IoTDBMaintainAuthIT.java | 18 +- .../it/query/recent/IoTDBTableAggregationIT.java | 53 ++ .../view/old/query/IoTDBPaginationTableViewIT.java | 15 + .../relational/it/schema/IoTDBDatabaseIT.java | 122 +++- .../iotdb/relational/it/schema/IoTDBTableIT.java | 14 +- .../IoTDBSnapshotDevicePullConsumerDataSetIT.java | 3 + .../java/org/apache/iotdb/cli/AbstractCli.java | 4 +- iotdb-client/client-cpp/pom.xml | 86 +-- iotdb-client/client-cpp/src/main/Column.cpp | 359 ++++++++++++ iotdb-client/client-cpp/src/main/Column.h | 353 ++++++++++++ iotdb-client/client-cpp/src/main/ColumnDecoder.cpp | 178 ++++++ iotdb-client/client-cpp/src/main/ColumnDecoder.h | 75 +++ iotdb-client/client-cpp/src/main/Common.cc | 186 ------- iotdb-client/client-cpp/src/main/Common.cpp | 456 +++++++++++++++ iotdb-client/client-cpp/src/main/Common.h | 118 +++- .../client-cpp/src/main/IoTDBRpcDataSet.cpp | 614 +++++++++++++++++++++ iotdb-client/client-cpp/src/main/IoTDBRpcDataSet.h | 159 ++++++ iotdb-client/client-cpp/src/main/NodesSupplier.cpp | 61 +- iotdb-client/client-cpp/src/main/NodesSupplier.h | 30 +- iotdb-client/client-cpp/src/main/Session.cpp | 384 ++++--------- iotdb-client/client-cpp/src/main/Session.h | 576 ++----------------- .../client-cpp/src/main/SessionConnection.cpp | 47 +- .../client-cpp/src/main/SessionConnection.h | 1 + .../client-cpp/src/main/SessionDataSet.cpp | 252 +++++++++ iotdb-client/client-cpp/src/main/SessionDataSet.h | 146 +++++ iotdb-client/client-cpp/src/main/TableSession.cpp | 15 +- iotdb-client/client-cpp/src/main/TableSession.h | 10 +- .../client-cpp/src/main/ThriftConnection.cpp | 109 ++-- .../client-cpp/src/main/ThriftConnection.h | 25 +- iotdb-client/client-cpp/src/main/TsBlock.cpp | 109 ++++ iotdb-client/client-cpp/src/main/TsBlock.h | 55 ++ iotdb-client/client-cpp/src/test/cpp/sessionIT.cpp | 13 +- .../src/test/cpp/sessionRelationalIT.cpp | 112 +++- .../iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java | 2 +- .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 6 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 9 +- .../IoTDBRelationalDatabaseMetadata.java | 8 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 5 +- .../response/PipeSubscribeHeartbeatResp.java | 17 +- .../apache/iotdb/session/SessionConnection.java | 3 + .../base/AbstractSubscriptionProviders.java | 13 +- iotdb-core/ainode/ainode/core/constant.py | 1 + .../ainode/core/model/built_in_model_factory.py | 4 +- .../exception/DatabaseNotExistsException.java | 2 +- .../iotdb/confignode/manager/ConfigManager.java | 29 + .../apache/iotdb/confignode/manager/IManager.java | 8 + .../load/cache/detector/PhiAccrualDetector.java | 2 +- .../manager/load/service/TopologyService.java | 2 +- .../iotdb/confignode/manager/node/NodeManager.java | 88 +++ .../iotdb/confignode/persistence/ModelInfo.java | 8 +- .../confignode/procedure/InternalProcedure.java | 6 +- .../iotdb/confignode/procedure/Procedure.java | 62 +-- .../confignode/procedure/ProcedureExecutor.java | 4 - .../exception/ProcedureAbortedException.java | 30 - .../exception/ProcedureTimeoutException.java | 26 - .../exception/ProcedureYieldException.java | 36 -- .../procedure/impl/StateMachineProcedure.java | 8 +- .../procedure/impl/cq/CreateCQProcedure.java | 4 +- .../procedure/impl/node/RemoveAINodeProcedure.java | 4 +- .../impl/pipe/AbstractOperatePipeProcedureV2.java | 4 +- .../pipe/plugin/CreatePipePluginProcedure.java | 4 +- .../impl/pipe/plugin/DropPipePluginProcedure.java | 4 +- .../impl/region/AddRegionPeerProcedure.java | 4 +- .../region/NotifyRegionMigrationProcedure.java | 4 +- .../impl/region/ReconstructRegionProcedure.java | 4 +- .../impl/region/RemoveRegionPeerProcedure.java | 4 +- .../impl/schema/AlterLogicalViewProcedure.java | 4 +- .../impl/schema/DeactivateTemplateProcedure.java | 4 +- .../impl/schema/DeleteDatabaseProcedure.java | 4 +- .../impl/schema/DeleteLogicalViewProcedure.java | 4 +- .../impl/schema/DeleteTimeSeriesProcedure.java | 4 +- .../procedure/impl/schema/SetTTLProcedure.java | 4 +- .../impl/schema/SetTemplateProcedure.java | 4 +- .../impl/schema/UnsetTemplateProcedure.java | 4 +- .../impl/schema/table/AddTableColumnProcedure.java | 4 +- .../impl/schema/table/CreateTableProcedure.java | 4 +- .../impl/schema/table/DeleteDevicesProcedure.java | 4 +- .../schema/table/DropTableColumnProcedure.java | 4 +- .../impl/schema/table/DropTableProcedure.java | 4 +- .../schema/table/RenameTableColumnProcedure.java | 4 +- .../impl/schema/table/RenameTableProcedure.java | 4 +- .../schema/table/SetTablePropertiesProcedure.java | 4 +- .../AbstractOperateSubscriptionProcedure.java | 4 +- .../impl/sync/AbstractOperatePipeProcedure.java | 4 +- .../AddNeverFinishSubProcedureProcedure.java | 4 +- .../impl/testonly/NeverFinishProcedure.java | 4 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 12 + .../confignode/procedure/entity/IncProcedure.java | 5 +- .../confignode/procedure/entity/NoopProcedure.java | 5 +- .../procedure/entity/SimpleLockProcedure.java | 5 +- .../procedure/entity/SimpleSTMProcedure.java | 4 +- .../procedure/entity/SleepProcedure.java | 5 +- .../procedure/entity/StuckSTMProcedure.java | 4 +- .../apache/iotdb/consensus/ConsensusFactory.java | 4 +- .../apache/iotdb/consensus/pipe/PipeConsensus.java | 7 +- .../iotdb/consensus/ratis/RatisConsensus.java | 2 + .../iotdb/consensus/simple/SimpleConsensus.java | 3 +- .../metadata/DatabaseNotSetException.java | 13 - .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 11 +- .../agent/task/connection/PipeEventCollector.java | 9 +- .../subtask/connector/PipeConnectorSubtask.java | 71 +-- .../evolvable/batch/PipeTabletEventBatch.java | 34 +- .../batch/PipeTransferBatchReqBuilder.java | 7 +- .../connector/protocol/opcua/OpcUaConnector.java | 6 +- .../pipeconsensus/PipeConsensusAsyncConnector.java | 350 ++++++++---- .../pipeconsensus/PipeConsensusSyncConnector.java | 34 +- ...r.java => PipeConsensusDeleteEventHandler.java} | 64 +-- .../PipeConsensusTabletBatchEventHandler.java | 2 +- .../PipeConsensusTabletInsertionEventHandler.java | 21 +- .../PipeConsensusTsFileInsertionEventHandler.java | 10 + .../PipeConsensusTransferBatchReqBuilder.java | 13 +- .../protocol/writeback/WriteBackConnector.java | 3 +- .../statement/PipeStatementInsertionEvent.java | 4 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 4 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 4 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 9 +- .../db/pipe/event/realtime/PipeRealtimeEvent.java | 5 + ...oricalDataRegionTsFileAndDeletionExtractor.java | 13 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 2 + .../PipeRealtimeDataRegionHybridExtractor.java | 204 ++++--- .../realtime/assigner/PipeDataRegionAssigner.java | 34 +- ...ava => PipeTsFileEpochProgressIndexKeeper.java} | 53 +- .../dataregion/realtime/epoch/TsFileEpoch.java | 14 +- .../db/pipe/extractor/mqtt/MQTTPublishHandler.java | 11 +- .../PipeDataNodeRemainingEventAndTimeMetrics.java | 50 +- .../PipeDataNodeRemainingEventAndTimeOperator.java | 41 +- .../pipeconsensus/PipeConsensusReceiver.java | 140 +++-- .../resource/memory/InsertNodeMemoryEstimator.java | 4 + .../resource/memory/PipeDynamicMemoryBlock.java | 156 ++++++ .../pipe/resource/memory/PipeMemoryBlockType.java | 2 + .../db/pipe/resource/memory/PipeMemoryManager.java | 54 ++ .../pipe/resource/memory/PipeMemoryWeightUtil.java | 18 +- .../resource/memory/PipeModelFixedMemoryBlock.java | 125 +++++ .../strategy/DynamicMemoryAllocationStrategy.java} | 26 +- .../strategy/ThresholdAllocationStrategy.java | 134 +++++ .../iotdb/db/protocol/client/ConfigNodeClient.java | 17 + .../db/protocol/mqtt/JSONPayloadFormatter.java | 9 +- .../db/protocol/mqtt/LinePayloadFormatter.java | 14 +- .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 11 +- .../iotdb/db/protocol/mqtt/PayloadFormatter.java | 12 + .../protocol/thrift/impl/ClientRPCServiceImpl.java | 29 +- .../impl/DataNodeInternalRPCServiceImpl.java | 21 +- .../execution/executor/RegionWriteExecutor.java | 2 + .../execution/fragment/FragmentInstanceInfo.java | 4 + .../execution/fragment/FragmentInstanceState.java | 2 +- .../execution/operator/OperatorContext.java | 6 +- .../relational/AbstractAggTableScanOperator.java | 4 + .../InformationSchemaContentSupplierFactory.java | 306 ++++++++-- .../AbstractApproxMostFrequentAccumulator.java | 57 ++ .../relational/aggregation/AccumulatorFactory.java | 62 +++ .../ApproxCountDistinctAccumulator.java | 5 +- .../BinaryApproxMostFrequentAccumulator.java | 130 +++++ .../BlobApproxMostFrequentAccumulator.java | 39 ++ .../BooleanApproxMostFrequentAccumulator.java | 129 +++++ .../DoubleApproxMostFrequentAccumulator.java | 129 +++++ .../FloatApproxMostFrequentAccumulator.java | 129 +++++ .../IntApproxMostFrequentAccumulator.java | 129 +++++ .../LongApproxMostFrequentAccumulator.java | 128 +++++ .../ApproxMostFrequentBucketDeserializer.java | 21 + .../ApproxMostFrequentBucketSerializer.java | 21 + .../aggregation/approximate/Counter.java | 71 +++ .../aggregation/approximate/DoublyLinkedList.java | 173 ++++++ .../approximate/ExternalizableUtil.java | 31 ++ .../aggregation/{ => approximate}/HyperLogLog.java | 2 +- .../{ => approximate}/HyperLogLogStateFactory.java | 2 +- .../relational/aggregation/approximate/ITopK.java | 43 ++ .../aggregation/approximate/ListNode2.java | 42 ++ .../relational/aggregation/approximate/Pair.java | 56 ++ .../aggregation/approximate/SpaceSaving.java | 147 +++++ .../approximate/SpaceSavingByteCalculator.java | 21 + .../approximate/SpaceSavingStateFactory.java | 90 +++ .../aggregation/approximate/StreamSummary.java | 294 ++++++++++ ...stractGroupedApproxMostFrequentAccumulator.java | 67 +++ ...BinaryGroupedApproxMostFrequentAccumulator.java | 105 ++++ .../BlobGroupedApproxMostFrequentAccumulator.java | 40 ++ ...ooleanGroupedApproxMostFrequentAccumulator.java | 104 ++++ ...DoubleGroupedApproxMostFrequentAccumulator.java | 104 ++++ .../FloatGroupedApproxMostFrequentAccumulator.java | 104 ++++ .../GroupedApproxCountDistinctAccumulator.java | 6 +- .../IntGroupedApproxMostFrequentAccumulator.java | 104 ++++ .../LongGroupedApproxMostFrequentAccumulator.java | 104 ++++ .../grouped/array/HyperLogLogBigArray.java | 2 +- .../grouped/array/SpaceSavingBigArray.java | 93 ++++ .../iotdb/db/queryengine/plan/Coordinator.java | 4 +- .../analyze/cache/partition/PartitionCache.java | 6 +- .../plan/analyze/load/LoadTsFileAnalyzer.java | 12 +- .../execution/config/TableConfigTaskVisitor.java | 50 +- .../config/executor/ClusterConfigTaskExecutor.java | 22 +- .../config/executor/IConfigTaskExecutor.java | 2 +- .../config/metadata/relational/DropDBTask.java | 7 +- .../db/queryengine/plan/parser/ASTVisitor.java | 1 + .../plan/planner/LogicalPlanVisitor.java | 55 +- .../plan/planner/TableOperatorGenerator.java | 44 +- .../plan/AbstractFragmentParallelPlanner.java | 5 +- .../function/tvf/ForecastTableFunction.java | 2 +- .../relational/metadata/TableMetadataImpl.java | 11 +- .../fetcher/cache/TreeDeviceNormalSchema.java | 2 +- .../plan/relational/planner/RelationPlanner.java | 38 +- .../distribute/TableDistributedPlanGenerator.java | 121 +++- .../iterative/rule/PruneTableScanColumns.java | 4 +- .../rule/PushDownOffsetIntoTableScan.java | 69 +++ .../planner/node/AggregationTableScanNode.java | 26 +- .../node/AggregationTreeDeviceViewScanNode.java | 6 +- .../planner/node/DeviceTableScanNode.java | 32 +- .../node/TreeAlignedDeviceViewScanNode.java | 6 +- .../planner/node/TreeDeviceViewScanNode.java | 10 +- .../node/TreeNonAlignedDeviceViewScanNode.java | 6 +- .../DataNodeLocationSupplierFactory.java | 3 + .../optimizations/DistributedOptimizeFactory.java | 5 +- .../optimizations/PushPredicateIntoTableScan.java | 2 +- .../optimizations/UnaliasSymbolReferences.java | 4 +- .../relational/planner/optimizations/Util.java | 4 +- .../relational/security/AccessControlImpl.java | 2 - .../plan/relational/sql/ast/DropDB.java | 6 - .../plan/relational/sql/parser/AstBuilder.java | 28 +- .../plan/relational/sql/rewrite/ShowRewrite.java | 26 +- .../sql/rewrite/StatementRewriteFactory.java | 8 +- .../scheduler/FixedRateFragInsStateTracker.java | 65 ++- .../scheduler/load/LoadTsFileDispatcherImpl.java | 44 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 39 +- .../java/org/apache/iotdb/db/service/DataNode.java | 17 +- .../iotdb/db/storageengine/StorageEngine.java | 5 +- .../dataregion/tsfile/TsFileResource.java | 10 +- .../dataregion/wal/utils/WALInsertNodeCache.java | 96 ++-- .../db/storageengine/load/LoadTsFileManager.java | 24 +- .../load/config/LoadTsFileConfigurator.java | 18 +- .../agent/SubscriptionBrokerAgent.java | 11 + .../db/subscription/broker/SubscriptionBroker.java | 19 + .../batch/SubscriptionPipeTsFileEventBatch.java | 11 + .../receiver/SubscriptionReceiverV1.java | 6 +- .../apache/iotdb/db/utils/ErrorHandlingUtils.java | 4 +- .../iotdb/db/utils/constant/SqlConstant.java | 1 + .../iotdb/db/utils/sort/SortBufferManager.java | 4 +- .../db/protocol/mqtt/JSONPayloadFormatterTest.java | 12 +- .../db/protocol/mqtt/LinePayloadFormatterTest.java | 9 +- ...nAlignedTreeDeviceViewScanOperatorTreeTest.java | 29 +- .../plan/relational/analyzer/AnalyzerTest.java | 14 +- .../plan/relational/analyzer/JoinTest.java | 140 +++++ .../analyzer/LimitOffsetPushDownTest.java | 13 + .../plan/relational/analyzer/SortTest.java | 8 +- .../plan/relational/analyzer/TestPlanBuilder.java | 4 +- .../plan/relational/planner/PlanTester.java | 9 +- .../planner/assertions/PlanMatchPattern.java | 4 + .../iotdb/commons/client/ainode/AINodeClient.java | 4 +- ...ner.java => IoTV2GlobalComponentContainer.java} | 45 +- .../iotdb/commons/concurrent/ThreadName.java | 1 + .../apache/iotdb/commons/conf/CommonConfig.java | 179 +++++- ...sensusRetryWithIncreasingIntervalException.java | 9 +- .../task/subtask/PipeAbstractConnectorSubtask.java | 129 +++-- .../agent/task/subtask/PipeReportableSubtask.java | 30 +- .../iotdb/commons/pipe/config/PipeConfig.java | 62 +++ .../iotdb/commons/pipe/config/PipeDescriptor.java | 58 ++ .../config/constant/PipeConnectorConstant.java | 14 +- .../pipe/connector/protocol/IoTDBConnector.java | 59 +- .../iotdb/commons/pipe/event/EnrichedEvent.java | 10 + .../pipe/receiver/PipeReceiverStatusHandler.java | 55 +- .../commons/pipe/resource/log/PipeLogManager.java | 14 + .../schema/column/ColumnHeaderConstant.java | 17 +- .../commons/schema/table/InformationSchema.java | 61 ++ .../TableBuiltinAggregationFunction.java | 2 +- .../org/apache/iotdb/commons/utils/RetryUtils.java | 12 + .../thrift-commons/src/main/thrift/common.thrift | 3 +- .../src/main/thrift/confignode.thrift | 37 +- .../src/main/thrift/datanode.thrift | 2 +- scripts/sbin/stop-ainode.sh | 11 +- 273 files changed, 9948 insertions(+), 2531 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 87db5cd25c1,e928fcc4d03..d9606010913 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@@ -523,9 -521,13 +529,13 @@@ public class LoadTsFileManager writer.endFile(); final DataRegion dataRegion = entry.getKey().getDataRegion(); -- final TsFileResource tsFileResource = dataPartition2Resource.get(entry.getKey()); - endTsFileResource(writer, tsFileResource, progressIndex); ++ final TsFileResource tsFileResource = dataPartition2Resource.get(entry.getKey()) + endTsFileResource( + writer, + tsFileResource, + timePartitionProgressIndexMap.getOrDefault( + entry.getKey().getTimePartitionSlot(), MinimumProgressIndex.INSTANCE)); - dataRegion.loadNewTsFile(tsFileResource, true, isGeneratedByPipe); + dataRegion.loadNewTsFile(tsFileResource, true, isGeneratedByPipe, false); // Metrics dataRegion
