This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch RpcCompress in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1033960a3279185e475c83bc3543adbca322fbaa Merge: 075459acc72 1d2f36ec00c Author: Tian Jiang <[email protected]> AuthorDate: Tue Jul 22 16:00:05 2025 +0800 Merge branch 'master' into RpcCompress .github/workflows/multi-language-client.yml | 10 +- RELEASE_NOTES.md | 22 + code-coverage/pom.xml | 2 +- dependencies.json | 14 +- distribution/pom.xml | 12 +- docker/ReadMe.md | 28 +- docker/src/main/DockerCompose/do-docker-build.sh | 12 +- .../{entrypoint.sh => docker-compose-ainode.yml} | 60 +- .../docker-compose-cluster-1c1d1a.yml | 68 +++ docker/src/main/DockerCompose/entrypoint.sh | 41 +- .../main/DockerCompose/replace-conf-from-env.sh | 19 +- docker/src/main/Dockerfile-1.0.0-ainode | 66 +++ example/client-cpp-example/pom.xml | 2 +- example/jdbc/pom.xml | 2 +- example/mqtt-customize/pom.xml | 2 +- example/mqtt/pom.xml | 2 +- example/pipe-count-point-processor/pom.xml | 2 +- example/pipe-opc-ua-sink/pom.xml | 2 +- example/pom.xml | 2 +- example/rest-java-example/pom.xml | 2 +- .../java/org/apache/iotdb/TableHttpExample.java | 2 +- .../java/org/apache/iotdb/TableHttpsExample.java | 2 +- example/schema/pom.xml | 2 +- example/session/pom.xml | 2 +- example/trigger/pom.xml | 2 +- example/udf/pom.xml | 2 +- integration-test/pom.xml | 32 +- .../it/env/cluster/config/MppCommonConfig.java | 23 + .../env/cluster/config/MppSharedCommonConfig.java | 26 + .../iotdb/it/env/cluster/node/AINodeWrapper.java | 2 +- .../it/env/remote/config/RemoteCommonConfig.java | 16 + .../org/apache/iotdb/itbase/env/CommonConfig.java | 8 + .../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 268 --------- .../iotdb/ainode/it/AINodeClusterConfigIT.java | 113 ++++ .../iotdb/ainode/it/AINodeInferenceSQLIT.java | 292 ++++++++++ .../iotdb/ainode/it/AINodeModelManageIT.java | 215 +++++++ .../apache/iotdb/ainode/utils/AINodeTestUtils.java | 126 ++++ .../it/IoTDBConfigNodeSnapshot2IT.java} | 38 +- .../confignode/it/IoTDBConfigNodeSnapshotIT.java | 5 +- .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 3 + .../org/apache/iotdb/db/it/IoTDBNestedQueryIT.java | 4 +- .../org/apache/iotdb/db/it/IoTDBRestServiceIT.java | 145 ++++- .../apache/iotdb/db/it/IoTDBRestartRatisIT.java | 372 ++++++++++++ .../maxby/IoTDBMaxByAlignedSeriesIT.java | 20 + .../db/it/aggregation/maxby/IoTDBMaxByIT.java | 21 +- .../db/it/aggregation/minby/IoTDBMinByIT.java | 5 +- .../db/it/last/IoTDBLastQueryLastCacheIT.java | 15 +- .../IoTDBArithmeticDivAndModWithoutDoubleIT.java | 70 +++ .../org/apache/iotdb/db/it/utils/TestUtils.java | 33 +- .../withoutNull/IoTDBWithoutNullAllFilterIT.java | 48 +- .../withoutNull/IoTDBWithoutNullAnyFilterIT.java | 48 +- .../manual/AbstractPipeTableModelDualManualIT.java | 6 +- .../manual/basic/IoTDBPipeDataSinkIT.java | 14 +- .../manual/enhanced/IoTDBPipeTypeConversionIT.java | 3 + .../treemodel/auto/basic/IoTDBPipeExtractorIT.java | 6 +- .../treemodel/auto/basic/IoTDBPipeProcessorIT.java | 6 +- .../treemodel/auto/basic/IoTDBPipeProtocolIT.java | 6 +- .../auto/enhanced/IoTDBPipeAutoConflictIT.java | 6 +- .../auto/enhanced/IoTDBPipeClusterIT.java | 6 +- .../enhanced/IoTDBPipeConnectorCompressionIT.java | 7 +- .../auto/enhanced/IoTDBPipeIdempotentIT.java | 6 +- .../auto/enhanced/IoTDBPipeWithLoadIT.java | 6 +- .../manual/AbstractPipeDualTreeModelManualIT.java | 6 +- .../manual/IoTDBPipeTypeConversionISessionIT.java | 12 +- .../manual/IoTDBPipeTypeConversionIT.java | 5 +- .../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 1 + .../db/it/IoTDBMultiTAGsWithAttributesTableIT.java | 28 +- .../it/db/it/IoTDBPatternAggregationIT.java | 492 ++++++++++++++++ .../IoTDBQueryAttributeTableIT.java} | 49 +- ...ableAggregationQueryWithNetworkPartitionIT.java | 186 ++++++ .../query/recent/extract/IoTDBExtractTable2IT.java | 72 +++ .../query/recent/extract/IoTDBExtractTable3IT.java | 72 +++ .../query/recent/extract/IoTDBExtractTableIT.java | 317 ++++++++++ .../view/recent/IoTDBNullValueTableViewIT.java | 14 +- .../query/view/recent/IoTDBTableViewQueryIT.java | 17 + .../IoTDBTableViewQueryWithCachedDeviceIT.java | 127 ++++ .../relational/it/rest/it/IoTDBRestServiceIT.java | 32 +- .../relational/it/schema/IoTDBDatabaseIT.java | 4 +- .../iotdb/session/it/IoTDBSessionSimpleIT.java | 16 + .../it/cluster/IoTDBSubscriptionRestartIT.java | 3 +- .../it/dual/AbstractSubscriptionDualIT.java | 3 + .../IoTDBSubscriptionConsumerGroupIT.java | 3 + .../IoTDBSubscriptionTimePrecisionIT.java | 2 + .../dual/treemodel/IoTDBSubscriptionTopicIT.java | 3 + .../it/local/AbstractSubscriptionLocalIT.java | 7 +- .../it/triple/AbstractSubscriptionTripleIT.java | 10 + .../AbstractSubscriptionTreeRegressionIT.java | 71 ++- .../IoTDBRootPullConsumeTsfileIT.java | 15 +- .../format/IoTDBDBTsfilePullConsumerIT.java | 18 +- .../IoTDBAllTsTsfilePullConsumerIT.java | 16 +- .../IoTDBAllTsfilePullConsumerSnapshotIT.java | 14 +- .../IoTDBPathDeviceTsfilePullConsumerIT.java | 17 +- .../IoTDBTimeTsTsfilePullConsumerIT.java | 17 +- .../IoTDBSnapshotDevicePullConsumerTsfileIT.java | 16 +- .../multi/IoTDBOneConsumerMultiTopicsMixIT.java | 33 +- .../multi/IoTDBOneConsumerMultiTopicsTsfileIT.java | 39 +- .../pattern/IoTDBDBPatternPullConsumeTsfileIT.java | 15 +- .../IoTDBDevicePatternPullConsumeTsfileIT.java | 14 +- .../IoTDBRootPatternPullConsumeTsfileIT.java | 12 +- .../pattern/IoTDBTSPatternPullConsumeTsfileIT.java | 15 +- .../multi/IoTDBOneConsumerMultiTopicsMixIT.java | 8 + .../multi/IoTDBOneConsumerMultiTopicsTsfileIT.java | 11 + .../topic/IoTDBDataSet1TopicConsumerSpecialIT.java | 4 +- .../apache/iotdb/tools/it/ExportTsFileTestIT.java | 6 +- .../src/test/resources/ainode-example/config.yaml | 8 +- iotdb-api/external-api/pom.xml | 2 +- iotdb-api/pipe-api/pom.xml | 2 +- .../event/dml/insertion/TsFileInsertionEvent.java | 10 + iotdb-api/pom.xml | 2 +- iotdb-api/trigger-api/pom.xml | 2 +- iotdb-api/udf-api/pom.xml | 2 +- .../processor/TableFunctionDataProcessor.java | 5 + .../processor/TableFunctionLeafProcessor.java | 5 + iotdb-client/cli/pom.xml | 22 +- .../apache/iotdb/tool/data/ExportDataTable.java | 5 +- iotdb-client/client-cpp/pom.xml | 4 +- iotdb-client/client-py/pom.xml | 8 +- iotdb-client/isession/pom.xml | 8 +- iotdb-client/jdbc/pom.xml | 8 +- iotdb-client/pom.xml | 2 +- iotdb-client/service-rpc/pom.xml | 6 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 2 + iotdb-client/session/pom.xml | 10 +- .../java/org/apache/iotdb/session/Session.java | 17 +- .../apache/iotdb/session/util/SessionUtils.java | 103 ++-- .../iotdb/session/util/SessionUtilsTest.java | 26 +- iotdb-core/ainode/ainode/core/ainode.py | 165 ++++++ iotdb-core/ainode/ainode/core/config.py | 84 +-- iotdb-core/ainode/ainode/core/constant.py | 52 +- iotdb-core/ainode/ainode/core/exception.py | 5 + .../core/{util/status.py => inference/__init__.py} | 16 - .../ainode/core/inference/inference_request.py | 120 ++++ .../core/inference/inference_request_pool.py | 152 +++++ .../status.py => inference/strategy/__init__.py} | 16 - .../strategy/abstract_inference_pipeline.py | 60 ++ .../strategy/timer_sundial_inference_pipeline.py | 51 ++ iotdb-core/ainode/ainode/core/inference/utils.py | 80 +++ iotdb-core/ainode/ainode/core/ingress/dataset.py | 2 + iotdb-core/ainode/ainode/core/ingress/iotdb.py | 39 +- .../ainode/core/manager/inference_manager.py | 145 ++++- .../ainode/ainode/core/manager/model_manager.py | 23 +- .../ainode/core/model/built_in_model_factory.py | 14 +- .../ainode/ainode/core/model/model_storage.py | 165 +++++- .../core/model/sundial/configuration_sundial.py | 2 - .../core/model/sundial/ts_generation_mixin.py | 5 + .../ainode/core/model/timerxl/modeling_timer.py | 3 - .../core/{util/status.py => rpc/__init__.py} | 16 - iotdb-core/ainode/ainode/core/{ => rpc}/client.py | 32 +- iotdb-core/ainode/ainode/core/{ => rpc}/handler.py | 17 +- iotdb-core/ainode/ainode/core/rpc/service.py | 101 ++++ .../ainode/ainode/core/{util => rpc}/status.py | 0 iotdb-core/ainode/ainode/core/script.py | 120 +--- iotdb-core/ainode/ainode/core/service.py | 57 -- iotdb-core/ainode/poetry.lock | 528 +++++++++++------ iotdb-core/ainode/pom.xml | 10 +- iotdb-core/ainode/pyproject.toml | 9 +- .../ainode/resources/conf/iotdb-ainode.properties | 22 +- iotdb-core/antlr/pom.xml | 2 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 9 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 + iotdb-core/confignode/pom.xml | 28 +- .../rpc/DataNodeAsyncRequestRPCHandler.java | 2 +- .../handlers/rpc/PipeHeartbeatRPCHandler.java | 2 +- .../request/write/ainode/RemoveAINodePlan.java | 5 + .../iotdb/confignode/manager/ConfigManager.java | 23 +- .../apache/iotdb/confignode/manager/IManager.java | 7 +- .../iotdb/confignode/manager/ModelManager.java | 9 +- .../manager/load/service/HeartbeatService.java | 4 +- .../iotdb/confignode/manager/node/NodeManager.java | 20 +- .../agent/task/PipeConfigNodeSubtaskExecutor.java | 4 +- .../pipe/agent/task/PipeConfigNodeTaskAgent.java | 30 +- .../protocol/IoTDBConfigRegionAirGapConnector.java | 5 + .../protocol/IoTDBConfigRegionConnector.java | 13 +- .../runtime/heartbeat/PipeHeartbeatParser.java | 7 +- .../runtime/heartbeat/PipeHeartbeatScheduler.java | 2 +- .../receiver/protocol/IoTDBConfigNodeReceiver.java | 7 +- .../iotdb/confignode/persistence/TriggerInfo.java | 3 - .../iotdb/confignode/persistence/UDFInfo.java | 3 - .../confignode/persistence/pipe/PipeTaskInfo.java | 7 +- .../procedure/impl/model/DropModelProcedure.java | 57 +- .../procedure/impl/node/RemoveAINodeProcedure.java | 24 + .../impl/pipe/AbstractOperatePipeProcedureV2.java | 18 +- .../impl/pipe/runtime/PipeMetaSyncProcedure.java | 4 +- .../impl/pipe/task/AlterPipeProcedureV2.java | 16 +- .../impl/pipe/task/CreatePipeProcedureV2.java | 18 +- .../procedure/state/RemoveAINodeState.java | 1 + .../confignode/service/ConfigNodeShutdownHook.java | 3 - .../thrift/ConfigNodeRPCServiceProcessor.java | 15 +- .../request/ConfigPhysicalPlanSerDeTest.java | 12 +- .../consensus/response/pipe/PipeTableRespTest.java | 6 +- .../agent/PipeConfigNodeSubtaskExecutorTest.java | 3 +- .../connector/PipeConfigNodeThriftRequestTest.java | 4 - .../iotdb/confignode/persistence/PipeInfoTest.java | 4 +- iotdb-core/consensus/pom.xml | 28 +- iotdb-core/datanode/pom.xml | 42 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 20 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 +- .../iotdb/db/exception/load/LoadFileException.java | 4 + .../db/pipe/agent/runtime/PipeAgentLauncher.java | 2 - .../agent/runtime/PipeDataNodeRuntimeAgent.java | 10 - .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 637 ++++++++++----------- .../task/builder/PipeDataNodeTaskBuilder.java | 91 +-- .../execution/PipeConnectorSubtaskExecutor.java | 8 +- .../execution/PipeProcessorSubtaskExecutor.java | 2 +- .../task/execution/PipeSubtaskExecutorManager.java | 18 +- .../agent/task/stage/PipeTaskConnectorStage.java | 6 +- .../subtask/connector/PipeConnectorSubtask.java | 5 + .../connector/PipeConnectorSubtaskLifeCycle.java | 5 + .../connector/PipeConnectorSubtaskManager.java | 19 +- .../PipeRealtimePriorityBlockingQueue.java | 151 ++++- .../subtask/processor/PipeProcessorSubtask.java | 4 +- .../client/IoTDBDataNodeAsyncClientManager.java | 105 +++- .../evolvable/batch/PipeTabletEventBatch.java | 39 +- .../evolvable/batch/PipeTabletEventPlainBatch.java | 10 +- .../batch/PipeTabletEventTsFileBatch.java | 7 + .../evolvable/request/PipeTransferPlanNodeReq.java | 1 - .../request/PipeTransferTabletBatchReq.java | 1 - .../request/PipeTransferTabletBinaryReq.java | 1 - .../request/PipeTransferTabletInsertNodeReq.java | 1 - .../request/PipeTransferTabletRawReq.java | 1 - .../airgap/IoTDBDataRegionAirGapConnector.java | 34 +- .../airgap/IoTDBSchemaRegionAirGapConnector.java | 5 + .../pipeconsensus/PipeConsensusAsyncConnector.java | 3 +- .../pipeconsensus/PipeConsensusSyncConnector.java | 2 +- .../PipeConsensusTransferBatchReqBuilder.java | 3 +- .../async/IoTDBDataRegionAsyncConnector.java | 111 ++-- .../handler/PipeTransferTrackableHandler.java | 2 + .../async/handler/PipeTransferTsFileHandler.java | 73 ++- .../thrift/sync/IoTDBDataRegionSyncConnector.java | 29 +- .../thrift/sync/IoTDBSchemaRegionConnector.java | 5 + .../protocol/writeback/WriteBackConnector.java | 3 +- .../connector/util/builder/PipeTsFileBuilder.java | 40 +- .../consensus/PipeConsensusSubtaskExecutor.java | 6 +- .../event/common/heartbeat/PipeHeartbeatEvent.java | 6 +- .../statement/PipeStatementInsertionEvent.java | 6 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 113 ++-- .../common/tablet/PipeRawTabletInsertionEvent.java | 6 +- .../event/common/terminate/PipeTerminateEvent.java | 44 -- .../tsfile/PipeCompactedTsFileInsertionEvent.java | 264 +++++++++ .../common/tsfile/PipeTsFileInsertionEvent.java | 104 ++-- .../parser/TsFileInsertionEventParserProvider.java | 4 +- .../event/realtime/PipeRealtimeEventFactory.java | 12 +- .../dataregion/DataRegionWatermarkInjector.java | 4 - .../dataregion/IoTDBDataRegionExtractor.java | 121 ++-- ...oricalDataRegionTsFileAndDeletionExtractor.java | 152 +++-- .../realtime/PipeRealtimeDataRegionExtractor.java | 35 ++ .../PipeRealtimeDataRegionHybridExtractor.java | 181 ++---- .../PipeRealtimeDataRegionLogExtractor.java | 5 +- .../PipeRealtimeDataRegionTsFileExtractor.java | 7 + .../realtime/assigner/PipeDataRegionAssigner.java | 59 +- .../PipeTsFileEpochProgressIndexKeeper.java | 30 +- .../dataregion/realtime/epoch/TsFileEpoch.java | 17 +- .../realtime/epoch/TsFileEpochManager.java | 4 +- .../listener/PipeInsertionDataNodeListener.java | 8 +- .../schemaregion/IoTDBSchemaRegionExtractor.java | 4 +- .../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 9 +- .../PipeDataNodeRemainingEventAndTimeOperator.java | 36 +- ...ics.java => PipeDataNodeSinglePipeMetrics.java} | 99 ++-- .../pipe/metric/overview/PipeResourceMetrics.java | 52 +- .../overview/PipeWALInsertNodeCacheMetrics.java | 86 --- .../sink/PipeDataRegionConnectorMetrics.java | 34 ++ .../twostage/exchange/payload/CombineRequest.java | 1 - .../payload/FetchCombineResultRequest.java | 1 - .../pipeconsensus/PipeConsensusReceiver.java | 75 ++- .../protocol/thrift/IoTDBDataNodeReceiver.java | 21 + .../pipe/resource/PipeDataNodeResourceManager.java | 25 +- .../db/pipe/resource/memory/PipeMemoryManager.java | 33 +- .../pipe/resource/memory/PipeMemoryWeightUtil.java | 8 +- .../resource/tsfile/PipeCompactionManager.java | 68 +++ ...Resource.java => PipeTsFilePublicResource.java} | 158 +---- .../pipe/resource/tsfile/PipeTsFileResource.java | 253 +------- .../resource/tsfile/PipeTsFileResourceManager.java | 331 ++++++----- .../db/pipe/resource/wal/PipeWALResource.java | 169 ------ .../pipe/resource/wal/PipeWALResourceManager.java | 142 ----- .../wal/hardlink/PipeWALHardlinkResource.java | 47 -- .../hardlink/PipeWALHardlinkResourceManager.java | 152 ----- .../selfhost/PipeWALSelfHostResourceManager.java | 38 -- .../iotdb/db/protocol/client/ConfigNodeClient.java | 11 +- .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 4 + .../rest/table/v1/handler/QueryDataSetHandler.java | 36 +- .../table/v1/handler/RequestValidationHandler.java | 12 +- .../v1/handler/StatementConstructionHandler.java | 4 +- .../rest/v1/handler/QueryDataSetHandler.java | 54 +- .../rest/v2/handler/QueryDataSetHandler.java | 37 +- .../protocol/rest/v2/impl/RestApiServiceImpl.java | 13 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 8 +- .../impl/DataNodeInternalRPCServiceImpl.java | 95 ++- .../db/queryengine/common/MPPQueryContext.java | 33 ++ .../execution/MemoryEstimationHelper.java | 28 + .../execution/exchange/SharedTsBlockQueue.java | 2 +- .../queryengine/execution/exchange/sink/ISink.java | 4 +- .../execution/exchange/sink/LocalSinkChannel.java | 14 +- .../execution/exchange/sink/ShuffleSinkHandle.java | 40 +- .../execution/exchange/sink/SinkChannel.java | 10 +- .../execution/fragment/DataNodeQueryContext.java | 21 +- .../fragment/FragmentInstanceContext.java | 6 +- .../fragment/FragmentInstanceExecution.java | 44 +- .../fragment/FragmentInstanceManager.java | 10 + .../process/PatternRecognitionOperator.java | 8 + .../operator/process/ProjectOperator.java | 3 +- .../function/TableFunctionLeafOperator.java | 18 +- .../process/function/TableFunctionOperator.java | 4 + .../join/SimpleNestedLoopCrossJoinOperator.java | 3 +- .../last/AbstractUpdateLastCacheOperator.java | 24 +- .../last/AlignedUpdateLastCacheOperator.java | 14 +- .../AlignedUpdateViewPathLastCacheOperator.java | 10 +- .../operator/process/last/LastQueryOperator.java | 1 + .../process/last/LastQuerySortOperator.java | 10 +- .../process/last/LastQueryTransformOperator.java | 4 +- .../operator/process/last/LastQueryUtil.java | 34 ++ .../process/last/UpdateLastCacheOperator.java | 27 +- .../last/UpdateViewPathLastCacheOperator.java | 4 +- .../rowpattern/PatternAggregationTracker.java | 156 +++++ .../process/rowpattern/PatternAggregator.java | 178 ++++++ .../process/rowpattern/PatternAggregators.java | 74 +++ .../rowpattern/PatternPartitionExecutor.java | 13 + .../rowpattern/PatternVariableRecognizer.java | 8 +- .../rowpattern/PhysicalAggregationPointer.java} | 15 +- .../process/rowpattern/expression/Computation.java | 15 +- .../expression/PatternExpressionComputation.java | 18 +- .../process/rowpattern/matcher/Matcher.java | 29 +- .../AbstractSeriesAggregationScanOperator.java | 70 ++- .../source/AbstractSeriesScanOperator.java | 46 +- .../execution/operator/source/SeriesScanUtil.java | 73 ++- .../relational/AbstractAggTableScanOperator.java | 75 ++- .../AbstractDefaultAggTableScanOperator.java | 4 +- .../relational/AbstractMergeSortJoinOperator.java | 14 + .../relational/AbstractTableScanOperator.java | 16 +- .../relational/DeviceIteratorScanOperator.java | 3 +- .../InformationSchemaContentSupplierFactory.java | 4 +- .../relational/LastQueryAggTableScanOperator.java | 4 +- .../relational/aggregation/FirstAccumulator.java | 4 +- .../relational/aggregation/FirstByAccumulator.java | 6 - .../relational/aggregation/LastAccumulator.java | 4 +- .../relational/aggregation/LastByAccumulator.java | 6 - .../relational/aggregation/MaxAccumulator.java | 4 +- .../relational/aggregation/MinAccumulator.java | 4 +- .../relational/ColumnTransformerBuilder.java | 23 + .../iotdb/db/queryengine/plan/ClusterTopology.java | 16 +- .../iotdb/db/queryengine/plan/Coordinator.java | 6 + .../db/queryengine/plan/analyze/Analysis.java | 33 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 92 ++- .../plan/analyze/ExpressionTypeAnalyzer.java | 29 + .../queryengine/plan/analyze/ExpressionUtils.java | 14 +- .../queryengine/plan/analyze/PredicateUtils.java | 13 +- .../plan/analyze/load/LoadTsFileAnalyzer.java | 26 +- .../queryengine/plan/execution/QueryExecution.java | 32 +- .../execution/config/TableConfigTaskVisitor.java | 36 ++ .../execution/config/TreeConfigTaskVisitor.java | 22 +- .../config/executor/ClusterConfigTaskExecutor.java | 54 +- .../config/executor/IConfigTaskExecutor.java | 11 +- .../CreateModelTask.java => RemoveAINodeTask.java} | 22 +- .../config/metadata/ai/CreateModelTask.java | 14 +- .../execution/config/sys/pipe/ShowPipeTask.java | 4 +- .../memory/StatementMemorySourceVisitor.java | 3 +- .../visitor/ColumnTransformerVisitor.java | 11 + .../db/queryengine/plan/parser/ASTVisitor.java | 32 +- .../plan/planner/LocalExecutionPlanContext.java | 13 +- .../plan/planner/LogicalPlanBuilder.java | 158 ++--- .../plan/planner/OperatorTreeGenerator.java | 179 +++--- .../plan/planner/SubPlanTypeExtractor.java | 6 - .../plan/planner/TableOperatorGenerator.java | 191 +++++- .../planner/distribution/ExchangeNodeAdder.java | 15 +- .../planner/distribution/NodeGroupContext.java | 12 + .../SimpleFragmentParallelPlanner.java | 12 +- .../plan/planner/distribution/SourceRewriter.java | 152 ++++- .../plan/planner/plan/TableModelTimePredicate.java | 6 +- .../plan/planner/plan/TimePredicate.java | 4 +- .../plan/planner/plan/TreeModelTimePredicate.java | 4 +- .../plan/planner/plan/node/PlanGraphPrinter.java | 18 +- .../plan/planner/plan/node/PlanNodeType.java | 14 +- .../plan/planner/plan/node/PlanVisitor.java | 5 - .../plan/node/process/MultiChildProcessNode.java | 4 + .../plan/node/process/last/LastQueryNode.java | 149 ++++- .../plan/node/source/AlignedLastQueryScanNode.java | 245 -------- .../plan/node/source/LastQueryScanNode.java | 215 +++++-- .../relational/analyzer/AggregationAnalyzer.java | 6 + .../relational/analyzer/ExpressionAnalyzer.java | 173 +++++- .../relational/analyzer/StatementAnalyzer.java | 17 + .../predicate/ConvertPredicateToFilterVisitor.java | 7 +- .../ConvertPredicateToTimeFilterVisitor.java | 80 ++- .../PredicateCombineIntoTableScanChecker.java | 23 +- .../ConvertSchemaPredicateToFilterVisitor.java | 8 +- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 35 +- .../planner/IrExpressionInterpreter.java | 19 + .../plan/relational/planner/IrTypeAnalyzer.java | 7 + .../plan/relational/planner/QueryPlanner.java | 2 +- .../plan/relational/planner/RelationPlanner.java | 57 +- .../distribute/TableDistributedPlanGenerator.java | 8 +- .../relational/planner/ir/ExpressionRewriter.java | 6 + .../planner/ir/ExpressionTreeRewriter.java | 20 + .../ir/GlobalTimePredicateExtractVisitor.java | 16 +- .../optimizations/PushPredicateIntoTableScan.java | 13 +- .../planner/optimizations/SymbolMapper.java | 39 +- .../planner/rowpattern/AggregationLabelSet.java | 100 ++++ .../rowpattern/AggregationValuePointer.java | 161 ++++++ .../rowpattern/ExpressionAndValuePointers.java | 16 + .../plan/relational/sql/ast/AstVisitor.java | 16 + .../sql/ast/CreateModel.java} | 51 +- .../plan/relational/sql/ast/CreateTraining.java | 27 +- .../sql/ast/DefaultTraversalVisitor.java | 5 + .../plan/relational/sql/ast/DropModel.java} | 48 +- .../plan/relational/sql/ast/Expression.java | 3 + .../plan/relational/sql/ast/Extract.java | 139 +++++ .../sql/ast/RemoveAINode.java} | 43 +- .../relational/sql/ast/TableExpressionType.java | 3 +- .../plan/relational/sql/parser/AstBuilder.java | 86 ++- .../relational/sql/util/ExpressionFormatter.java | 6 + .../scheduler/load/LoadTsFileDispatcherImpl.java | 1 + .../plan/scheduler/load/LoadTsFileScheduler.java | 2 +- .../queryengine/plan/statement/StatementType.java | 4 +- .../plan/statement/StatementVisitor.java | 5 + .../plan/statement/crud/QueryStatement.java | 10 +- ...elStatement.java => RemoveAINodeStatement.java} | 43 +- .../metadata/model/CreateModelStatement.java | 10 +- .../metadata/model/DropModelStatement.java | 10 +- .../metadata/model/ShowModelsStatement.java | 12 +- .../metadata/view/CreateLogicalViewStatement.java | 13 +- .../column/unary/scalar/ExtractTransformer.java | 106 ++++ .../dag/input/QueryDataSetInputLayer.java | 3 + .../apache/iotdb/db/schemaengine/SchemaEngine.java | 10 +- .../iotdb/db/service/DataNodeShutdownHook.java | 41 +- .../org/apache/iotdb/db/service/MQTTService.java | 2 +- .../metrics/IoTDBInternalLocalReporter.java | 2 +- .../iotdb/db/storageengine/StorageEngine.java | 7 - .../db/storageengine/dataregion/DataRegion.java | 4 +- .../task/InsertionCrossSpaceCompactionTask.java | 2 +- .../compaction/execute/utils/CompactionUtils.java | 7 +- .../utils/reader/SeriesDataBlockReader.java | 20 +- .../dataregion/flush/NotifyFlushMemTable.java | 7 - .../dataregion/memtable/AbstractMemTable.java | 13 - .../dataregion/memtable/IMemTable.java | 4 - .../dataregion/memtable/TsFileProcessor.java | 31 +- .../dataregion/snapshot/SnapshotLoader.java | 67 ++- .../dataregion/tsfile/TsFileManager.java | 13 + .../dataregion/tsfile/TsFileResource.java | 31 +- .../tsfile/generator/TsFileNameGenerator.java | 63 +- .../allocation/AbstractNodeAllocationStrategy.java | 13 +- .../dataregion/wal/buffer/WALBuffer.java | 6 - .../dataregion/wal/buffer/WALEntry.java | 4 +- .../wal/checkpoint/CheckpointManager.java | 70 +-- .../dataregion/wal/checkpoint/MemTableInfo.java | 19 +- .../dataregion/wal/io/WALByteBufReader.java | 13 - .../dataregion/wal/node/WALFakeNode.java | 4 +- .../storageengine/dataregion/wal/node/WALNode.java | 96 +--- .../dataregion/wal/utils/WALEntryHandler.java | 233 -------- .../dataregion/wal/utils/WALEntryPosition.java | 242 -------- .../dataregion/wal/utils/WALInsertNodeCache.java | 338 ----------- .../wal/utils/listener/WALFlushListener.java | 12 +- .../db/storageengine/load/LoadTsFileManager.java | 13 +- ...leStatementDataTypeConvertExecutionVisitor.java | 90 ++- ...eeStatementDataTypeConvertExecutionVisitor.java | 172 ++---- .../converter/LoadTsFileDataTypeConverter.java | 49 +- .../storageengine/load/disk/ILoadDiskSelector.java | 3 +- .../InheritSystemMultiDisksStrategySelector.java | 42 +- .../db/storageengine/load/disk/MinIOSelector.java | 3 +- .../load/limiter/LoadTsFileRateLimiter.java | 64 +-- .../load/splitter/TsFileSplitter.java | 3 + .../storageengine/rescon/disk/FolderManager.java | 63 ++ .../db/storageengine/rescon/disk/TierManager.java | 4 + .../rescon/disk/strategy/DirectoryStrategy.java | 34 ++ .../strategy/MaxDiskUsableSpaceFirstStrategy.java | 3 + .../MinFolderOccupiedSpaceFirstStrategy.java | 4 + .../strategy/RandomOnDiskUsableSpaceStrategy.java | 4 + .../rescon/disk/strategy/SequenceStrategy.java | 4 +- .../rescon/memory/AbstractPoolManager.java | 16 +- .../db/subscription/broker/SubscriptionBroker.java | 35 +- .../broker/SubscriptionPrefetchingQueue.java | 387 ++++++++++++- .../broker/SubscriptionPrefetchingTabletQueue.java | 7 +- .../broker/SubscriptionPrefetchingTsFileQueue.java | 6 - .../SubscriptionPipeEventBatchSegmentLock.java | 75 +++ .../event/batch/SubscriptionPipeEventBatches.java | 118 ++-- .../batch/SubscriptionPipeTsFileEventBatch.java | 40 +- .../receiver/SubscriptionReceiverV1.java | 2 + .../execution/SubscriptionSubtaskExecutor.java | 9 +- .../task/stage/SubscriptionTaskConnectorStage.java | 4 +- .../TsFileResourcePipeStatisticsSetTool.java | 302 ++++++++++ .../org/apache/iotdb/db/utils/DateTimeUtils.java | 14 +- .../apache/iotdb/db/utils/ErrorHandlingUtils.java | 4 +- .../iotdb/db/utils/sort/FileSpillerReader.java | 4 +- .../connector/PipeDataNodeThriftRequestTest.java | 12 - .../pipe/event/TsFileInsertionEventParserTest.java | 1 + .../db/pipe/extractor/PipeRealtimeExtractTest.java | 101 ++-- .../resource/PipeTsFileResourceManagerTest.java | 96 ++-- .../PipeWALHardlinkResourceManagerTest.java | 114 ---- .../queryengine/execution/exchange/StubSink.java | 6 +- .../execution/operator/LastQueryOperatorTest.java | 6 +- .../operator/LastQueryTreeSortOperatorTest.java | 6 +- ...nAlignedTreeDeviceViewScanOperatorTreeTest.java | 69 ++- .../process/TableStreamSortOperatorTest.java | 11 +- .../operator/process/rowpattern/MatcherTest.java | 4 +- .../db/queryengine/plan/analyze/AnalyzeTest.java | 4 +- .../plan/planner/distribution/LastQueryTest.java | 23 +- .../logical/DataQueryLogicalPlannerTest.java | 89 +-- .../node/source/LastQueryScanNodeSerdeTest.java | 72 +++ .../relational/analyzer/ExtractExpressionTest.java | 66 +++ .../plan/relational/analyzer/JoinTest.java | 136 +++++ .../TsFileResourceProgressIndexTest.java | 23 +- .../BatchedCompactionWithTsFileSplitterTest.java | 2 +- .../AlignedSeriesScanLimitOffsetPushDownTest.java | 97 ++-- .../AlignedSeriesScanPredicatePushDownTest.java | 47 +- ...gleColumnSeriesScanLimitOffsetPushDownTest.java | 48 +- .../read/reader/series/SeriesReaderTestUtil.java | 17 + .../series/SeriesScanLimitOffsetPushDownTest.java | 60 +- .../series/SeriesScanPredicatePushDownTest.java | 40 +- .../wal/checkpoint/CheckpointManagerTest.java | 172 ------ .../dataregion/wal/node/WALEntryHandlerTest.java | 586 ------------------- .../dataregion/wal/node/WALNodeTest.java | 17 - .../wal/node/WalDeleteOutdatedNewTest.java | 65 --- .../wal/utils/WALInsertNodeCacheTest.java | 190 ------ iotdb-core/metrics/core/pom.xml | 4 +- iotdb-core/metrics/interface/pom.xml | 8 +- iotdb-core/metrics/pom.xml | 2 +- iotdb-core/node-commons/pom.xml | 26 +- .../apache/iotdb/commons/client/ClientManager.java | 22 +- .../iotdb/commons/client/ainode/AINodeClient.java | 26 +- .../async/AsyncPipeDataTransferServiceClient.java | 2 +- .../commons/concurrent/IoTDBThreadPoolFactory.java | 15 + .../iotdb/commons/concurrent/ThreadName.java | 12 +- .../apache/iotdb/commons/conf/CommonConfig.java | 498 ++++++---------- .../commons/consensus/index/ProgressIndex.java | 2 +- .../consensus/index/impl/IoTProgressIndex.java | 27 +- .../iotdb/commons/partition/DataPartition.java | 11 +- .../org/apache/iotdb/commons/path/PartialPath.java | 4 + .../agent/runtime/PipePeriodicalJobExecutor.java | 33 -- .../commons/pipe/agent/task/PipeTaskAgent.java | 63 +- .../agent/task/execution/PipeSubtaskExecutor.java | 45 +- .../commons/pipe/agent/task/meta/PipeMeta.java | 6 +- .../pipe/agent/task/meta/PipeRuntimeMeta.java | 51 +- .../commons/pipe/agent/task/meta/PipeTaskMeta.java | 142 +---- .../task/progress/PipeEventCommitManager.java | 25 +- .../agent/task/progress/PipeEventCommitter.java | 12 +- .../task/subtask/PipeAbstractConnectorSubtask.java | 5 - .../agent/task/subtask/PipeReportableSubtask.java | 21 +- .../iotdb/commons/pipe/config/PipeConfig.java | 178 ++---- .../iotdb/commons/pipe/config/PipeDescriptor.java | 160 ++---- .../config/constant/PipeConnectorConstant.java | 7 +- .../connector/client/IoTDBSyncClientManager.java | 14 + .../connector/limiter/GlobalRPCRateLimiter.java} | 17 +- .../pipe/connector/limiter/GlobalRateLimiter.java | 14 +- .../connector/limiter/TsFileSendRateLimiter.java} | 32 +- .../thrift/request/PipeTransferFilePieceReq.java | 1 - .../thrift/request/PipeTransferFileSealReqV1.java | 1 - .../thrift/request/PipeTransferFileSealReqV2.java | 1 - .../thrift/request/PipeTransferHandshakeV1Req.java | 1 - .../thrift/request/PipeTransferHandshakeV2Req.java | 1 - .../thrift/request/PipeTransferSliceReq.java | 1 - .../connector/protocol/IoTDBAirGapConnector.java | 14 + .../pipe/connector/protocol/IoTDBConnector.java | 16 +- .../connector/protocol/IoTDBSslSyncConnector.java | 17 +- .../iotdb/commons/pipe/event/EnrichedEvent.java | 13 + .../commons/pipe/receiver/IoTDBFileReceiver.java | 66 ++- .../commons/pipe/resource/log/PipeLogManager.java | 2 +- .../commons/pipe/resource/log/PipeLogStatus.java | 6 +- .../iotdb/commons/service/metric/enums/Metric.java | 10 +- .../apache/iotdb/commons/utils/JVMCommonUtils.java | 56 +- .../apache/iotdb/commons/utils/StatusUtils.java | 1 - .../iotdb/commons/pipe/task/PipeMetaDeSerTest.java | 22 +- iotdb-core/pom.xml | 2 +- iotdb-core/relational-grammar/pom.xml | 2 +- .../db/relational/grammar/sql/RelationalSql.g4 | 15 +- iotdb-doap.rdf | 8 + iotdb-protocol/openapi/pom.xml | 2 +- .../src/main/openapi3/iotdb_rest_table_v1.yaml | 2 +- iotdb-protocol/pom.xml | 2 +- iotdb-protocol/thrift-ainode/pom.xml | 4 +- .../thrift-ainode/src/main/thrift/ainode.thrift | 18 +- iotdb-protocol/thrift-commons/pom.xml | 2 +- .../thrift-commons/src/main/thrift/common.thrift | 7 + iotdb-protocol/thrift-confignode/pom.xml | 4 +- .../src/main/thrift/confignode.thrift | 5 +- iotdb-protocol/thrift-consensus/pom.xml | 4 +- iotdb-protocol/thrift-datanode/pom.xml | 4 +- .../src/main/thrift/datanode.thrift | 9 +- library-udf/pom.xml | 4 +- pom.xml | 26 +- scripts/sbin/stop-ainode.sh | 22 +- scripts/sbin/windows/stop-ainode.bat | 18 +- .../tools/tsfile/reset-resource-pipe-statistics.sh | 51 ++ .../tsfile/reset-resource-pipe-statistics.bat | 59 ++ 580 files changed, 14283 insertions(+), 9222 deletions(-) diff --cc iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java index bae3665c10a,7a43578b975..1a7695b261a --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java @@@ -35,9 -34,9 +35,11 @@@ import org.apache.tsfile.utils.ReadWrit import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.nio.ByteBuffer; import java.time.LocalDate; import java.util.ArrayList;
