This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/forecast in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1c0826be5019e897589db3379ceb582abed9693b Merge: a4cfe9bad23 e27245e129e Author: JackieTien97 <[email protected]> AuthorDate: Mon Apr 28 10:47:22 2025 +0800 Merge remote-tracking branch 'origin/master' into ty/forecast distribution/src/assembly/ainode.xml | 6 + distribution/src/assembly/all.xml | 50 +- distribution/src/assembly/cli.xml | 17 +- distribution/src/assembly/confignode.xml | 41 +- distribution/src/assembly/datanode.xml | 45 +- .../DockerCompose/docker-compose-host-3c3d.yml | 2 + .../DockerCompose/docker-compose-standalone.yml | 4 +- .../main/DockerCompose/replace-conf-from-env.sh | 11 +- .../TableModelSubscriptionSessionExample.java | 18 +- .../iotdb/udf/table/ExcludeColumnExample.java | 11 +- .../org/apache/iotdb/udf/table/RepeatExample.java | 21 +- .../org/apache/iotdb/udf/table/SplitExample.java | 27 +- integration-test/src/assembly/mpp-test.xml | 32 +- .../example/relational/MyErrorTableFunction.java | 15 +- .../udf/example/relational/MyExcludeColumn.java | 11 +- .../udf/example/relational/MyRepeatWithIndex.java | 17 +- .../example/relational/MyRepeatWithoutIndex.java | 17 +- .../{MyExcludeColumn.java => MySelectColumn.java} | 21 +- .../db/query/udf/example/relational/MySplit.java | 27 +- .../it/env/cluster/config/MppConfigNodeConfig.java | 12 + .../it/env/cluster/config/MppDataNodeConfig.java | 12 + .../iotdb/it/env/cluster/env/AbstractEnv.java | 13 +- .../env/remote/config/RemoteConfigNodeConfig.java | 10 + .../it/env/remote/config/RemoteDataNodeConfig.java | 10 + .../iotdb/it/env/remote/env/RemoteServerEnv.java | 13 +- .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 11 +- .../apache/iotdb/itbase/env/ConfigNodeConfig.java | 5 + .../apache/iotdb/itbase/env/DataNodeConfig.java | 4 + .../apache/iotdb/cli/it/StartClientScriptIT.java | 4 +- .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 54 ++ .../org/apache/iotdb/db/it/IoTDBRestartIT.java | 2 - .../apache/iotdb/db/it/metric/IoTDBMetricIT.java | 92 ++- .../org/apache/iotdb/db/it/utils/TestUtils.java | 29 +- .../manual/basic/IoTDBPipeWithLoadIT.java | 107 +++ .../relational/it/db/it/IoTDBWindowTVFIT.java | 23 +- .../db/it/udf/IoTDBUserDefinedTableFunctionIT.java | 16 + .../it/query/recent/IoTDBTableAggregationIT.java | 77 +++ .../it/local/IoTDBSubscriptionPermissionIT.java | 339 ++++++++++ .../apache/iotdb/tools/it/ExportDataTestIT.java | 6 +- .../apache/iotdb/tools/it/ExportSchemaTestIT.java | 10 +- .../apache/iotdb/tools/it/ExportTsFileTestIT.java | 4 +- .../apache/iotdb/tools/it/ImportDataTestIT.java | 2 +- .../apache/iotdb/tools/it/ImportSchemaTestIT.java | 22 +- .../api/relational/EmptyTableFunctionHandle.java | 18 +- .../iotdb/udf/api/relational/TableFunction.java | 10 +- .../relational/table/MapTableFunctionHandle.java | 186 ++++++ .../relational/table/TableFunctionAnalysis.java | 21 +- .../api/relational/table/TableFunctionHandle.java | 27 +- .../table/argument/ScalarArgumentChecker.java | 17 +- .../processor/TableFunctionDataProcessor.java | 12 +- .../ScalarParameterSpecification.java | 23 +- iotdb-client/cli/src/assembly/cli.xml | 10 + .../src/main/java/org/apache/iotdb/cli/Cli.java | 1 + .../apache/iotdb/tool/data/AbstractDataTool.java | 12 + .../org/apache/iotdb/tool/data/ImportDataTree.java | 57 +- .../iotdb/tool/tsfile/ImportTsFileRemotely.java | 6 +- .../subscription/SubscriptionTableTsFile.java | 2 +- iotdb-client/client-cpp/src/main/Session.cpp | 2 +- .../client-cpp/src/main/SessionConnection.h | 2 +- iotdb-client/client-py/README.md | 4 +- iotdb-client/client-py/iotdb/Session.py | 18 +- iotdb-client/client-py/iotdb/SessionPool.py | 4 +- iotdb-client/client-py/iotdb/utils/Field.py | 29 +- .../client-py/iotdb/utils/SessionDataSet.py | 19 +- .../client-py/iotdb/utils/iotdb_rpc_dataset.py | 38 +- iotdb-client/client-py/iotdb/utils/rpc_utils.py | 41 ++ iotdb-client/client-py/requirements.txt | 1 + iotdb-client/client-py/resources/pyproject.toml | 3 +- .../session_aligned_timeseries_example.py | 4 +- iotdb-client/client-py/session_example.py | 4 +- iotdb-client/client-py/session_pool_example.py | 2 +- iotdb-client/client-py/session_ssl_example.py | 4 +- .../client-py/table_model_session_example.py | 4 +- .../integration/tablet_performance_comparison.py | 4 +- .../tests/integration/test_new_data_types.py | 12 +- .../tests/integration/test_tablemodel_query.py | 22 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 134 ++-- .../rpc/subscription/annotation/TableModel.java | 2 +- .../rpc/subscription/config/ConsumerConfig.java | 8 + .../response/PipeSubscribeHeartbeatResp.java | 48 +- .../apache/iotdb/session/SessionConnection.java | 148 ++++- .../SubscriptionSessionConnection.java | 32 - .../base/AbstractSubscriptionConsumer.java | 2 +- .../base/AbstractSubscriptionConsumerBuilder.java | 20 - .../base/AbstractSubscriptionProvider.java | 13 +- .../base/AbstractSubscriptionProviders.java | 6 +- .../SubscriptionTablePullConsumerBuilder.java | 3 +- .../SubscriptionTablePushConsumerBuilder.java | 3 +- .../tree/SubscriptionTreePullConsumerBuilder.java | 3 +- .../tree/SubscriptionTreePushConsumerBuilder.java | 3 +- .../payload/SubscriptionSessionDataSet.java | 93 ++- iotdb-core/ainode/ainode.xml | 27 + iotdb-core/confignode/src/assembly/confignode.xml | 28 +- .../iotdb/confignode/manager/ConfigManager.java | 2 +- .../iotdb/confignode/manager/ProcedureManager.java | 6 +- .../region/GreedyCopySetRegionGroupAllocator.java | 358 +++++++++- .../region/GreedyRegionGroupAllocator.java | 13 + .../balancer/region/IRegionGroupAllocator.java | 21 + .../PartiteGraphPlacementRegionGroupAllocator.java | 13 + .../persistence/schema/ClusterSchemaInfo.java | 15 +- .../procedure/env/RemoveDataNodeHandler.java | 186 ++++++ .../procedure/impl/StateMachineProcedure.java | 11 +- .../impl/node/RemoveDataNodesProcedure.java | 6 +- .../consumer/CreateConsumerProcedure.java | 4 +- .../subscription/CreateSubscriptionProcedure.java | 21 +- .../GreedyCopySetRemoveNodeReplicaSelectTest.java | 342 ++++++++++ iotdb-core/datanode/src/assembly/server.xml | 44 +- .../apache/iotdb/db/conf/DataNodeMemoryConfig.java | 12 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 - .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 14 +- .../task/builder/PipeDataNodeTaskBuilder.java | 51 +- .../subtask/connector/PipeConnectorSubtask.java | 13 +- .../PipeRealtimePriorityBlockingQueue.java | 26 +- .../IoTDBDataNodeCacheLeaderClientManager.java | 12 +- .../batch/PipeTabletEventTsFileBatch.java | 9 +- .../batch/PipeTransferBatchReqBuilder.java | 41 +- .../pipeconsensus/PipeConsensusSyncConnector.java | 3 - .../PipeConsensusTransferBatchReqBuilder.java | 19 +- .../async/IoTDBDataRegionAsyncConnector.java | 31 +- ...ilder.java => PipeTableModelTsFileBuilder.java} | 6 +- .../builder/PipeTableModelTsFileBuilderV2.java | 259 ++++++++ .../deletion/persist/PageCacheDeletionBuffer.java | 12 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 5 + .../event/common/terminate/PipeTerminateEvent.java | 6 +- .../scan/TsFileInsertionEventScanParser.java | 5 +- ...ileInsertionEventTableParserTabletIterator.java | 16 +- ...oricalDataRegionTsFileAndDeletionExtractor.java | 30 +- .../PipeRealtimeDataRegionHybridExtractor.java | 5 +- .../realtime/assigner/PipeDataRegionAssigner.java | 5 +- .../pipe/processor/twostage/combiner/Combiner.java | 5 +- .../protocol/thrift/IoTDBDataNodeReceiver.java | 10 +- .../db/pipe/resource/memory/PipeMemoryManager.java | 54 +- .../pipe/resource/tsfile/PipeTsFileResource.java | 14 +- .../resource/tsfile/PipeTsFileResourceManager.java | 2 +- .../rest/table/v1/handler/ExceptionHandler.java | 4 + .../protocol/rest/v1/handler/ExceptionHandler.java | 4 + .../protocol/rest/v2/handler/ExceptionHandler.java | 4 + .../execution/executor/RegionReadExecutor.java | 17 +- .../process/function/TableFunctionOperator.java | 18 +- .../relational/aggregation/AccumulatorFactory.java | 5 + .../ApproxCountDistinctAccumulator.java | 265 ++++++++ .../source/relational/aggregation/HyperLogLog.java | 246 +++++++ .../aggregation/HyperLogLogStateFactory.java | 88 +++ .../GroupedApproxCountDistinctAccumulator.java | 314 +++++++++ .../aggregation/grouped/array/BinaryBigArray.java | 4 - .../grouped/array/HyperLogLogBigArray.java | 84 +++ .../aggregation/grouped/array/MapBigArray.java | 4 - .../aggregation/grouped/array/ObjectBigArray.java | 12 +- .../relational/ColumnTransformerBuilder.java | 2 +- .../iotdb/db/queryengine/plan/Coordinator.java | 4 - .../plan/analyze/load/LoadTsFileAnalyzer.java | 66 +- .../analyze/load/LoadTsFileTableSchemaCache.java | 78 ++- .../analyze/schema/AutoCreateSchemaExecutor.java | 14 +- .../analyze/schema/ClusterSchemaFetchExecutor.java | 9 +- .../plan/analyze/schema/ClusterSchemaFetcher.java | 1 - .../config/executor/ClusterConfigTaskExecutor.java | 4 +- .../plan/planner/TableOperatorGenerator.java | 5 +- .../queryengine/plan/planner/TreeModelPlanner.java | 2 - .../plan/planner/plan/node/PlanGraphPrinter.java | 30 +- .../plan/node/metadata/write/MeasurementGroup.java | 25 +- .../relational/analyzer/StatementAnalyzer.java | 9 + .../TableFunctionInvocationAnalysis.java | 8 + .../relational/metadata/TableMetadataImpl.java | 15 + .../plan/relational/planner/RelationPlanner.java | 2 +- .../plan/relational/planner/TableModelPlanner.java | 9 - .../rule/ImplementTableFunctionSource.java | 4 +- .../rule/PruneTableFunctionProcessorColumns.java | 2 +- .../PruneTableFunctionProcessorSourceColumns.java | 2 +- .../relational/planner/node/TableFunctionNode.java | 57 +- .../planner/node/TableFunctionProcessorNode.java | 51 +- .../optimizations/UnaliasSymbolReferences.java | 6 +- .../plan/relational/sql/ast/Expression.java | 3 + .../plan/relational/sql/ast/LoadTsFile.java | 10 +- .../queryengine/plan/relational/sql/ast/Row.java | 33 + .../relational/sql/ast/TableExpressionType.java | 3 +- .../plan/relational/sql/parser/AstBuilder.java | 9 + .../plan/scheduler/AsyncPlanNodeSender.java | 21 +- .../plan/scheduler/ClusterScheduler.java | 5 - .../FailedFragmentInstanceWithStatus.java | 26 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 248 ++++--- .../db/schemaengine/table/DataNodeTableCache.java | 18 +- .../performer/ICrossCompactionPerformer.java | 6 + ...rformer.java => IInnerCompactionPerformer.java} | 11 +- .../execute/performer/ISeqCompactionPerformer.java | 2 +- .../performer/IUnseqCompactionPerformer.java | 2 +- .../performer/impl/FastCompactionPerformer.java | 15 + .../impl/ReadChunkCompactionPerformer.java | 8 + .../impl/ReadPointCompactionPerformer.java | 8 + .../execute/task/AbstractCompactionTask.java | 10 + .../execute/task/InnerSpaceCompactionTask.java | 18 +- .../estimator/AbstractCompactionEstimator.java | 124 +++- .../estimator/AbstractCrossSpaceEstimator.java | 6 +- .../estimator/AbstractInnerSpaceEstimator.java | 5 +- .../estimator/CompactionEstimateUtils.java | 91 ++- .../selector/estimator/CompactionTaskInfo.java | 6 - ...taInfo.java => CompactionTaskMetadataInfo.java} | 12 +- .../FastCompactionInnerCompactionEstimator.java | 44 +- .../FastCrossSpaceCompactionEstimator.java | 29 +- .../compaction/selector/estimator/FileInfo.java | 39 +- .../ReadChunkInnerCompactionEstimator.java | 25 +- .../RepairUnsortedFileCompactionEstimator.java | 21 +- .../impl/NewSizeTieredCompactionSelector.java | 76 ++- .../impl/RewriteCrossSpaceCompactionSelector.java | 2 +- .../dataregion/wal/buffer/WALBuffer.java | 9 +- .../dataregion/wal/io/WALInputStream.java | 2 + .../receiver/SubscriptionReceiverV1.java | 47 +- .../apache/iotdb/db/utils/ErrorHandlingUtils.java | 4 +- .../apache/iotdb/db/utils/ModificationUtils.java | 23 +- .../iotdb/db/utils/constant/SqlConstant.java | 2 + .../db/queryengine/plan/function/Exclude.java | 11 +- .../iotdb/db/queryengine/plan/function/Repeat.java | 22 +- .../iotdb/db/queryengine/plan/function/Split.java | 27 +- .../relational/analyzer/TableFunctionTest.java | 92 +-- .../assertions/TableFunctionProcessorMatcher.java | 178 +---- .../org/apache/iotdb/db/script/EnvScriptIT.java | 12 +- .../utils/CompactionTaskMemCostEstimatorTest.java | 87 ++- .../wal/compression/WALCompressionTest.java | 45 +- iotdb-core/metrics/interface/pom.xml | 5 + .../apache/iotdb/metrics/config/MetricConfig.java | 38 ++ .../metrics/config/MetricConfigDescriptor.java | 19 + .../reporter/prometheus/PrometheusReporter.java | 62 +- .../conf/iotdb-system.properties.template | 11 + .../apache/iotdb/commons/conf/CommonConfig.java | 445 +++++++++++++ .../iotdb/commons/conf/CommonDescriptor.java | 486 +------------- .../exception/QuerySchemaFetchFailedException.java | 19 +- .../task/connection/BlockingPendingQueue.java | 13 +- .../agent/task/execution/PipeSubtaskScheduler.java | 15 +- .../agent/task/progress/PipeEventCommitter.java | 28 +- .../iotdb/commons/pipe/config/PipeConfig.java | 5 +- .../config/PipeDescriptor.java} | 726 +++++---------------- .../config/constant/PipeConnectorConstant.java | 3 + .../pipe/connector/protocol/IoTDBConnector.java | 10 + .../commons/pipe/receiver/IoTDBFileReceiver.java | 18 +- .../meta/consumer/ConsumerGroupMeta.java | 49 ++ .../subscription/meta/consumer/ConsumerMeta.java | 12 + .../commons/subscription/meta/topic/TopicMeta.java | 9 +- .../TableBuiltinAggregationFunction.java | 2 + .../relational/tvf/CapacityTableFunction.java | 27 +- .../relational/tvf/CumulateTableFunction.java | 40 +- .../builtin/relational/tvf/HOPTableFunction.java | 44 +- .../relational/tvf/SessionTableFunction.java | 24 +- .../relational/tvf/TumbleTableFunction.java | 32 +- .../relational/tvf/VariationTableFunction.java | 25 +- .../apache/iotdb/commons/utils/StatusUtils.java | 2 +- .../thrift-commons/src/main/thrift/common.thrift | 3 +- library-udf/src/assembly/tools/register-UDF.bat | 110 ++-- .../resources => scripts}/conf/ainode-env.sh | 0 .../resources => scripts}/conf/confignode-env.sh | 21 +- .../resources => scripts}/conf/datanode-env.sh | 24 +- .../sbin => scripts/conf}/iotdb-common.sh | 0 .../conf => scripts/conf/windows}/ainode-env.bat | 11 +- .../conf/windows}/confignode-env.bat | 2 +- .../conf => scripts/conf/windows}/datanode-env.bat | 2 +- .../sbin => scripts/sbin/cluster}/start-all.sh | 2 +- .../sbin => scripts/sbin/cluster}/stop-all.sh | 2 +- .../resources => scripts}/sbin/start-ainode.sh | 17 +- .../resources => scripts}/sbin/start-cli.sh | 0 .../resources => scripts}/sbin/start-confignode.sh | 2 +- .../resources => scripts}/sbin/start-datanode.sh | 2 +- .../resources => scripts}/sbin/start-standalone.sh | 0 .../resources => scripts}/sbin/stop-ainode.sh | 0 .../resources => scripts}/sbin/stop-confignode.sh | 2 +- .../resources => scripts}/sbin/stop-datanode.sh | 2 +- .../resources => scripts}/sbin/stop-standalone.sh | 0 .../sbin => scripts/sbin/windows}/start-ainode.bat | 12 +- .../sbin/windows/start-cli-table.bat | 4 +- .../sbin => scripts/sbin/windows}/start-cli.bat | 2 +- .../sbin/windows}/start-confignode.bat | 8 +- .../sbin/windows}/start-datanode.bat | 14 +- .../sbin/windows}/start-standalone.bat | 12 +- .../sbin => scripts/sbin/windows}/stop-ainode.bat | 8 +- .../sbin/windows}/stop-confignode.bat | 13 +- .../sbin/windows}/stop-datanode.bat | 13 +- .../sbin/windows}/stop-standalone.bat | 10 +- .../resources => scripts}/tools/export-data.sh | 0 .../resources => scripts}/tools/import-data.sh | 0 .../resources => scripts}/tools/load-tsfile.sh | 2 +- .../tools => scripts/tools/ops}/backup.sh | 2 +- .../tools => scripts/tools/ops}/collect-info.sh | 2 +- .../tools/ops}/daemon-confignode.sh | 2 +- .../sbin => scripts/tools/ops}/daemon-datanode.sh | 2 +- .../sbin => scripts/tools/ops}/destroy-all.sh | 18 +- .../tools/ops}/destroy-confignode.sh | 2 +- .../sbin => scripts/tools/ops}/destroy-datanode.sh | 2 +- .../sbin => scripts/tools/ops}/health_check.sh | 4 +- .../sbin => scripts/tools/ops}/remove-ainode.sh | 23 +- .../tools/schema}/export-schema.sh | 2 +- .../tools/schema}/import-schema.sh | 2 +- .../tools/schema/print-pb-tree-file.sh | 2 +- .../tools/schema/print-schema-log.sh | 2 +- .../check-overlap-sequence-files-and-repair.sh | 2 +- .../tools/tsfile/overlap-statistic-tool.sh | 2 +- .../tools/tsfile/print-iotdb-data-dir.sh | 2 +- .../tools/tsfile/print-tsfile-resource-files.sh | 2 +- .../tools/tsfile/print-tsfile.sh | 2 +- .../tools/tsfile/settle-tsfile.sh | 0 .../tools/tsfile/split-tsfile-tool.sh | 2 +- .../tools/tsfile/validate-tsfile.sh | 2 +- .../tools/windows}/export-data.bat | 2 +- .../tools/windows}/import-data.bat | 2 +- .../tools/windows}/load-tsfile.bat | 2 +- .../tools => scripts/tools/windows/ops}/backup.bat | 6 +- .../tools/windows/ops}/collect-info.bat | 8 +- .../tools/windows/ops}/destroy-all.bat | 8 +- .../tools/windows/ops}/destroy-confignode.bat | 4 +- .../tools/windows/ops}/destroy-datanode.bat | 4 +- .../tools/windows/ops}/health_check.bat | 14 +- .../tools/windows/ops}/remove-ainode.bat | 20 +- .../tools/windows/schema}/export-schema.bat | 2 +- .../tools/windows/schema}/import-schema.bat | 2 +- .../tools/windows}/schema/print-pb-tree-file.bat | 2 +- .../tools/windows}/schema/print-schema-log.bat | 2 +- .../check-overlap-sequence-files-and-repair.bat | 2 +- .../windows}/tsfile/overlap-statistic-tool.bat | 2 +- .../tools/windows}/tsfile/print-iotdb-data-dir.bat | 2 +- .../tsfile/print-tsfile-resource-files.bat | 2 +- .../tools/windows}/tsfile/print-tsfile.bat | 2 +- .../tools/windows}/tsfile/settle-tsfile.bat | 2 +- .../tools/windows}/tsfile/split-tsfile-tool.bat | 2 +- .../tools/windows}/tsfile/validate-tsfile.bat | 2 +- 320 files changed, 7017 insertions(+), 2776 deletions(-)
