This is an automated email from the ASF dual-hosted git repository. chenyz pushed a commit to branch builtin-udtf in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5385332fd379c1641a3f712dde1059780be47f28 Merge: d969f365a65 3327dfa7809 Author: Chen YZ <[email protected]> AuthorDate: Mon Mar 17 15:28:21 2025 +0800 merge .github/workflows/vulnerability-check.yml | 7 +- a | 0 dependencies.json | 1 + .../server/CustomizedJsonPayloadFormatter.java | 8 +- .../org/apache/iotdb/mqtt/server/MyMessage.java | 80 -- .../java/org/apache/iotdb/mqtt/MQTTClient.java | 2 +- .../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 17 +- .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 73 -- .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 132 +++ .../it/auth/IoTDBClusterAuthorityRelationalIT.java | 1 - .../iotdb/db/it/auth/IoTDBRelationalAuthIT.java | 62 +- .../iotdb/db/it/auth/IoTDBSystemPermissionIT.java | 2 - .../org/apache/iotdb/db/it/utils/TestUtils.java | 1 - .../pipe/it/dual/tablemodel/TableModelUtils.java | 6 + .../manual/basic/IoTDBPipeDataSinkIT.java | 6 + .../manual/basic/IoTDBPipeExtractorIT.java | 7 + .../manual/basic/IoTDBPipeLifeCycleIT.java | 64 +- .../manual/basic/IoTDBPipePermissionIT.java | 226 +++++ .../manual/basic/IoTDBPipeProtocolIT.java | 4 + .../manual/basic/IoTDBPipeSwitchStatusIT.java | 4 + .../tablemodel/manual/basic/IoTDBPipeSyntaxIT.java | 1 + .../manual/basic/IoTDBPipeWithLoadIT.java | 4 + .../manual/basic/IoTDBTablePatternFormatIT.java | 7 + .../manual/enhanced/IoTDBPipeAutoConflictIT.java | 4 + .../manual/enhanced/IoTDBPipeAutoDropIT.java | 2 + .../manual/enhanced/IoTDBPipeClusterIT.java | 13 + .../enhanced/IoTDBPipeConnectorCompressionIT.java | 12 +- .../enhanced/IoTDBPipeConnectorParallelIT.java | 2 + .../manual/enhanced/IoTDBPipeIdempotentIT.java | 13 + .../manual/enhanced/IoTDBPipeMetaIT.java | 4 + .../manual/enhanced/IoTDBPipeNullValueIT.java | 1 + .../pipe/it/single/IoTDBPipePermissionIT.java | 157 ++++ .../it/db/it/IoTDBAuthenticationTableIT.java | 114 ++- .../it/db/it/IoTDBFlushQueryTableIT.java | 30 +- .../relational/it/db/it/IoTDBInsertTableIT.java | 47 +- .../db/it/IoTDBMultiTAGsWithAttributesTableIT.java | 12 +- .../db/it/udf/IoTDBUserDefinedTableFunctionIT.java | 13 + .../relational/it/mqtt/IoTDBMQTTServiceIT.java | 2 +- .../scalar/IoTDBFormatFunctionTableIT.java | 18 +- .../it/query/old/query/IoTDBArithmeticTableIT.java | 12 + .../query/old/query/IoTDBGreatestLeastTableIT.java | 2 +- .../it/query/recent/IoTDBColumnsMatchTableIT.java | 354 ++++++++ .../it/query/recent/IoTDBMaintainAuthIT.java | 89 +- .../it/query/recent/IoTDBTableAggregationIT.java | 4 +- .../recent/IoTDBTableAggregationNonStreamIT.java | 56 ++ .../IoTDBCorrelatedExistsSubqueryIT.java | 428 ++++++++++ .../IoTDBUncorrelatedExistsSubqueryIT.java | 391 +++++++++ .../relational/it/schema/IoTDBDatabaseIT.java | 59 +- .../iotdb/relational/it/schema/IoTDBTableIT.java | 37 +- .../it/session/IoTDBSessionRelationalIT.java | 133 +-- .../iotdb/session/it/IoTDBSessionSimpleIT.java | 48 -- .../it/local/IoTDBSubscriptionBasicIT.java | 38 +- .../IoTDBDefaultPullConsumerDataSetIT.java | 28 +- ...sensusRetryWithIncreasingIntervalException.java | 15 +- .../iotdb/udf/api/relational/TableFunction.java | 20 +- .../relational/table/TableFunctionAnalysis.java | 2 +- .../specification/TableParameterSpecification.java | 18 + .../java/org/apache/iotdb/cli/AbstractCli.java | 20 +- .../org/apache/iotdb/tool/common/Constants.java | 3 +- .../org/apache/iotdb/tool/common/OptionsUtil.java | 55 +- .../org/apache/iotdb/tool/data/ExportData.java | 15 +- .../apache/iotdb/tool/data/ExportDataTable.java | 192 +++-- .../org/apache/iotdb/tool/data/ExportDataTree.java | 185 ++-- .../org/apache/iotdb/tool/data/ImportData.java | 2 +- .../apache/iotdb/tool/data/ImportDataTable.java | 16 +- .../iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java | 88 +- .../org/apache/iotdb/jdbc/IoTDBConnection.java | 7 + .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 64 ++ .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 25 +- .../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 31 +- .../IoTDBRelationalDatabaseMetadata.java | 167 +++- .../java/org/apache/iotdb/session/Session.java | 17 +- .../base/AbstractSubscriptionConsumer.java | 29 +- .../SubscriptionTablePushConsumerBuilder.java | 3 +- iotdb-core/ainode/poetry.lock | 149 +++- iotdb-core/confignode/pom.xml | 4 + .../iotdb/confignode/conf/ConfigNodeConfig.java | 51 +- .../confignode/conf/ConfigNodeDescriptor.java | 30 + .../consensus/request/ConfigPhysicalPlan.java | 8 + .../consensus/request/ConfigPhysicalPlanType.java | 2 + .../request/ConfigPhysicalPlanVisitor.java | 15 + .../write/table/SetTableColumnCommentPlan.java | 62 ++ .../request/write/table/SetTableCommentPlan.java | 57 ++ .../iotdb/confignode/manager/ConfigManager.java | 23 + .../apache/iotdb/confignode/manager/IManager.java | 3 + .../iotdb/confignode/manager/ModelManager.java | 5 + .../confignode/manager/PermissionManager.java | 4 + .../manager/load/balancer/PartitionBalancer.java | 8 +- .../manager/load/balancer/RouteBalancer.java | 154 +++- .../manager/load/cache/AbstractLoadCache.java | 26 +- .../cache/IFailureDetector.java} | 26 +- .../manager/load/cache/detector/FixedDetector.java | 58 ++ .../load/cache/detector/PhiAccrualDetector.java | 181 ++++ .../load/cache/node/AINodeHeartbeatCache.java | 42 +- .../load/cache/node/ConfigNodeHeartbeatCache.java | 30 +- .../load/cache/node/DataNodeHeartbeatCache.java | 40 +- .../manager/load/cache/region/RegionCache.java | 30 +- .../iotdb/confignode/manager/node/NodeManager.java | 53 +- .../pipe/agent/task/PipeConfigNodeSubtask.java | 2 +- .../pipe/agent/task/PipeConfigNodeTaskAgent.java | 4 +- .../pipe/event/PipeConfigRegionSnapshotEvent.java | 14 +- .../pipe/event/PipeConfigRegionWritePlanEvent.java | 26 +- .../extractor/ConfigRegionListeningFilter.java | 6 +- .../pipe/extractor/IoTDBConfigRegionExtractor.java | 112 ++- ...ConfigPhysicalPlanTablePatternParseVisitor.java | 22 + ...nfigPhysicalPlanTablePrivilegeParseVisitor.java | 364 ++++++++ ...eConfigPhysicalPlanTreePatternParseVisitor.java | 2 +- .../manager/pipe/metric/PipeConfigNodeMetrics.java | 9 + .../PipeConfigNodeRemainingTimeMetrics.java | 2 +- .../PipeConfigNodeRemainingTimeOperator.java | 2 +- .../PipeConfigNodeResourceMetrics.java | 2 +- .../{ => overview}/PipeProcedureMetrics.java | 2 +- .../metric/{ => overview}/PipeTaskInfoMetrics.java | 2 +- .../PipeTemporaryMetaInCoordinatorMetrics.java | 2 +- .../PipeConfigNodeReceiverMetrics.java | 2 +- .../PipeConfigRegionConnectorMetrics.java | 2 +- .../PipeConfigNodeListenerMetrics.java | 2 +- .../PipeConfigRegionExtractorMetrics.java | 2 +- .../receiver/protocol/IoTDBConfigNodeReceiver.java | 120 ++- .../PipeConfigPhysicalPlanTSStatusVisitor.java | 36 +- .../resource/PipeConfigNodeResourceManager.java | 1 + .../PipeConfigNodeSnapshotResourceManager.java | 2 +- .../manager/schema/ClusterSchemaManager.java | 52 +- .../iotdb/confignode/persistence/AuthorInfo.java | 4 + .../iotdb/confignode/persistence/ModelInfo.java | 2 +- .../iotdb/confignode/persistence/TTLInfo.java | 2 +- .../persistence/executor/ConfigPlanExecutor.java | 6 + .../partition/DatabasePartitionTable.java | 14 +- .../confignode/persistence/pipe/PipeInfo.java | 2 +- .../persistence/schema/ClusterSchemaInfo.java | 59 ++ .../confignode/persistence/schema/ConfigMTree.java | 31 + .../procedure/PartitionTableAutoCleaner.java | 17 +- .../procedure/impl/StateMachineProcedure.java | 9 +- .../impl/pipe/AbstractOperatePipeProcedureV2.java | 4 +- .../impl/pipe/task/AlterPipeProcedureV2.java | 67 +- .../impl/pipe/task/CreatePipeProcedureV2.java | 97 +++ .../impl/schema/UnsetTemplateProcedure.java | 146 ++-- .../iotdb/confignode/service/ConfigNode.java | 44 +- .../request/ConfigPhysicalPlanSerDeTest.java | 33 +- .../manager/load/cache/detector/DetectorTest.java | 179 ++++ ...igPhysicalPlanTablePatternParseVisitorTest.java | 18 + ...figPhysicalPlanTreePatternParseVisitorTest.java | 2 +- .../common/request/IConsensusRequest.java | 5 + .../common/request/IndexedConsensusRequest.java | 8 +- .../iotdb/consensus/config/IoTConsensusConfig.java | 30 +- .../consensus/config/PipeConsensusConfig.java | 21 +- .../apache/iotdb/consensus/iot/IoTConsensus.java | 4 +- .../consensus/iot/client/DispatchLogHandler.java | 27 +- .../logdispatcher/IoTConsensusMemoryManager.java | 50 +- .../consensus/iot/logdispatcher/LogDispatcher.java | 12 +- .../apache/iotdb/consensus/pipe/PipeConsensus.java | 80 +- .../consensus/pipe/PipeConsensusServerImpl.java | 25 +- .../pipe/consensuspipe/ConsensusPipeManager.java | 8 +- ...xManager.java => ReplicateProgressManager.java} | 8 +- .../pipe/metric/PipeConsensusSyncLagManager.java | 77 +- .../apache/iotdb/consensus/iot/StabilityTest.java | 49 +- iotdb-core/datanode/pom.xml | 8 + .../ArithmeticBinaryColumnTransformer.ftl | 11 + .../org/apache/iotdb/db/auth/AuthorityChecker.java | 11 + .../apache/iotdb/db/conf/DataNodeMemoryConfig.java | 684 +++++++++++++++ .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 363 +------- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 313 +------ .../db/consensus/DataRegionConsensusImpl.java | 15 +- .../PipeDataRegionConnectorConstructor.java | 4 + .../runtime/PipeSchemaRegionListenerManager.java | 2 +- .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 58 +- .../pipe/agent/task/connection/EnrichedDeque.java | 2 +- .../agent/task/connection/PipeEventCollector.java | 34 +- .../agent/task/stage/PipeTaskProcessorStage.java | 7 +- .../subtask/connector/PipeConnectorSubtask.java | 4 +- .../connector/PipeConnectorSubtaskManager.java | 2 +- .../PipeRealtimePriorityBlockingQueue.java | 51 +- .../subtask/processor/PipeProcessorSubtask.java | 7 +- .../client/IoTDBDataNodeAsyncClientManager.java | 4 + .../evolvable/batch/PipeTabletEventBatch.java | 37 +- .../evolvable/batch/PipeTabletEventPlainBatch.java | 41 +- .../batch/PipeTabletEventTsFileBatch.java | 11 +- .../connector/protocol/opcda/OpcDaConnector.java | 138 +++ .../pipe/connector/protocol/opcda/OpcDaHeader.java | 200 +++++ .../protocol/opcda/OpcDaServerHandle.java | 390 +++++++++ .../connector/protocol/opcua/OpcUaConnector.java | 50 +- .../connector/protocol/opcua/OpcUaNameSpace.java | 2 +- .../pipeconsensus/PipeConsensusAsyncConnector.java | 16 +- .../pipeconsensus/PipeConsensusSyncConnector.java | 9 +- .../PipeConsensusTabletInsertionEventHandler.java | 8 +- .../PipeConsensusTsFileInsertionEventHandler.java | 8 +- .../PipeConsensusTransferBatchReqBuilder.java | 5 +- .../async/IoTDBDataRegionAsyncConnector.java | 21 +- .../async/handler/PipeTransferTsFileHandler.java | 15 +- .../protocol/writeback/WriteBackConnector.java | 83 +- ....java => ReplicateProgressDataNodeManager.java} | 36 +- .../deletion/DeletionResourceManager.java | 4 +- .../deletion/persist/PageCacheDeletionBuffer.java | 14 +- .../db/pipe/event/UserDefinedEnrichedEvent.java | 14 +- .../db/pipe/event/common/PipeInsertionEvent.java | 30 +- .../common/deletion/PipeDeleteDataNodeEvent.java | 31 +- .../event/common/heartbeat/PipeHeartbeatEvent.java | 19 +- .../schema/PipeSchemaRegionSnapshotEvent.java | 16 +- .../schema/PipeSchemaRegionWritePlanEvent.java | 18 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 127 ++- .../common/tablet/PipeRawTabletInsertionEvent.java | 48 +- .../event/common/terminate/PipeTerminateEvent.java | 13 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 120 ++- .../parser/TsFileInsertionEventParserProvider.java | 5 +- .../scan/TsFileInsertionEventScanParser.java | 3 - .../table/TsFileInsertionEventTableParser.java | 111 ++- ...ileInsertionEventTableParserTabletIterator.java | 421 +++++++-- .../db/pipe/event/realtime/PipeRealtimeEvent.java | 28 +- .../event/realtime/PipeRealtimeEventFactory.java | 92 +- .../dataregion/IoTDBDataRegionExtractor.java | 6 +- ...oricalDataRegionTsFileAndDeletionExtractor.java | 42 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 22 +- .../PipeRealtimeDataRegionHybridExtractor.java | 9 +- .../realtime/assigner/PipeDataRegionAssigner.java | 14 +- .../dataregion/realtime/epoch/TsFileEpoch.java | 2 +- .../realtime/epoch/TsFileEpochManager.java | 8 +- .../listener/PipeInsertionDataNodeListener.java | 18 +- .../matcher/CachedSchemaPatternMatcher.java | 40 +- .../realtime/matcher/PipeDataRegionMatcher.java | 3 + .../schemaregion/IoTDBSchemaRegionExtractor.java | 82 +- .../PipePlanTablePrivilegeParseVisitor.java | 81 ++ .../schemaregion/PipeStatementToPlanVisitor.java | 48 ++ .../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 12 + .../PipeDataNodeRemainingEventAndTimeMetrics.java | 2 +- .../PipeDataNodeRemainingEventAndTimeOperator.java | 2 +- .../{ => overview}/PipeHeartbeatEventMetrics.java | 2 +- .../metric/{ => overview}/PipeResourceMetrics.java | 2 +- .../PipeWALInsertNodeCacheMetrics.java | 2 +- .../{ => processor}/PipeProcessorMetrics.java | 2 +- .../PipeDataNodeReceiverMetrics.java | 2 +- .../PipeSchemaRegionConnectorMetrics.java | 2 +- .../PipeSchemaRegionExtractorMetrics.java | 2 +- .../PipeSchemaRegionListenerMetrics.java | 2 +- .../{ => sink}/PipeDataRegionConnectorMetrics.java | 2 +- .../metric/{ => source}/PipeAssignerMetrics.java | 2 +- .../{ => source}/PipeDataRegionEventCounter.java | 2 +- .../PipeDataRegionExtractorMetrics.java | 2 +- .../pipeconsensus/PipeConsensusProcessor.java | 43 +- .../pipeconsensus/PipeConsensusReceiver.java | 368 ++++---- .../protocol/thrift/IoTDBDataNodeReceiver.java | 12 +- ...leStatementDataTypeConvertExecutionVisitor.java | 1 + .../resource/memory/InsertNodeMemoryEstimator.java | 2 +- .../db/pipe/resource/memory/PipeMemoryBlock.java | 2 +- .../db/pipe/resource/memory/PipeMemoryManager.java | 121 +-- .../pipe/resource/memory/PipeMemoryWeightUtil.java | 65 ++ .../pipe/resource/tsfile/PipeTsFileResource.java | 6 +- .../db/protocol/mqtt/JSONPayloadFormatter.java | 7 +- .../db/protocol/mqtt/LinePayloadFormatter.java | 7 +- .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 11 +- .../iotdb/db/protocol/mqtt/PayloadFormatter.java | 6 + .../iotdb/db/protocol/session/SessionManager.java | 12 + .../protocol/thrift/impl/ClientRPCServiceImpl.java | 2 +- .../impl/DataNodeInternalRPCServiceImpl.java | 71 +- .../db/queryengine/execution/driver/Driver.java | 15 +- .../execution/exchange/SharedTsBlockQueue.java | 13 +- .../execution/exchange/sink/ISinkChannel.java | 2 + .../execution/exchange/sink/LocalSinkChannel.java | 21 +- .../execution/exchange/sink/ShuffleSinkHandle.java | 5 +- .../execution/exchange/sink/SinkChannel.java | 5 +- .../exchange/source/LocalSourceHandle.java | 12 +- .../execution/exchange/source/SourceHandle.java | 2 +- .../execution/executor/RegionWriteExecutor.java | 18 +- .../fragment/FragmentInstanceContext.java | 32 +- .../fragment/FragmentInstanceExecution.java | 18 +- .../fragment/FragmentInstanceManager.java | 24 +- .../execution/fragment/QueryStatistics.java | 10 + .../execution/memory/LocalMemoryManager.java | 5 +- .../queryengine/execution/memory/MemoryPool.java | 42 +- .../execution/operator/AbstractOperator.java | 5 +- ...owOperator.java => AssignUniqueIdOperator.java} | 77 +- .../operator/process/EnforceSingleRowOperator.java | 2 +- .../operator/process/ai/InferenceOperator.java | 42 +- .../process/function/PartitionRecognizer.java | 86 +- .../process/function/TableFunctionOperator.java | 47 +- .../{SliceCache.java => PartitionCache.java} | 10 +- .../operator/process/function/partition/Slice.java | 157 ++-- .../join/SimpleNestedLoopCrossJoinOperator.java | 2 +- .../process/window/TableWindowOperator.java | 2 +- .../operator/schema/SchemaQueryScanOperator.java | 12 +- .../operator/schema/source/ISchemaSource.java | 5 + .../schema/source/TableDeviceFetchSource.java | 21 + .../schema/source/TableDeviceQuerySource.java | 36 +- .../operator/sink/ShuffleHelperOperator.java | 2 +- .../relational/AbstractAggTableScanOperator.java | 17 +- .../relational/AbstractMergeSortJoinOperator.java | 4 +- .../relational/AbstractTableScanOperator.java | 2 +- .../InformationSchemaContentSupplierFactory.java | 15 +- .../relational/LastQueryAggTableScanOperator.java | 6 +- .../relational/MergeSortLeftJoinOperator.java | 163 ++++ .../relational/aggregation/TableAggregator.java | 9 +- .../aggregation/grouped/GroupedAggregator.java | 4 +- .../grouped/GroupedFirstAccumulator.java | 11 +- .../grouped/GroupedFirstByAccumulator.java | 2 +- .../grouped/GroupedLastAccumulator.java | 1 + .../grouped/GroupedLastByAccumulator.java | 2 +- .../GroupedUserDefinedAggregateAccumulator.java | 4 - .../grouped/GroupedVarianceAccumulator.java | 2 +- .../grouped/StreamingHashAggregationOperator.java | 1 + .../execution/schedule/AbstractDriverThread.java | 27 +- .../execution/schedule/DriverScheduler.java | 8 +- .../metric/SeriesScanCostMetricSet.java | 40 +- .../iotdb/db/queryengine/plan/Coordinator.java | 8 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 78 +- .../analyze/cache/partition/PartitionCache.java | 16 +- .../cache/schema/DataNodeDevicePathCache.java | 23 +- .../schema/dualkeycache/impl/DualKeyCacheImpl.java | 9 +- .../plan/analyze/load/LoadTsFileAnalyzer.java | 562 +++++++++--- .../analyze/load/LoadTsFileTableSchemaCache.java | 24 +- .../load/LoadTsFileToTableModelAnalyzer.java | 226 ----- .../load/LoadTsFileToTreeModelAnalyzer.java | 187 ---- .../load/TreeSchemaAutoCreatorAndVerifier.java | 4 +- .../plan/analyze/lock/SchemaLockType.java | 13 - .../analyze/schema/AutoCreateSchemaExecutor.java | 6 +- .../analyze/schema/ClusterSchemaFetchExecutor.java | 11 +- .../queryengine/plan/execution/QueryExecution.java | 9 +- .../execution/config/TableConfigTaskVisitor.java | 218 ++++- .../config/executor/ClusterConfigTaskExecutor.java | 85 ++ .../config/executor/IConfigTaskExecutor.java | 16 + .../relational/AlterTableCommentColumnTask.java | 53 ++ .../relational/AlterTableCommentTableTask.java | 48 ++ .../relational/DescribeTableDetailsTask.java | 6 + .../metadata/relational/ShowTablesDetailsTask.java | 7 + .../execution/config/sys/pipe/ShowPipeTask.java | 2 +- .../db/queryengine/plan/parser/ASTVisitor.java | 3 + .../plan/planner/LocalExecutionPlanContext.java | 3 +- .../plan/planner/LocalExecutionPlanner.java | 124 ++- .../plan/planner/LogicalPlanBuilder.java | 1 + .../plan/planner/LogicalPlanVisitor.java | 6 +- .../plan/planner/OperatorTreeGenerator.java | 1 + .../plan/planner/TableOperatorGenerator.java | 56 ++ .../SimpleFragmentParallelPlanner.java | 2 +- .../distribution/WriteFragmentParallelPlanner.java | 3 +- .../plan/planner/plan/node/PlanGraphPrinter.java | 16 +- .../plan/planner/plan/node/PlanNodeType.java | 18 +- .../plan/planner/plan/node/PlanVisitor.java | 9 +- .../plan/node/process/AI/InferenceNode.java | 18 +- .../plan/planner/plan/node/write/InsertNode.java | 6 +- .../planner/plan/node/write/InsertRowNode.java | 3 - .../plan/node/write/RelationalInsertRowNode.java | 5 - .../node/write/RelationalInsertTabletNode.java | 8 - .../relational/analyzer/ExpressionAnalyzer.java | 6 + .../relational/analyzer/StatementAnalyzer.java | 941 ++++++++++++++++++++- .../analyzer/StatementAnalyzerFactory.java | 4 + .../PredicatePushIntoMetadataChecker.java | 7 +- .../relational/metadata/AlignedDeviceEntry.java | 20 +- .../plan/relational/metadata/DeviceEntry.java | 26 +- .../metadata/NonAlignedAlignedDeviceEntry.java | 20 +- .../relational/metadata/TableMetadataImpl.java | 2 +- .../fetcher/DeviceInCacheFilterVisitor.java | 4 +- .../metadata/fetcher/SchemaPredicateUtil.java | 10 +- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 22 +- .../fetcher/TableDeviceSchemaValidator.java | 6 +- .../fetcher/TableHeaderSchemaValidator.java | 47 +- .../fetcher/cache/TableAttributeSchema.java | 6 +- .../fetcher/cache/TableDeviceLastCache.java | 5 +- .../fetcher/cache/TableDeviceSchemaCache.java | 19 +- .../plan/relational/planner/OrderingScheme.java | 1 - .../plan/relational/planner/QueryPlanner.java | 4 +- .../plan/relational/planner/RelationPlanner.java | 3 +- .../relational/planner/TableLogicalPlanner.java | 13 +- .../distribute/TableDistributedPlanGenerator.java | 195 +++-- .../distribute/TableModelQueryFragmentPlanner.java | 2 +- .../TableModelTypeProviderExtractor.java | 4 +- .../plan/relational/planner/ir/IrUtils.java | 13 +- .../iterative/rule/AggregationDecorrelation.java | 83 ++ .../rule/ImplementTableFunctionSource.java | 4 +- .../rule/PruneAssignUniqueIdColumns.java} | 33 +- .../rule/PruneTableFunctionProcessorColumns.java | 1 - .../iterative/rule/RemoveRedundantExists.java | 104 +++ ...orrelatedDistinctAggregationWithProjection.java | 186 ++++ ...elatedDistinctAggregationWithoutProjection.java | 167 ++++ ...mCorrelatedGlobalAggregationWithProjection.java | 318 +++++++ ...rrelatedGlobalAggregationWithoutProjection.java | 302 +++++++ ...CorrelatedGroupedAggregationWithProjection.java | 252 ++++++ ...relatedGroupedAggregationWithoutProjection.java | 233 +++++ .../rule/TransformCorrelatedJoinToJoin.java | 99 +++ .../rule/TransformExistsApplyToCorrelatedJoin.java | 214 +++++ .../relational/planner/node/AssignUniqueId.java | 97 +++ .../plan/relational/planner/node/ExchangeNode.java | 6 +- .../{SortBasedGroupNode.java => GroupNode.java} | 39 +- .../plan/relational/planner/node/OutputNode.java | 2 + .../plan/relational/planner/node/Patterns.java | 27 +- .../planner/optimizations/JoinUtils.java | 249 ++++-- .../optimizations/LogicalOptimizeFactory.java | 40 +- .../planner/optimizations/ParallelizeGrouping.java | 86 +- .../optimizations/PlanNodeDecorrelator.java | 699 +++++++++++++++ .../PushLimitOffsetIntoTableScan.java | 14 +- .../optimizations/PushPredicateIntoTableScan.java | 158 +++- .../optimizations/TransformSortToStreamSort.java | 4 +- .../optimizations/UnaliasSymbolReferences.java | 21 +- .../relational/planner/optimizations/Util.java | 19 + .../plan/relational/security/AccessControl.java | 18 +- .../relational/security/AccessControlImpl.java | 29 +- .../relational/security/AllowAllAccessControl.java | 11 +- .../relational/security/ITableAuthChecker.java | 3 + .../relational/security/ITableAuthCheckerImpl.java | 18 + .../relational/security/TableModelPrivilege.java | 5 - .../relational/sql/ast/AbstractTraverseDevice.java | 2 + .../plan/relational/sql/ast/AstVisitor.java | 20 +- .../plan/relational/sql/ast/ColumnDefinition.java | 27 +- .../ast/{EmptyTableTreatment.java => Columns.java} | 56 +- .../plan/relational/sql/ast/CreateTable.java | 8 + .../plan/relational/sql/ast/InsertRows.java | 3 - .../plan/relational/sql/ast/LoadTsFile.java | 14 +- .../plan/relational/sql/ast/SetColumnComment.java | 116 +++ .../plan/relational/sql/ast/SetProperties.java | 12 - .../plan/relational/sql/ast/SetTableComment.java | 98 +++ .../sql/ast/ShowQueriesStatement.java} | 24 +- .../plan/relational/sql/ast/SingleColumn.java | 21 + .../sql/ast/TableFunctionInvocation.java | 41 +- .../queryengine/plan/relational/sql/ast/Trim.java | 2 +- .../plan/relational/sql/parser/AstBuilder.java | 104 ++- .../plan/relational/sql/rewrite/ShowRewrite.java | 26 +- .../sql/rewrite/StatementRewriteFactory.java | 6 +- .../relational/sql/util/ExpressionFormatter.java | 6 + .../plan/relational/sql/util/SqlFormatter.java | 54 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 54 +- .../scheduler/load/LoadTsFileDispatcherImpl.java | 11 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 96 ++- .../plan/statement/crud/InsertBaseStatement.java | 28 +- .../plan/statement/crud/InsertRowStatement.java | 3 - .../plan/statement/crud/LoadTsFileStatement.java | 18 +- .../plan/statement/crud/QueryStatement.java | 9 + .../metadata/CountTimeSlotListStatement.java | 7 + .../statement/metadata/GetRegionIdStatement.java | 7 + .../metadata/GetSeriesSlotListStatement.java | 7 + .../metadata/GetTimeSlotListStatement.java | 7 + .../metadata/RemoveConfigNodeStatement.java | 9 +- .../metadata/RemoveDataNodeStatement.java | 9 +- .../statement/metadata/ShowClusterIdStatement.java | 7 + .../statement/metadata/ShowClusterStatement.java | 9 +- .../metadata/ShowConfigNodesStatement.java | 4 +- .../statement/metadata/ShowDataNodesStatement.java | 4 +- .../statement/metadata/ShowRegionStatement.java | 4 +- .../metadata/model/ShowAINodesStatement.java | 7 + .../metadata/region/ExtendRegionStatement.java | 9 +- .../metadata/region/MigrateRegionStatement.java | 9 +- .../region/ReconstructRegionStatement.java | 9 +- .../metadata/region/RemoveRegionStatement.java | 9 +- .../plan/statement/sys/KillQueryStatement.java | 9 +- .../plan/statement/sys/ShowQueriesStatement.java | 9 +- .../statement/sys/TestConnectionStatement.java | 3 +- .../unary/scalar/FormatColumnTransformer.java | 4 +- .../metric/SchemaEngineCachedMetric.java | 5 +- .../rescon/ISchemaRegionStatistics.java | 2 + .../rescon/MemSchemaEngineStatistics.java | 41 +- .../rescon/MemSchemaRegionStatistics.java | 16 + .../attribute/DeviceAttributeStore.java | 48 +- .../attribute/IDeviceAttributeStore.java | 16 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 36 +- .../ReleaseFlushStrategySizeBasedImpl.java | 6 +- .../read/resp/info/impl/ShowDevicesResult.java | 2 +- .../db/schemaengine/table/DataNodeTableCache.java | 28 +- .../template/TemplateInternalRPCUpdateType.java | 5 +- .../java/org/apache/iotdb/db/service/DataNode.java | 64 +- .../iotdb/db/service/ExternalRPCService.java | 2 +- ...viceMBean.java => ExternalRPCServiceMBean.java} | 2 +- .../db/service/metrics/CompactionMetrics.java | 2 +- .../iotdb/db/service/metrics/WritingMetrics.java | 2 +- .../metrics/memory/ConsensusMemoryMetrics.java | 50 +- .../metrics/memory/GlobalMemoryMetrics.java | 82 +- .../metrics/memory/OffHeapMemoryMetrics.java | 102 +++ .../metrics/memory/QueryEngineMemoryMetrics.java | 281 +++--- .../metrics/memory/SchemaEngineMemoryMetrics.java | 159 ++-- .../metrics/memory/StorageEngineMemoryMetrics.java | 361 +++++--- .../metrics/memory/StreamEngineMemoryMetrics.java | 50 +- .../iotdb/db/storageengine/StorageEngine.java | 19 +- .../db/storageengine/buffer/BloomFilterCache.java | 27 +- .../iotdb/db/storageengine/buffer/ChunkCache.java | 27 +- .../buffer/TimeSeriesMetadataCache.java | 47 +- .../db/storageengine/dataregion/DataRegion.java | 11 +- .../storageengine/dataregion/DataRegionInfo.java | 6 +- .../execute/task/CrossSpaceCompactionTask.java | 2 +- .../execute/task/InnerSpaceCompactionTask.java | 2 +- .../task/InsertionCrossSpaceCompactionTask.java | 10 +- .../task/RepairUnsortedFileCompactionTask.java | 2 +- .../compaction/execute/utils/CompactionUtils.java | 12 +- .../execute/utils/MultiTsFileDeviceIterator.java | 5 +- .../dataregion/memtable/AbstractMemTable.java | 82 +- .../memtable/AlignedReadOnlyMemChunk.java | 318 +++---- .../memtable/AlignedWritableMemChunk.java | 147 ++-- .../memtable/AlignedWritableMemChunkGroup.java | 14 +- .../dataregion/memtable/IMemTable.java | 2 +- .../memtable/IWritableMemChunkGroup.java | 2 +- .../dataregion/memtable/ReadOnlyMemChunk.java | 16 +- .../dataregion/memtable/TsFileProcessor.java | 43 +- .../dataregion/memtable/WritableMemChunk.java | 132 +-- .../dataregion/memtable/WritableMemChunkGroup.java | 18 +- .../read/reader/chunk/MemAlignedChunkReader.java | 188 ++-- .../read/reader/chunk/MemChunkReader.java | 4 +- .../read/reader/chunk/MemPageReader.java | 6 +- .../dataregion/tsfile/TsFileResource.java | 35 +- .../dataregion/tsfile/TsFileResourceBlockType.java | 3 + .../tsfile/timeindex/ArrayDeviceTimeIndex.java | 3 +- .../dataregion/tsfile/timeindex/FileTimeIndex.java | 2 +- .../dataregion/wal/buffer/IWALBuffer.java | 10 + .../dataregion/wal/buffer/WALBuffer.java | 13 + .../storageengine/dataregion/wal/node/WALNode.java | 8 +- .../wal/utils/MemoryControlledWALEntryQueue.java | 9 +- .../dataregion/wal/utils/WALEntryHandler.java | 4 +- .../dataregion/wal/utils/WALInsertNodeCache.java | 16 +- .../load/config/LoadTsFileConfigurator.java | 24 - ...leStatementDataTypeConvertExecutionVisitor.java | 1 + .../memory/LoadTsFileAnalyzeSchemaMemoryBlock.java | 2 +- .../memory/LoadTsFileDataCacheMemoryBlock.java | 4 +- .../load/memory/LoadTsFileMemoryManager.java | 29 +- .../rescon/memory/PrimitiveArrayManager.java | 16 +- .../db/storageengine/rescon/memory/SystemInfo.java | 187 ++-- .../rescon/memory/TimePartitionManager.java | 31 +- .../rescon/memory/TsFileResourceManager.java | 40 +- .../broker/SubscriptionPrefetchingQueue.java | 10 +- .../broker/SubscriptionPrefetchingTabletQueue.java | 12 +- .../broker/SubscriptionPrefetchingTsFileQueue.java | 12 +- .../db/subscription/event/SubscriptionEvent.java | 4 +- .../event/batch/SubscriptionPipeEventBatch.java | 2 +- .../event/batch/SubscriptionPipeEventBatches.java | 2 +- .../batch/SubscriptionPipeTabletEventBatch.java | 50 +- .../SubscriptionPipeTabletIterationSnapshot.java | 44 +- .../batch/SubscriptionPipeTsFileEventBatch.java | 2 +- .../event/cache/SubscriptionPollResponseCache.java | 11 + .../event/pipe/SubscriptionPipeEmptyEvent.java | 2 +- .../event/pipe/SubscriptionPipeEvents.java | 2 +- .../pipe/SubscriptionPipeTabletBatchEvents.java | 11 +- .../pipe/SubscriptionPipeTsFileBatchEvents.java | 4 +- .../pipe/SubscriptionPipeTsFilePlainEvent.java | 2 +- .../SubscriptionConnectorSubtaskManager.java | 2 +- .../db/tools/schema/SRStatementGenerator.java | 25 +- .../apache/iotdb/db/utils/ErrorHandlingUtils.java | 3 +- .../db/utils/datastructure/AlignedTVList.java | 116 +-- .../iotdb/db/utils/datastructure/BinaryTVList.java | 40 +- .../db/utils/datastructure/BooleanTVList.java | 40 +- .../iotdb/db/utils/datastructure/DoubleTVList.java | 40 +- .../datastructure/FixedPriorityBlockingQueue.java | 2 +- .../iotdb/db/utils/datastructure/FloatTVList.java | 40 +- .../iotdb/db/utils/datastructure/IntTVList.java | 40 +- .../iotdb/db/utils/datastructure/LongTVList.java | 40 +- .../MergeSortAlignedTVListIterator.java | 174 ++-- ...tIterator.java => MergeSortTVListIterator.java} | 90 +- .../utils/datastructure/PageColumnAccessInfo.java | 52 -- .../iotdb/db/utils/datastructure/TVList.java | 74 +- .../db/pipe/consensus/DeletionResourceTest.java | 7 +- .../pipe/event/TsFileInsertionEventParserTest.java | 5 +- .../db/pipe/extractor/PipeRealtimeExtractTest.java | 5 + .../pattern/CachedSchemaPatternMatcherTest.java | 53 +- .../db/protocol/mqtt/PayloadFormatManagerTest.java | 2 +- .../execution/exchange/LocalSinkChannelTest.java | 7 +- .../exchange/MPPDataExchangeManagerTest.java | 7 +- .../execution/exchange/SharedTsBlockQueueTest.java | 4 +- .../execution/exchange/ShuffleSinkHandleTest.java | 4 +- .../execution/exchange/SinkChannelTest.java | 7 +- .../execution/exchange/SourceHandleTest.java | 15 +- .../execution/memory/MemoryPoolTest.java | 6 +- .../execution/operator/OperatorMemoryTest.java | 2 - .../process/tvf/TableFunctionOperatorTest.java | 3 - .../analyzer/AggregationCornerCaseTest.java | 323 +++++++ .../plan/relational/analyzer/AggregationTest.java | 305 +++---- .../plan/relational/analyzer/AnalyzerTest.java | 146 ++-- .../plan/relational/analyzer/DistinctTest.java | 48 +- .../plan/relational/analyzer/JoinTest.java | 94 +- .../analyzer/LimitOffsetPushDownTest.java | 183 ++-- .../analyzer/MockTableModelDataPartition.java | 63 +- .../plan/relational/analyzer/SortTest.java | 288 +++---- .../plan/relational/analyzer/SubQueryTest.java | 44 +- .../plan/relational/analyzer/TSBSMetadata.java | 22 +- .../relational/analyzer/TableFunctionTest.java | 115 ++- .../plan/relational/analyzer/TestMatadata.java | 12 +- .../plan/relational/analyzer/TestUtils.java | 26 + .../plan/relational/analyzer/TreeViewTest.java | 72 +- .../fetcher/cache/TableDeviceSchemaCacheTest.java | 21 +- .../relational/planner/CorrelatedSubqueryTest.java | 153 ++++ .../plan/relational/planner/ExampleTest.java | 34 +- .../plan/relational/planner/PlanTester.java | 9 +- ...ueryTest.java => UncorrelatedSubqueryTest.java} | 268 ++++-- ...NodeMatcher.java => AssignUniqueIdMatcher.java} | 27 +- .../planner/assertions/ExchangeNodeMatcher.java | 8 +- .../planner/assertions/PlanMatchPattern.java | 15 +- .../planner/assertions/PlanMatchingVisitor.java | 12 +- .../relational/sql/parser/AuthorStatementTest.java | 5 - .../compaction/CompactionWithMinTimestampTest.java | 136 +++ .../compaction/CompactionWorkerTest.java | 16 +- .../FastCrossCompactionPerformerTest.java | 5 +- .../cross/CrossSpaceCompactionSelectorTest.java | 5 +- .../CompactionTableModelTestFileWriter.java | 14 + .../compaction/utils/CompactionClearUtils.java | 2 +- .../compaction/utils/CompactionTaskQueueTest.java | 13 +- .../dataregion/memtable/TsFileProcessorTest.java | 36 +- .../read/reader/chunk/MemChunkLoaderTest.java | 26 +- .../rescon/memory/ResourceManagerTest.java | 6 +- .../rescon/memory/TimePartitionManagerTest.java | 8 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 5 +- .../datastructure/PrimitiveArrayManagerTest.java | 8 +- .../conf/iotdb-system.properties.template | 25 +- .../commons/auth/authorizer/BasicAuthorizer.java | 35 +- .../iotdb/commons/auth/authorizer/IAuthorizer.java | 10 +- .../iotdb/commons/auth/entity/PrivilegeType.java | 2 +- .../org/apache/iotdb/commons/auth/entity/Role.java | 4 + .../iotdb/commons/auth/user/BasicUserManager.java | 12 +- .../async/AsyncPipeDataTransferServiceClient.java | 14 + .../iotdb/commons/concurrent/ThreadName.java | 1 + .../apache/iotdb/commons/conf/CommonConfig.java | 78 +- .../iotdb/commons/conf/CommonDescriptor.java | 48 +- .../commons/memory/AtomicLongMemoryBlock.java | 152 ++++ .../apache/iotdb/commons/memory/IMemoryBlock.java | 135 +++ .../iotdb/commons/memory/MemoryBlockType.java} | 17 +- .../RetryUtils.java => memory/MemoryConfig.java} | 35 +- .../iotdb/commons/memory/MemoryException.java | 64 ++ .../apache/iotdb/commons/memory/MemoryManager.java | 523 ++++++++++++ .../commons/partition/DataPartitionTable.java | 10 +- .../commons/partition/SeriesPartitionTable.java | 20 +- .../iotdb/commons/path/ExtendedPartialPath.java | 4 + .../agent/plugin/builtin/BuiltinPipePlugin.java | 5 + .../builtin/connector/opcda/OpcDaConnector.java} | 23 +- .../commons/pipe/agent/task/PipeTaskAgent.java | 2 +- .../task/subtask/PipeAbstractConnectorSubtask.java | 4 +- .../agent/task/subtask/PipeReportableSubtask.java | 13 +- .../iotdb/commons/pipe/config/PipeConfig.java | 27 +- .../config/constant/PipeConnectorConstant.java | 12 +- .../config/constant/PipeExtractorConstant.java | 12 + .../connector/protocol/IoTDBAirGapConnector.java | 21 +- .../queue/ConcurrentIterableLinkedQueue.java | 30 + .../iotdb/commons/pipe/event/EnrichedEvent.java | 45 +- .../commons/pipe/event/PipeSnapshotEvent.java | 4 + .../commons/pipe/event/PipeWritePlanEvent.java | 4 + .../commons/pipe/event/ProgressReportEvent.java | 25 +- .../commons/pipe/extractor/IoTDBExtractor.java | 48 +- .../extractor/IoTDBNonDataRegionExtractor.java | 74 +- .../pipe/receiver/PipeReceiverStatusHandler.java | 8 + .../schema/column/ColumnHeaderConstant.java | 7 +- .../table/AlterOrDropTableOperationType.java | 8 +- .../commons/schema/table/InformationSchema.java | 6 + .../apache/iotdb/commons/schema/table/TsTable.java | 3 +- .../viewExpression/unary/LikeViewExpression.java | 53 +- .../apache/iotdb/commons/service/ServiceType.java | 1 + .../iotdb/commons/service/metric/enums/Metric.java | 1 + .../subscription/config/SubscriptionConfig.java | 21 + .../commons/subscription/meta/topic/TopicMeta.java | 3 + .../iotdb/commons/udf/access/RecordIterator.java | 146 ++-- .../relational/TableBuiltinTableFunction.java | 3 - .../builtin/relational/tvf/HOPTableFunction.java | 7 +- .../udf/builtin/relational/tvf/RepeatExample.java | 125 --- .../org/apache/iotdb/commons/utils/RetryUtils.java | 8 + .../iotdb/commons/memory/MemoryBlockTest.java | 67 ++ .../iotdb/commons/memory/MemoryManagerTest.java | 171 ++++ .../ConcurrentIterableLinkedQueueTest.java | 1 + .../db/relational/grammar/sql/RelationalSql.g4 | 32 +- .../thrift-commons/src/main/thrift/common.thrift | 1 + .../src/main/thrift/confignode.thrift | 1 + .../src/main/thrift/pipeconsensus.thrift | 5 +- pom.xml | 8 +- 649 files changed, 21621 insertions(+), 7775 deletions(-) diff --cc iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinTableFunction.java index a529ef3d6d5,bcd35af658c..1e6208adcd7 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinTableFunction.java @@@ -19,11 -19,6 +19,10 @@@ package org.apache.iotdb.commons.udf.builtin.relational; +import org.apache.iotdb.commons.udf.builtin.relational.tvf.HOPTableFunction; +import org.apache.iotdb.commons.udf.builtin.relational.tvf.NumberTableFunction; - import org.apache.iotdb.commons.udf.builtin.relational.tvf.RepeatExample; +import org.apache.iotdb.commons.udf.builtin.relational.tvf.SessionTableFunction; +import org.apache.iotdb.commons.udf.builtin.relational.tvf.VariationTableFunction; import org.apache.iotdb.udf.api.relational.TableFunction; import java.util.Arrays; @@@ -65,16 -55,6 +64,14 @@@ public enum TableBuiltinTableFunction public static TableFunction getBuiltinTableFunction(String functionName) { switch (functionName.toLowerCase()) { + case "hop": + return new HOPTableFunction(); + case "session": + return new SessionTableFunction(); + case "variation": + return new VariationTableFunction(); + case "number": + return new NumberTableFunction(); - case "repeat": - return new RepeatExample(); default: throw new UnsupportedOperationException("Unsupported table function: " + functionName); } diff --cc iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/HOPTableFunction.java index cf34cbda4a8,10cc00588b4..8e33c3aedac --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/HOPTableFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/tvf/HOPTableFunction.java @@@ -46,9 -46,9 +46,9 @@@ public class HOPTableFunction implement private static final String DATA_PARAMETER_NAME = "DATA"; private static final String TIMECOL_PARAMETER_NAME = "TIMECOL"; - private static final String SLIDE_PARAMETER_NAME = "SLIDE"; private static final String SIZE_PARAMETER_NAME = "SIZE"; + private static final String SLIDE_PARAMETER_NAME = "SLIDE"; - private static final String START_PARAMETER_NAME = "START"; + private static final String ORIGIN_PARAMETER_NAME = "ORIGIN"; @Override public List<ParameterSpecification> getArgumentsSpecifications() { @@@ -61,11 -61,12 +61,12 @@@ ScalarParameterSpecification.builder() .name(TIMECOL_PARAMETER_NAME) .type(Type.STRING) + .defaultValue("time") .build(), - ScalarParameterSpecification.builder().name(SLIDE_PARAMETER_NAME).type(Type.INT64).build(), ScalarParameterSpecification.builder().name(SIZE_PARAMETER_NAME).type(Type.INT64).build(), + ScalarParameterSpecification.builder().name(SLIDE_PARAMETER_NAME).type(Type.INT64).build(), ScalarParameterSpecification.builder() - .name(START_PARAMETER_NAME) + .name(ORIGIN_PARAMETER_NAME) .type(Type.TIMESTAMP) .defaultValue(0L) .build());
