This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch agg_table_scan in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a89d2da67760acb21fd4ad338434ee4495ab811a Merge: 26f00875c18 7ceddaee37d Author: Beyyes <[email protected]> AuthorDate: Mon Sep 23 14:52:49 2024 +0800 Merge branch 'master' into agg_table_scan .../org/apache/iotdb/db/it/utils/TestUtils.java | 1 + .../pipe/it/autocreate/IoTDBPipeProtocolIT.java | 2 +- .../db/it/IoTDBMultiIDsWithAttributesTableIT.java | 119 ++++++ .../write/pipe/plugin/CreatePipePluginPlan.java | 2 +- .../pipe/runtime/PipeHandleMetaChangePlan.java | 2 +- .../request/write/pipe/task/AlterPipePlanV2.java | 4 +- .../request/write/pipe/task/CreatePipePlanV2.java | 4 +- .../write/pipe/task/SetPipeStatusPlanV2.java | 2 +- .../response/pipe/plugin/PipePluginTableResp.java | 2 +- .../response/pipe/task/PipeTableResp.java | 10 +- .../iotdb/confignode/manager/ProcedureManager.java | 2 +- .../agent/plugin/PipeConfigNodePluginAgent.java | 8 +- .../PipeConfigRegionConnectorConstructor.java | 6 +- .../PipeConfigRegionExtractorConstructor.java | 6 +- .../PipeConfigRegionProcessorConstructor.java | 6 +- .../agent/runtime/PipeConfigNodeRuntimeAgent.java | 2 +- .../task}/PipeConfigNodeSubtask.java | 10 +- .../task}/PipeConfigNodeSubtaskExecutor.java | 4 +- .../pipe/{ => agent}/task/PipeConfigNodeTask.java | 4 +- .../pipe/agent/task/PipeConfigNodeTaskAgent.java | 11 +- .../task/PipeConfigNodeTaskBuilder.java | 12 +- .../{ => agent}/task/PipeConfigNodeTaskStage.java | 8 +- .../coordinator/plugin/PipePluginCoordinator.java | 2 +- .../runtime/heartbeat/PipeHeartbeat.java | 4 +- .../runtime/heartbeat/PipeHeartbeatParser.java | 12 +- .../pipe/event/PipeConfigRegionSnapshotEvent.java | 4 +- .../pipe/event/PipeConfigRegionWritePlanEvent.java | 4 +- .../pipe/extractor/IoTDBConfigRegionExtractor.java | 2 +- .../PipeConfigPhysicalPlanPatternParseVisitor.java | 2 +- .../metric/PipeConfigNodeRemainingTimeMetrics.java | 2 +- .../PipeConfigNodeRemainingTimeOperator.java | 2 +- .../metric/PipeConfigRegionConnectorMetrics.java | 2 +- .../pipe/metric/PipeTemporaryMetaMetrics.java | 4 +- .../receiver/protocol/IoTDBConfigNodeReceiver.java | 4 +- .../PipeConfigNodeCopiedFileDirStartupCleaner.java | 2 +- .../resource/PipeConfigNodeResourceManager.java | 3 +- .../PipeConfigNodeSnapshotResourceManager.java | 4 +- .../confignode/persistence/pipe/PipeInfo.java | 4 +- .../persistence/pipe/PipePluginInfo.java | 12 +- .../confignode/persistence/pipe/PipeTaskInfo.java | 20 +- .../procedure/env/ConfigNodeProcedureEnv.java | 2 +- .../impl/pipe/AbstractOperatePipeProcedureV2.java | 2 +- .../pipe/plugin/CreatePipePluginProcedure.java | 2 +- .../runtime/PipeHandleMetaChangeProcedure.java | 2 +- .../impl/pipe/runtime/PipeMetaSyncProcedure.java | 2 +- .../impl/pipe/task/AlterPipeProcedureV2.java | 10 +- .../impl/pipe/task/CreatePipeProcedureV2.java | 10 +- .../impl/pipe/task/StartPipeProcedureV2.java | 2 +- .../impl/pipe/task/StopPipeProcedureV2.java | 2 +- ...bstractOperateSubscriptionAndPipeProcedure.java | 2 +- .../subscription/CreateSubscriptionProcedure.java | 2 +- .../subscription/DropSubscriptionProcedure.java | 2 +- .../request/ConfigPhysicalPlanSerDeTest.java | 15 +- .../response/pipe/PipePluginTableRespTest.java | 6 +- .../consensus/response/pipe/PipeTableRespTest.java | 8 +- .../PipeConfigNodeSubtaskExecutorTest.java | 12 +- .../connector/IoTDBConfigRegionConnectorTest.java | 2 +- ...eConfigPhysicalPlanPatternParseVisitorTest.java | 2 +- .../iotdb/confignode/persistence/PipeInfoTest.java | 10 +- .../pipe/plugin/CreatePipePluginProcedureTest.java | 2 +- .../consensus/config/PipeConsensusConfig.java | 2 +- .../apache/iotdb/consensus/pipe/PipeConsensus.java | 2 +- .../consensus/pipe/PipeConsensusServerImpl.java | 2 +- .../pipe/consensuspipe/ConsensusPipeManager.java | 2 +- .../pipe/consensuspipe/ConsensusPipeName.java | 2 +- .../pipe/consensuspipe/ConsensusPipeSelector.java | 2 +- .../db/consensus/DataRegionConsensusImpl.java | 2 +- .../pipe/agent/plugin/PipeDataNodePluginAgent.java | 10 +- .../PipeDataRegionConnectorConstructor.java | 8 +- .../PipeDataRegionExtractorConstructor.java | 8 +- .../dataregion/PipeDataRegionPluginAgent.java | 10 +- .../PipeDataRegionProcessorConstructor.java | 10 +- .../PipeSchemaRegionConnectorConstructor.java | 6 +- .../PipeSchemaRegionExtractorConstructor.java | 6 +- .../schemaregion/PipeSchemaRegionPluginAgent.java | 8 +- .../PipeSchemaRegionProcessorConstructor.java | 6 +- .../db/pipe/agent/runtime/PipeAgentLauncher.java | 8 +- .../agent/runtime/PipeDataNodeRuntimeAgent.java | 3 +- .../runtime/PipeSchemaRegionListenerManager.java | 2 +- .../runtime}/SimpleProgressIndexAssigner.java | 2 +- .../db/pipe/{ => agent}/task/PipeDataNodeTask.java | 6 +- .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 17 +- .../task/builder/PipeDataNodeBuilder.java | 12 +- .../task/builder/PipeDataNodeTaskBuilder.java | 22 +- .../{ => agent}/task/connection/EnrichedDeque.java | 2 +- .../task/connection/PipeEventCollector.java | 8 +- .../execution/PipeConnectorSubtaskExecutor.java | 4 +- .../execution/PipeProcessorSubtaskExecutor.java | 4 +- .../execution/PipeSubtaskExecutorManager.java | 4 +- .../task/stage/PipeTaskConnectorStage.java | 10 +- .../task/stage/PipeTaskExtractorStage.java | 8 +- .../task/stage/PipeTaskProcessorStage.java | 18 +- .../subtask/connector/PipeConnectorSubtask.java | 6 +- .../connector/PipeConnectorSubtaskLifeCycle.java | 6 +- .../connector/PipeConnectorSubtaskManager.java | 10 +- .../PipeRealtimePriorityBlockingQueue.java | 8 +- .../subtask/processor/PipeProcessorSubtask.java | 12 +- .../processor/PipeProcessorSubtaskWorker.java | 2 +- .../PipeProcessorSubtaskWorkerManager.java | 2 +- .../pipeconsensus/PipeConsensusAsyncConnector.java | 2 +- .../async/IoTDBDataRegionAsyncConnector.java | 2 +- .../consensus/PipeConsensusSubtaskExecutor.java | 2 +- .../db/pipe/event/UserDefinedEnrichedEvent.java | 4 +- .../event/common/heartbeat/PipeHeartbeatEvent.java | 6 +- .../db/pipe/event/common/row/PipeRowCollector.java | 2 +- .../schema/PipeSchemaRegionSnapshotEvent.java | 4 +- .../schema/PipeSchemaRegionWritePlanEvent.java | 4 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 4 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 4 +- .../tablet/TabletInsertionDataContainer.java | 4 +- .../event/common/terminate/PipeTerminateEvent.java | 6 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 4 +- .../common/tsfile/TsFileInsertionPointCounter.java | 2 +- .../container/TsFileInsertionDataContainer.java | 4 +- .../TsFileInsertionDataContainerProvider.java | 6 +- .../query/TsFileInsertionQueryDataContainer.java | 4 +- .../scan/TsFileInsertionScanDataContainer.java | 4 +- .../db/pipe/event/realtime/PipeRealtimeEvent.java | 4 +- .../dataregion/DataRegionListeningFilter.java | 2 +- .../dataregion/IoTDBDataRegionExtractor.java | 4 +- .../PipeHistoricalDataRegionTsFileExtractor.java | 4 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 6 +- .../realtime/assigner/PipeDataRegionAssigner.java | 4 +- .../matcher}/CachedSchemaPatternMatcher.java | 4 +- .../realtime/matcher}/PipeDataRegionMatcher.java | 2 +- .../schemaregion/PipePlanPatternParseVisitor.java | 2 +- .../PipeDataNodeRemainingEventAndTimeMetrics.java | 6 +- .../PipeDataNodeRemainingEventAndTimeOperator.java | 4 +- .../metric/PipeDataRegionConnectorMetrics.java | 2 +- .../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 2 +- .../metric/PipeSchemaRegionConnectorMetrics.java | 2 +- .../processor/aggregate/AggregateProcessor.java | 2 +- .../twostage/plugin/TwoStageCountProcessor.java | 2 +- .../protocol/thrift/IoTDBDataNodeReceiver.java | 4 +- ...peStatementDataTypeConvertExecutionVisitor.java | 2 +- .../visitor/PipeStatementPatternParseVisitor.java | 2 +- ...aNodeHardlinkOrCopiedFileDirStartupCleaner.java | 2 +- .../pipe/resource/PipeDataNodeResourceManager.java | 2 +- .../PipeDataNodeSnapshotResourceManager.java | 2 +- .../impl/DataNodeInternalRPCServiceImpl.java | 4 +- .../fragment/FragmentInstanceContext.java | 8 +- .../source/relational/InnerJoinOperator.java | 407 +++++++++++++++++++++ .../relational/ColumnTransformerBuilder.java | 10 +- .../db/queryengine/plan/analyze/TypeProvider.java | 4 + .../config/executor/ClusterConfigTaskExecutor.java | 6 +- .../config/metadata/ShowPipePluginsTask.java | 4 +- .../plan/planner/OperatorTreeGenerator.java | 2 +- .../plan/planner/TableOperatorGenerator.java | 50 ++- .../plan/planner/plan/node/PlanGraphPrinter.java | 3 + .../plan/planner/plan/node/PlanNodeType.java | 4 + .../planner/plan/node/process/ExchangeNode.java | 12 + .../plan/relational/analyzer/Analysis.java | 4 + .../plan/relational/planner/RelationPlanner.java | 10 +- .../planner/distribute/AddExchangeNodes.java | 1 + .../distribute/TableDistributedPlanner.java | 11 +- ...AddTableScanColumnsToTypeProviderOptimizer.java | 66 ++++ .../planner/iterative/rule/InlineProjections.java | 11 +- .../planner/iterative/rule/MergeLimitWithSort.java | 24 +- .../iterative/rule/PruneJoinChildrenColumns.java | 77 ++++ .../iterative/rule/PruneTableScanColumns.java | 18 +- .../relational/planner/iterative/rule/Util.java | 21 +- .../plan/relational/planner/node/JoinNode.java | 90 ++++- .../optimizations/DistributedOptimizeFactory.java | 56 +-- .../optimizations/LogicalOptimizeFactory.java | 163 ++++----- .../optimizations/PushPredicateIntoTableScan.java | 155 ++++---- .../planner/optimizations/SortElimination.java | 21 +- .../java/org/apache/iotdb/db/service/DataNode.java | 2 +- .../db/service/ResourcesInformationHolder.java | 2 +- .../broker/SubscriptionBlockingPendingQueue.java | 2 +- .../db/subscription/broker/SubscriptionBroker.java | 2 +- .../TsFileDeduplicationBlockingPendingQueue.java | 2 +- .../execution}/SubscriptionSubtaskExecutor.java | 4 +- .../task/stage/SubscriptionTaskConnectorStage.java | 6 +- .../task/subtask/SubscriptionConnectorSubtask.java | 4 +- .../SubscriptionConnectorSubtaskLifeCycle.java | 8 +- .../SubscriptionConnectorSubtaskManager.java | 16 +- .../agent/plugin/PipeDataNodePluginAgentTest.java | 8 +- .../task}/PipeConnectorSubtaskExecutorTest.java | 7 +- .../task}/PipeProcessorSubtaskExecutorTest.java | 9 +- .../task}/PipeSubtaskExecutorTest.java | 6 +- .../iotdb/db/pipe/connector/PipeConnectorTest.java | 2 +- .../PipeStatementPatternParseVisitorTest.java | 2 +- .../pipe/event/PipeTabletInsertionEventTest.java | 2 +- .../event/TsFileInsertionDataContainerTest.java | 6 +- .../extractor/PipePlanPatternParseVisitorTest.java | 2 +- .../pattern/CachedSchemaPatternMatcherTest.java | 5 +- .../db/pipe/pattern/IoTDBPipePatternTest.java | 2 +- .../db/pipe/pattern/PrefixPipePatternTest.java | 2 +- .../plan/relational/analyzer/JoinTest.java | 92 +++-- .../PipeRuntimeConnectorCriticalException.java | 2 +- .../pipe/PipeRuntimeCriticalException.java | 2 +- .../exception/pipe/PipeRuntimeExceptionType.java | 2 +- .../pipe/PipeRuntimeNonCriticalException.java | 2 +- .../PipeRuntimeOutOfMemoryCriticalException.java | 2 +- .../commons/pipe/agent/plugin/PipePluginAgent.java | 5 +- .../plugin/builtin/BuiltinPipePlugin.java | 50 +-- .../builtin/connector/PlaceholderConnector.java | 2 +- .../connector/donothing/DoNothingConnector.java | 2 +- .../iotdb/airgap/IoTDBAirGapConnector.java | 4 +- .../consensus/PipeConsensusAsyncConnector.java | 4 +- .../iotdb/thrift/IoTDBLegacyPipeConnector.java | 4 +- .../iotdb/thrift/IoTDBThriftAsyncConnector.java | 2 +- .../iotdb/thrift/IoTDBThriftConnector.java | 4 +- .../iotdb/thrift/IoTDBThriftSslConnector.java | 4 +- .../iotdb/thrift/IoTDBThriftSyncConnector.java | 2 +- .../builtin/connector/opcua/OpcUaConnector.java | 4 +- .../connector/websocket/WebSocketConnector.java | 4 +- .../connector/writeback/WriteBackConnector.java | 4 +- .../extractor/donothing/DoNothingExtractor.java | 2 +- .../builtin/extractor/iotdb/IoTDBExtractor.java | 2 +- .../builtin/processor/PlaceHolderProcessor.java | 2 +- .../processor/aggregate/AggregateProcessor.java | 4 +- .../aggregate/StandardStatisticsProcessor.java | 4 +- .../aggregate/TumblingWindowingProcessor.java | 4 +- .../processor/donothing/DoNothingProcessor.java | 2 +- .../ChangingValueSamplingProcessor.java | 4 +- .../SwingingDoorTrendingSamplingProcessor.java | 4 +- .../TumblingTimeSamplingProcessor.java | 4 +- .../pipeconsensus/PipeConsensusProcessor.java | 4 +- .../throwing/ThrowingExceptionProcessor.java | 2 +- .../processor/twostage/TwoStageCountProcessor.java | 4 +- .../PipeConnectorConstructor.java | 6 +- .../PipeExtractorConstructor.java | 6 +- .../{ => constructor}/PipePluginConstructor.java | 8 +- .../PipeProcessorConstructor.java | 6 +- .../meta/ConfigNodePipePluginMetaKeeper.java | 2 +- .../plugin/meta/DataNodePipePluginMetaKeeper.java | 2 +- .../{ => agent}/plugin/meta/PipePluginMeta.java | 2 +- .../plugin/meta/PipePluginMetaKeeper.java | 4 +- .../plugin/service/PipePluginClassLoader.java | 2 +- .../service/PipePluginClassLoaderManager.java | 2 +- .../service/PipePluginExecutableManager.java | 4 +- .../commons/pipe/{ => agent}/task/PipeTask.java | 2 +- .../commons/pipe/agent/task/PipeTaskAgent.java | 14 +- .../pipe/{ => agent}/task/PipeTaskManager.java | 4 +- .../task/connection/BlockingPendingQueue.java | 2 +- .../connection/BoundedBlockingPendingQueue.java | 2 +- .../task/connection}/EventSupplier.java | 2 +- .../connection/UnboundedBlockingPendingQueue.java | 2 +- .../task/execution}/PipeSubtaskExecutor.java | 5 +- .../task/execution}/PipeSubtaskScheduler.java | 3 +- .../pipe/{ => agent}/task/meta/PipeMeta.java | 2 +- .../pipe/{ => agent}/task/meta/PipeMetaKeeper.java | 2 +- .../{ => agent}/task/meta/PipeRuntimeMeta.java | 2 +- .../task/meta/PipeRuntimeMetaVersion.java | 2 +- .../pipe/{ => agent}/task/meta/PipeStaticMeta.java | 2 +- .../pipe/{ => agent}/task/meta/PipeStatus.java | 2 +- .../pipe/{ => agent}/task/meta/PipeTaskMeta.java | 2 +- .../{ => agent}/task/meta/PipeTemporaryMeta.java | 2 +- .../pipe/{ => agent}/task/meta/PipeType.java | 2 +- .../{ => agent/task}/progress/CommitterKey.java | 2 +- .../task}/progress/PipeEventCommitManager.java | 2 +- .../task}/progress/PipeEventCommitter.java | 2 +- .../pipe/{ => agent}/task/stage/PipeTaskStage.java | 4 +- .../task/subtask/PipeAbstractConnectorSubtask.java | 4 +- .../task/subtask/PipeReportableSubtask.java | 2 +- .../pipe/{ => agent}/task/subtask/PipeSubtask.java | 4 +- .../env/PipeTaskExtractorRuntimeEnvironment.java | 2 +- .../env/PipeTaskProcessorRuntimeEnvironment.java | 2 +- .../pipe/connector/protocol/IoTDBConnector.java | 2 +- .../connector/protocol/IoTDBSslSyncConnector.java | 6 +- .../pattern/IoTDBPipePattern.java | 2 +- .../{ => datastructure}/pattern/PipePattern.java | 2 +- .../pattern/PrefixPipePattern.java | 2 +- .../listening/AbstractPipeListeningQueue.java | 2 +- .../iotdb/commons/pipe/event/EnrichedEvent.java | 8 +- .../commons/pipe/event/PipeSnapshotEvent.java | 6 +- .../commons/pipe/event/PipeWritePlanEvent.java | 4 +- .../commons/pipe/event/ProgressReportEvent.java | 4 +- .../commons/pipe/extractor/IoTDBExtractor.java | 2 +- .../extractor/IoTDBNonDataRegionExtractor.java | 4 +- .../pipe/metric/PipeEventCommitMetrics.java | 2 +- .../PipeReceiverStatusHandler.java | 4 +- .../PipeSnapshotResourceManager.java | 2 +- .../exception/pipe/PipeRuntimeExceptionTest.java | 2 +- .../pipe/plugin/builtin/BuiltinPipePluginTest.java | 10 +- .../pipe/plugin/meta/PipePluginMetaTest.java | 4 +- .../pipe/task/{meta => }/PipeMetaDeSerTest.java | 7 +- 278 files changed, 1710 insertions(+), 878 deletions(-)
