This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch table_disk_usage_statistics_with_cache in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a71f0d799380a20a4b3d7aeb990a330f50ff905c Merge: 640f99811b1 8dc3a984ef7 Author: shuwenwei <[email protected]> AuthorDate: Thu Jan 8 16:23:48 2026 +0800 Merge branch 'iotdb_master' into table_disk_usage_statistics .github/workflows/vulnerability-check.yml | 42 +- LICENSE | 10 + code-coverage/pom.xml | 2 +- distribution/pom.xml | 12 +- example/client-cpp-example/pom.xml | 2 +- example/jdbc/pom.xml | 2 +- example/mqtt-customize/pom.xml | 2 +- example/mqtt/pom.xml | 2 +- example/pipe-count-point-processor/pom.xml | 2 +- example/pipe-opc-ua-sink/pom.xml | 2 +- example/pom.xml | 2 +- example/rest-java-example/pom.xml | 2 +- example/schema/pom.xml | 2 +- example/session/pom.xml | 2 +- .../apache/iotdb/TableModelSessionPoolExample.java | 2 +- example/trigger/pom.xml | 2 +- example/udf/pom.xml | 2 +- integration-test/pom.xml | 33 +- .../it/env/cluster/config/MppCommonConfig.java | 18 + .../it/env/cluster/config/MppDataNodeConfig.java | 6 + .../env/cluster/config/MppSharedCommonConfig.java | 21 + .../iotdb/it/env/cluster/node/AINodeWrapper.java | 2 +- .../iotdb/it/env/cluster/node/DataNodeWrapper.java | 4 + .../it/env/remote/config/RemoteCommonConfig.java | 15 + .../it/env/remote/config/RemoteDataNodeConfig.java | 5 + .../org/apache/iotdb/itbase/env/CommonConfig.java | 6 + .../apache/iotdb/itbase/env/DataNodeConfig.java | 2 + .../iotdb/itbase/runtime/ClusterTestResultSet.java | 16 +- .../iotdb/ainode/it/AINodeCallInferenceIT.java | 24 +- .../iotdb/ainode/it/AINodeClusterConfigIT.java | 3 - .../ainode/it/AINodeConcurrentForecastIT.java | 12 +- .../apache/iotdb/ainode/it/AINodeForecastIT.java | 114 +- .../iotdb/ainode/it/AINodeModelManageIT.java | 62 +- .../apache/iotdb/ainode/utils/AINodeTestUtils.java | 65 +- .../org/apache/iotdb/db/it/IoTDBFlushQueryIT.java | 82 + .../iotdb/db/it/audit/IoTDBAuditLogBasicIT.java | 2 + .../it/query/IoTDBPredicateConversionTreeIT.java | 293 +++ .../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 11 +- .../db/it/schema/IoTDBAlterTimeSeriesTypeIT.java | 2736 ++++++++++++++++++++ .../org/apache/iotdb/db/it/utils/TestUtils.java | 30 +- .../manual/basic/IoTDBPipeLifeCycleIT.java | 230 +- .../manual/basic/IoTDBPipePermissionIT.java | 4 +- .../manual/enhanced/IoTDBPipeIdempotentIT.java | 44 +- .../treemodel/auto/basic/IoTDBPipeDataSinkIT.java | 137 +- .../treemodel/auto/basic/IoTDBPipeLifeCycleIT.java | 217 +- .../treemodel/auto/basic/IoTDBPipeProcessorIT.java | 19 +- .../treemodel/auto/basic/IoTDBPipeProtocolIT.java | 87 +- .../auto/basic/IoTDBPipeSinkParallelIT.java | 20 +- .../treemodel/auto/basic/IoTDBPipeSourceIT.java | 460 ++-- .../auto/basic/IoTDBPipeSwitchStatusIT.java | 88 +- .../treemodel/auto/basic/IoTDBPipeSyntaxIT.java | 376 +-- .../auto/basic/IoTDBTreePatternFormatIT.java | 203 +- .../auto/enhanced/IoTDBPipeAutoConflictIT.java | 126 +- .../auto/enhanced/IoTDBPipeAutoDropIT.java | 42 +- .../auto/enhanced/IoTDBPipeClusterIT.java | 277 +- .../auto/enhanced/IoTDBPipeIdempotentIT.java | 30 +- .../auto/enhanced/IoTDBPipeNullValueIT.java | 22 +- .../auto/enhanced/IoTDBPipeSinkCompressionIT.java | 103 +- .../auto/enhanced/IoTDBPipeWithLoadIT.java | 17 +- .../treemodel/auto/enhanced/PipeNowFunctionIT.java | 153 +- .../treemodel/manual/IoTDBPipeInclusionIT.java | 67 +- .../manual/IoTDBPipeManualConflictIT.java | 102 +- .../manual/IoTDBPipeMetaHistoricalIT.java | 67 +- .../manual/IoTDBPipeMetaLeaderChangeIT.java | 52 +- .../treemodel/manual/IoTDBPipeMetaRestartIT.java | 52 +- .../manual/IoTDBPipeMultiSchemaRegionIT.java | 26 +- .../treemodel/manual/IoTDBPipePermissionIT.java | 204 ++ .../manual/IoTDBPipeTypeConversionISessionIT.java | 10 +- .../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 2 +- .../iotdb/pipe/it/single/IoTDBPipeAggregateIT.java | 14 +- .../iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java | 281 +- .../pipe/it/single/IoTDBPipePermissionIT.java | 3 +- .../iotdb/pipe/it/triple/IoTDBPipeForwardIT.java | 2 + .../relational/it/db/it/IoTDBDeletionTableIT.java | 21 + .../it/db/it/IoTDBSetSystemStatusTableIT.java | 1 + .../it/query/object/IoTDBObjectQuery2IT.java | 326 +++ .../it/query/object/IoTDBObjectQueryIT.java | 309 +++ .../it/query/old/IoTDBSimpleQueryTableIT.java | 63 +- .../scalar/IoTDBScalarFunctionTableIT.java | 16 +- .../relational/it/query/recent/IoTDBCteIT.java | 531 +++- .../it/query/recent/IoTDBLengthFunctionIT.java | 2 +- .../recent/IoTDBPredicateConversionTableIT.java | 379 +++ .../it/query/recent/IoTDBTableFunctionIT.java | 92 + .../it/query/recent/IoTExplainAnalyzeIT.java | 264 ++ .../informationschema/IoTDBCurrentQueriesIT.java | 263 ++ .../it/schema/IoTDBAlterColumnTypeIT.java | 2349 +++++++++++++++++ .../relational/it/schema/IoTDBDatabaseIT.java | 6 +- .../iotdb/relational/it/schema/IoTDBTableIT.java | 148 ++ .../relational/it/session/IoTDBObjectDeleteIT.java | 363 +++ .../it/session/IoTDBObjectDeleteIT2.java | 39 +- .../relational/it/session/IoTDBObjectInsertIT.java | 339 +++ .../it/session/IoTDBObjectInsertIT2.java | 170 ++ .../it/session/IoTDBSessionRelationalIT.java | 15 +- .../IoTDBSubscriptionConsumerGroupIT.java | 68 +- .../src/test/resources/object-example.pt | Bin 0 -> 1906 bytes iotdb-api/external-api/pom.xml | 2 +- iotdb-api/pipe-api/pom.xml | 7 +- .../iotdb/pipe/api/collector/DataCollector.java | 26 +- .../iotdb/pipe/api/collector/TabletCollector.java | 39 +- .../api/customizer/parameter/PipeParameters.java | 31 +- .../event/dml/insertion/TabletInsertionEvent.java | 11 + .../java/org/apache/iotdb/pipe/api/type/Type.java | 5 +- .../customizer/parameter/PipeParametersTest.java | 39 +- iotdb-api/pom.xml | 2 +- iotdb-api/trigger-api/pom.xml | 2 +- iotdb-api/udf-api/pom.xml | 2 +- .../iotdb/udf/api/relational/access/Record.java | 52 +- .../java/org/apache/iotdb/udf/api/type/Type.java | 10 +- iotdb-client/cli/pom.xml | 22 +- .../java/org/apache/iotdb/cli/AbstractCli.java | 5 +- .../org/apache/iotdb/tool/data/ExportData.java | 5 +- iotdb-client/client-cpp/pom.xml | 4 +- iotdb-client/client-cpp/src/main/Common.cpp | 1 + iotdb-client/client-cpp/src/main/Common.h | 3 +- .../client-cpp/src/main/IoTDBRpcDataSet.cpp | 1 + iotdb-client/client-cpp/src/main/Session.cpp | 30 +- iotdb-client/client-cpp/src/main/Session.h | 54 +- .../client-cpp/src/main/SessionDataSet.cpp | 2 + .../src/test/cpp/sessionRelationalIT.cpp | 12 +- iotdb-client/client-py/pom.xml | 8 +- iotdb-client/isession/pom.xml | 8 +- .../org/apache/iotdb/isession/SessionDataSet.java | 17 + iotdb-client/jdbc/pom.xml | 8 +- .../iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java | 1 + .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 29 +- iotdb-client/pom.xml | 2 +- iotdb-client/service-rpc/pom.xml | 6 +- .../org/apache/iotdb/rpc/IoTDBJDBCDataSet.java | 6 + .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 6 + .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 12 + .../java/org/apache/iotdb/rpc/TSStatusCode.java | 26 +- iotdb-client/session/pom.xml | 10 +- .../java/org/apache/iotdb/session/Session.java | 1 + .../payload/SubscriptionSessionDataSet.java | 1 + .../apache/iotdb/session/util/SessionUtils.java | 3 + iotdb-core/ainode/iotdb/ainode/core/config.py | 8 +- iotdb-core/ainode/iotdb/ainode/core/constant.py | 33 +- iotdb-core/ainode/iotdb/ainode/core/exception.py | 108 +- .../core/inference/dispatcher/basic_dispatcher.py | 6 +- .../ainode/core/inference/inference_request.py | 19 +- .../core/inference/inference_request_pool.py | 16 +- .../core/inference/pipeline/basic_pipeline.py | 174 +- .../iotdb/ainode/core/inference/pool_controller.py | 4 +- .../iotdb/ainode/core/inference/pool_group.py | 6 +- .../pool_scheduler/basic_pool_scheduler.py | 4 +- .../iotdb/ainode/core/manager/inference_manager.py | 66 +- .../iotdb/ainode/core/manager/model_manager.py | 47 +- .../ainode/iotdb/ainode/core/manager/utils.py | 4 +- .../{model_constants.py => chronos2/__init__.py} | 31 - .../iotdb/ainode/core/model/chronos2/base.py | 300 +++ .../ainode/core/model/chronos2/chronos_bolt.py | 703 +++++ .../iotdb/ainode/core/model/chronos2/config.py | 138 + .../iotdb/ainode/core/model/chronos2/dataset.py | 756 ++++++ .../iotdb/ainode/core/model/chronos2/layers.py | 481 ++++ .../iotdb/ainode/core/model/chronos2/model.py | 909 +++++++ .../core/model/chronos2/pipeline_chronos2.py | 476 ++++ .../iotdb/ainode/core/model/chronos2/utils.py | 242 ++ .../iotdb/ainode/core/model/model_constants.py | 7 - .../ainode/iotdb/ainode/core/model/model_info.py | 36 +- .../ainode/iotdb/ainode/core/model/model_loader.py | 21 +- .../iotdb/ainode/core/model/model_storage.py | 241 +- .../ainode/core/model/sktime/arima/config.json | 2 +- .../core/model/sktime/configuration_sktime.py | 44 +- .../model/sktime/exponential_smoothing/config.json | 2 +- .../ainode/core/model/sktime/modeling_sktime.py | 30 +- .../core/model/sktime/naive_forecaster/config.json | 2 +- .../ainode/core/model/sktime/pipeline_sktime.py | 121 +- .../core/model/sktime/stl_forecaster/config.json | 2 +- .../ainode/core/model/sundial/modeling_sundial.py | 13 +- .../ainode/core/model/sundial/pipeline_sundial.py | 90 +- .../ainode/core/model/timer_xl/modeling_timer.py | 6 +- .../ainode/core/model/timer_xl/pipeline_timer.py | 85 +- iotdb-core/ainode/iotdb/ainode/core/model/utils.py | 53 +- iotdb-core/ainode/iotdb/ainode/core/rpc/handler.py | 48 +- iotdb-core/ainode/iotdb/ainode/core/util/serde.py | 30 +- iotdb-core/ainode/pom.xml | 12 +- iotdb-core/ainode/pyproject.toml | 6 +- iotdb-core/antlr/pom.xml | 2 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 40 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 + iotdb-core/confignode/pom.xml | 26 +- .../iotdb/confignode/audit/CNAuditLogger.java | 50 +- .../client/async/CnToDnAsyncRequestType.java | 1 + .../CnToDnInternalServiceAsyncRequestManager.java | 6 + .../rpc/DataNodeAsyncRequestRPCHandler.java | 1 + .../confignode/conf/SystemPropertiesUtils.java | 15 + .../consensus/request/ConfigPhysicalPlan.java | 8 + .../consensus/request/ConfigPhysicalPlanType.java | 3 + .../request/ConfigPhysicalPlanVisitor.java | 16 + .../pipe/payload/PipeAlterTimeSeriesPlan.java | 110 + .../pipe/payload/PipeDeactivateTemplatePlan.java | 2 +- .../write/table/AlterColumnDataTypePlan.java | 63 + .../write/template/CreateSchemaTemplatePlan.java | 2 +- .../consensus/response/table/DescTableResp.java | 21 +- .../response/template/TemplateInfoResp.java | 2 +- .../response/template/TemplateSetInfoResp.java | 2 +- .../iotdb/confignode/manager/ConfigManager.java | 59 +- .../apache/iotdb/confignode/manager/IManager.java | 4 + .../confignode/manager/PermissionManager.java | 7 +- .../iotdb/confignode/manager/ProcedureManager.java | 39 +- .../iotdb/confignode/manager/node/NodeManager.java | 1 + .../pipe/agent/task/PipeConfigNodeTaskStage.java | 8 +- .../pipe/coordinator/task/PipeTaskCoordinator.java | 16 +- .../coordinator/task/PipeTaskCoordinatorLock.java | 52 +- .../receiver/protocol/IoTDBConfigNodeReceiver.java | 513 ++-- .../PipeConfigPhysicalPlanTSStatusVisitor.java | 18 + .../pipe/source/ConfigRegionListeningFilter.java | 8 +- .../pipe/source/IoTDBConfigRegionSource.java | 72 +- ...nfigPhysicalPlanTablePrivilegeParseVisitor.java | 389 --- ...ava => PipeConfigTablePatternParseVisitor.java} | 9 +- .../PipeConfigTablePrivilegeParseVisitor.java | 378 +++ ....java => PipeConfigTableScopeParseVisitor.java} | 2 +- ...java => PipeConfigTreePatternParseVisitor.java} | 29 +- .../PipeConfigTreePrivilegeParseVisitor.java | 610 +++++ ...r.java => PipeConfigTreeScopeParseVisitor.java} | 2 +- .../manager/schema/ClusterSchemaManager.java | 58 +- .../confignode/persistence/auth/AuthorInfo.java | 7 + .../persistence/auth/AuthorPlanExecutor.java | 20 + .../persistence/auth/IAuthorPlanExecutor.java | 5 + .../persistence/executor/ConfigPlanExecutor.java | 5 + .../persistence/pipe/PipePluginInfo.java | 33 +- .../schema/CNPhysicalPlanGenerator.java | 2 +- .../persistence/schema/ClusterSchemaInfo.java | 70 +- .../confignode/persistence/schema/ConfigMTree.java | 82 +- .../persistence/schema/ConfigMTreeStore.java | 2 +- .../persistence/schema/TemplateTable.java | 2 +- .../schema/mnode/impl/ConfigTableNode.java | 15 + .../schema/mnode/info/ConfigTableInfo.java | 16 + .../schema/AlterEncodingCompressorProcedure.java | 8 +- .../impl/schema/AlterLogicalViewProcedure.java | 10 +- .../schema/AlterTimeSeriesDataTypeProcedure.java | 413 +++ .../impl/schema/DataNodeRegionTaskExecutor.java | 17 +- .../impl/schema/DataNodeTSStatusTaskExecutor.java | 39 + .../impl/schema/DeactivateTemplateProcedure.java | 34 +- .../impl/schema/DeleteLogicalViewProcedure.java | 32 +- .../impl/schema/DeleteTimeSeriesProcedure.java | 25 +- .../procedure/impl/schema/SchemaUtils.java | 14 +- .../impl/schema/SetTemplateProcedure.java | 14 +- .../impl/schema/UnsetTemplateProcedure.java | 2 +- .../table/AlterTableColumnDataTypeProcedure.java | 212 ++ .../impl/schema/table/DeleteDevicesProcedure.java | 96 +- .../schema/table/DropTableColumnProcedure.java | 2 +- .../schema/AlterTableColumnDataTypeState.java} | 13 +- .../schema/AlterTimeSeriesDataTypeState.java} | 12 +- .../procedure/store/ProcedureFactory.java | 18 + .../confignode/procedure/store/ProcedureType.java | 5 + .../thrift/ConfigNodeRPCServiceProcessor.java | 6 + .../request/ConfigPhysicalPlanSerDeTest.java | 35 +- ...t.java => PipeConfigScopeParseVisitorTest.java} | 2 +- ...=> PipeConfigTablePatternParseVisitorTest.java} | 12 +- ... => PipeConfigTreePatternParseVisitorTest.java} | 25 +- .../PipeConfigTreePrivilegeParseVisitorTest.java | 316 +++ .../persistence/CNPhysicalPlanGeneratorTest.java | 2 +- .../iotdb/confignode/persistence/PipeInfoTest.java | 16 +- .../persistence/schema/ClusterSchemaInfoTest.java | 2 +- .../persistence/schema/ConfigMTreeTest.java | 2 +- .../persistence/schema/TemplateTableTest.java | 2 +- .../pipe/receiver/PipeEnrichedProcedureTest.java | 2 +- .../schema/DataNodeRegionTaskExecutorTest.java | 81 + .../schema/DeactivateTemplateProcedureTest.java | 2 +- .../impl/schema/UnsetTemplateProcedureTest.java | 2 +- iotdb-core/consensus/pom.xml | 14 +- iotdb-core/datanode/pom.xml | 52 +- .../org/apache/iotdb/db/audit/DNAuditLogger.java | 398 +-- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 2 +- .../iotdb/db/auth/ClusterAuthorityFetcher.java | 59 +- .../apache/iotdb/db/conf/DataNodeMemoryConfig.java | 5 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 43 + .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 59 + .../dataregion/DataExecutionVisitor.java | 13 + .../schemaregion/SchemaExecutionVisitor.java | 102 +- .../DataTypeInconsistentException.java} | 16 +- .../db/exception/ainode/GetModelInfoException.java | 2 +- .../agent/task/connection/PipeEventCollector.java | 22 +- .../subtask/processor/PipeProcessorSubtask.java | 28 +- .../agent/task/subtask/sink/PipeSinkSubtask.java | 2 +- .../db/pipe/event/common/PipeInsertionEvent.java | 1 + .../db/pipe/event/common/row/PipeRowCollector.java | 35 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 93 +- .../common/tablet/PipeRawTabletEventConverter.java | 72 + .../common/tablet/PipeRawTabletInsertionEvent.java | 34 +- .../event/common/tablet/PipeTabletCollector.java | 61 + .../tablet/parser/TabletInsertionEventParser.java | 7 +- .../TabletInsertionEventTablePatternParser.java | 18 +- .../TabletInsertionEventTreePatternParser.java | 49 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 124 +- .../tsfile/parser/TsFileInsertionEventParser.java | 8 +- .../parser/TsFileInsertionEventParserProvider.java | 26 +- .../query/TsFileInsertionEventQueryParser.java | 48 +- .../scan/TsFileInsertionEventScanParser.java | 60 +- .../table/TsFileInsertionEventTableParser.java | 38 +- .../processor/aggregate/AggregateProcessor.java | 3 + .../protocol/thrift/IoTDBDataNodeReceiver.java | 7 +- .../transform/converter/ValueConverter.java | 6 +- .../visitor/PipePlanToStatementVisitor.java | 1 + ...leStatementDataTypeConvertExecutionVisitor.java | 2 +- .../resource/memory/InsertNodeMemoryEstimator.java | 1 + .../evolvable/batch/PipeTabletEventBatch.java | 6 + .../batch/PipeTabletEventTsFileBatch.java | 6 +- .../sink/protocol/legacy/IoTDBLegacyPipeSink.java | 7 +- .../sink/protocol/opcda/OpcDaServerHandle.java | 2 + .../db/pipe/sink/protocol/opcua/OpcUaSink.java | 302 ++- .../sink/protocol/opcua/client/ClientRunner.java | 112 + .../opcua/client/IoTDBKeyStoreLoaderClient.java | 127 + .../protocol/opcua/client/IoTDBOpcUaClient.java | 301 +++ .../opcua/{ => server}/OpcUaKeyStoreLoader.java | 2 +- .../opcua/{ => server}/OpcUaNameSpace.java | 171 +- .../opcua/{ => server}/OpcUaServerBuilder.java | 97 +- .../thrift/sync/IoTDBSchemaRegionSink.java | 4 +- .../sink/protocol/writeback/WriteBackSink.java | 70 +- .../sink/util/sorter/PipeInsertEventSorter.java | 1 + .../matcher/CachedSchemaPatternMatcher.java | 8 +- .../schemaregion/IoTDBSchemaRegionSource.java | 80 +- .../PipePlanTreePrivilegeParseVisitor.java | 355 +++ ...r.java => PipeTableStatementToPlanVisitor.java} | 2 +- .../PipeTreeStatementToPlanVisitor.java | 60 + .../iotdb/db/protocol/client/ConfigNodeClient.java | 7 + .../iotdb/db/protocol/client/an/AINodeClient.java | 6 +- .../protocol/rest/v2/impl/RestApiServiceImpl.java | 16 +- .../iotdb/db/protocol/session/IClientSession.java | 3 +- .../db/protocol/session/InternalClientSession.java | 4 +- .../db/protocol/session/MqttClientSession.java | 3 +- .../iotdb/db/protocol/session/SessionManager.java | 13 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 18 +- .../impl/DataNodeInternalRPCServiceImpl.java | 38 + .../db/queryengine/common/MPPQueryContext.java | 128 +- .../iotdb/db/queryengine/common/QueryId.java | 7 + .../common/schematree/ClusterSchemaTree.java | 2 +- .../queryengine/common/schematree/ISchemaTree.java | 2 +- .../schematree/visitor/SchemaTreeVisitor.java | 2 +- .../SchemaTreeVisitorWithLimitOffsetWrapper.java | 2 +- .../execution/aggregation/AccumulatorFactory.java | 1 + .../execution/aggregation/AvgAccumulator.java | 1 + .../execution/aggregation/ExtremeAccumulator.java | 22 +- .../aggregation/FirstValueAccumulator.java | 37 +- .../aggregation/LastValueAccumulator.java | 37 +- .../aggregation/MaxMinByBaseAccumulator.java | 6 + .../execution/aggregation/MaxValueAccumulator.java | 14 +- .../execution/aggregation/MinValueAccumulator.java | 14 +- .../execution/aggregation/SumAccumulator.java | 1 + .../execution/aggregation/VarianceAccumulator.java | 1 + .../execution/exchange/SharedTsBlockQueue.java | 4 + .../exchange/source/LocalSourceHandle.java | 5 + .../execution/executor/RegionWriteExecutor.java | 27 +- .../fragment/FragmentInstanceContext.java | 39 +- .../fragment/FragmentInstanceExecution.java | 33 +- .../execution/memory/LocalMemoryManager.java | 1 - .../execution/operator/AggregationUtil.java | 1 + .../execution/operator/ExplainAnalyzeOperator.java | 76 +- .../execution/operator/process/TopKOperator.java | 14 +- .../operator/process/TransformOperator.java | 1 + .../operator/process/function/partition/Slice.java | 62 +- .../process/join/merge/MergeSortComparator.java | 1 + .../expression/PatternExpressionComputation.java | 5 +- .../process/window/function/value/LagFunction.java | 1 + .../window/function/value/LeadFunction.java | 1 + .../process/window/utils/RowComparator.java | 3 + .../operator/schema/SchemaFetchScanOperator.java | 2 +- .../schema/source/SchemaSourceFactory.java | 2 +- .../schema/source/TimeSeriesSchemaSource.java | 2 +- .../execution/operator/source/FileLoaderUtils.java | 66 +- .../execution/operator/source/SeriesScanUtil.java | 715 ++++- .../source/relational/CteScanOperator.java | 119 + .../InformationSchemaContentSupplierFactory.java | 167 +- .../relational/LastQueryAggTableScanOperator.java | 1 + .../relational/aggregation/AccumulatorFactory.java | 2 + .../ApproxCountDistinctAccumulator.java | 1 + .../relational/aggregation/AvgAccumulator.java | 2 + .../relational/aggregation/ExtremeAccumulator.java | 21 +- .../relational/aggregation/FirstAccumulator.java | 36 +- .../relational/aggregation/FirstByAccumulator.java | 22 +- .../relational/aggregation/LastAccumulator.java | 36 +- .../relational/aggregation/LastByAccumulator.java | 12 +- .../aggregation/MaskedRecordIterator.java | 2 - .../relational/aggregation/MaxAccumulator.java | 22 +- .../relational/aggregation/MinAccumulator.java | 22 +- .../relational/aggregation}/RecordIterator.java | 70 +- .../relational/aggregation/SumAccumulator.java | 2 + .../aggregation/TableMaxMinByBaseAccumulator.java | 5 + .../aggregation/TableVarianceAccumulator.java | 2 + .../UserDefinedAggregateFunctionAccumulator.java | 1 - .../source/relational/aggregation/Utils.java | 2 + .../GroupedApproxCountDistinctAccumulator.java | 1 + .../aggregation/grouped/GroupedAvgAccumulator.java | 1 + .../grouped/GroupedExtremeAccumulator.java | 8 + .../grouped/GroupedFirstAccumulator.java | 8 + .../grouped/GroupedFirstByAccumulator.java | 9 + .../grouped/GroupedLastAccumulator.java | 8 + .../grouped/GroupedLastByAccumulator.java | 9 + .../aggregation/grouped/GroupedMaxAccumulator.java | 8 + .../grouped/GroupedMaxMinByBaseAccumulator.java | 15 + .../aggregation/grouped/GroupedMinAccumulator.java | 8 + .../grouped/GroupedModeAccumulator.java | 4 + .../aggregation/grouped/GroupedSumAccumulator.java | 1 + .../GroupedUserDefinedAggregateAccumulator.java | 2 +- .../grouped/GroupedVarianceAccumulator.java | 1 + .../operator/window/WindowManagerFactory.java | 2 + .../relational/ColumnTransformerBuilder.java | 53 +- .../iotdb/db/queryengine/plan/Coordinator.java | 383 ++- .../db/queryengine/plan/analyze/Analysis.java | 2 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 6 +- .../plan/analyze/ClusterPartitionFetcher.java | 4 + .../plan/analyze/TemplatedAggregationAnalyze.java | 2 +- .../queryengine/plan/analyze/TemplatedAnalyze.java | 2 +- .../analyze/schema/AutoCreateSchemaExecutor.java | 2 +- .../analyze/schema/ClusterSchemaFetchExecutor.java | 2 +- .../plan/analyze/schema/ClusterSchemaFetcher.java | 2 +- .../plan/analyze/schema/ISchemaFetcher.java | 2 +- .../plan/analyze/schema/TemplateSchemaFetcher.java | 2 +- .../plan/execution/IQueryExecution.java | 2 + .../queryengine/plan/execution/QueryExecution.java | 5 + .../plan/execution/config/ConfigExecution.java | 5 + .../execution/config/TableConfigTaskVisitor.java | 90 +- .../execution/config/TreeConfigTaskVisitor.java | 43 +- .../config/executor/ClusterConfigTaskExecutor.java | 200 +- .../config/executor/IConfigTaskExecutor.java | 17 +- .../config/metadata/AlterTimeSeriesTask.java | 57 + ...eateTrainingTask.java => CreateTuningTask.java} | 8 +- .../relational/AlterColumnDataTypeTask.java | 54 + .../relational/DescribeTableDetailsTask.java | 14 +- .../template/ShowNodesInSchemaTemplateTask.java | 2 +- .../metadata/template/ShowSchemaTemplateTask.java | 2 +- .../plan/execution/config/session/PrepareTask.java | 3 +- .../execution/config/sys/pipe/AlterPipeTask.java | 10 +- .../execution/config/sys/pipe/CreatePipeTask.java | 8 +- .../execution/config/sys/pipe/ShowPipeTask.java | 5 +- .../TableModelStatementMemorySourceVisitor.java | 34 +- .../predicate/ConvertPredicateToFilterVisitor.java | 134 +- .../db/queryengine/plan/parser/ASTVisitor.java | 126 +- .../plan/planner/LocalExecutionPlanner.java | 8 + .../plan/planner/LogicalPlanBuilder.java | 2 +- .../plan/planner/LogicalPlanVisitor.java | 21 +- .../plan/planner/OperatorTreeGenerator.java | 2 + .../plan/planner/TableOperatorGenerator.java | 79 +- .../plan/planner/distribution/SourceRewriter.java | 7 +- .../memory/FakedMemoryReservationManager.java | 11 + .../planner/memory/MemoryReservationManager.java | 29 + .../NotThreadSafeMemoryReservationManager.java | 23 + .../memory/ThreadSafeMemoryReservationManager.java | 13 + .../plan/planner/plan/DistributedQueryPlan.java | 11 + .../plan/planner/plan/node/PlanGraphPrinter.java | 20 + .../plan/planner/plan/node/PlanNodeType.java | 8 + .../plan/planner/plan/node/PlanVisitor.java | 14 + .../metadata/read/LevelTimeSeriesCountNode.java | 2 +- .../metadata/read/SeriesSchemaFetchScanNode.java | 2 +- .../node/metadata/read/TimeSeriesCountNode.java | 2 +- .../metadata/read/TimeSeriesSchemaScanNode.java | 2 +- .../node/metadata/write/AlterTimeSeriesNode.java | 87 +- .../plan/node/pipe/PipeEnrichedInsertNode.java | 7 + .../plan/node/source/LastQueryScanNode.java | 4 + .../plan/node/write/InsertMultiTabletsNode.java | 9 + .../plan/planner/plan/node/write/InsertNode.java | 5 + .../planner/plan/node/write/InsertRowNode.java | 26 +- .../planner/plan/node/write/InsertRowsNode.java | 9 + .../plan/node/write/InsertRowsOfOneDeviceNode.java | 9 + .../planner/plan/node/write/InsertTabletNode.java | 40 +- .../plan/planner/plan/node/write/ObjectNode.java | 331 +++ .../plan/node/write/RelationalInsertRowNode.java | 11 + .../plan/node/write/RelationalInsertRowsNode.java | 57 +- .../node/write/RelationalInsertTabletNode.java | 147 ++ .../plan/relational/analyzer/Analysis.java | 31 + .../plan/relational/analyzer/Analyzer.java | 6 + .../relational/analyzer/ExpressionAnalyzer.java | 8 +- .../plan/relational/analyzer/Scope.java | 36 +- .../relational/analyzer/StatementAnalyzer.java | 81 +- .../analyzer/StatementAnalyzerFactory.java | 4 + .../predicate/ConvertPredicateToFilterVisitor.java | 116 +- .../function/TableBuiltinTableFunction.java | 6 +- .../function/tvf/ClassifyTableFunction.java | 383 +++ .../function/tvf/ForecastTableFunction.java | 325 +-- .../relational/metadata/TableMetadataImpl.java | 45 +- .../metadata/cache/DeviceSchemaRequestCache.java | 8 +- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 4 +- .../fetcher/TableDeviceSchemaValidator.java | 27 +- .../fetcher/TableHeaderSchemaValidator.java | 13 +- .../cache/TreeDeviceSchemaCacheManager.java | 2 +- .../plan/relational/planner/CteMaterializer.java | 355 +++ .../plan/relational/planner/QueryPlanner.java | 50 +- .../plan/relational/planner/RelationPlanner.java | 135 +- .../plan/relational/planner/SubqueryPlanner.java | 27 +- .../relational/planner/TableLogicalPlanner.java | 28 +- .../planner/distribute/AddExchangeNodes.java | 50 +- .../distribute/TableDistributedPlanGenerator.java | 8 +- .../distribute/TableDistributedPlanner.java | 17 +- ...ithUncorrelatedScalarSubqueryReconstructor.java | 236 ++ .../plan/relational/planner/node/CteScanNode.java | 132 + .../DataNodeLocationSupplierFactory.java | 2 + .../PushLimitOffsetIntoTableScan.java | 7 + .../optimizations/PushPredicateIntoTableScan.java | 85 + .../optimizations/TransformSortToStreamSort.java | 7 + .../optimizations/UnaliasSymbolReferences.java | 7 + .../plan/relational/security/AccessControl.java | 10 + .../relational/security/AccessControlImpl.java | 143 +- .../relational/security/AllowAllAccessControl.java | 17 + .../relational/security/ITableAuthCheckerImpl.java | 68 +- .../security/TreeAccessCheckVisitor.java | 313 ++- .../plan/relational/sql/AstMemoryEstimator.java | 67 - .../sql/ast/AbstractQueryDeviceWithCache.java | 21 + .../relational/sql/ast/AbstractTraverseDevice.java | 48 + .../plan/relational/sql/ast/AddColumn.java | 13 + .../plan/relational/sql/ast/AliasedRelation.java | 14 + .../plan/relational/sql/ast/AllColumns.java | 13 + .../plan/relational/sql/ast/AllRows.java | 9 + .../relational/sql/ast/AlterColumnDataType.java | 133 + .../plan/relational/sql/ast/AlterDB.java | 9 + .../plan/relational/sql/ast/AlterPipe.java | 15 + .../plan/relational/sql/ast/AnchorPattern.java | 11 + .../sql/ast/ArithmeticBinaryExpression.java | 12 + .../sql/ast/ArithmeticUnaryExpression.java | 11 + .../plan/relational/sql/ast/AsofJoinOn.java | 11 + .../sql/ast/AstMemoryEstimationHelper.java | 123 + .../plan/relational/sql/ast/AstVisitor.java | 4 + .../plan/relational/sql/ast/BetweenPredicate.java | 13 + .../plan/relational/sql/ast/BinaryLiteral.java | 11 + .../plan/relational/sql/ast/BooleanLiteral.java | 10 + .../queryengine/plan/relational/sql/ast/Cast.java | 12 + .../plan/relational/sql/ast/ClearCache.java | 11 + .../relational/sql/ast/CoalesceExpression.java | 11 + .../plan/relational/sql/ast/ColumnDefinition.java | 15 + .../plan/relational/sql/ast/Columns.java | 11 + .../relational/sql/ast/ComparisonExpression.java | 34 +- .../plan/relational/sql/ast/CountDevice.java | 8 + .../plan/relational/sql/ast/CountStatement.java | 13 + .../plan/relational/sql/ast/CreateDB.java | 9 + .../plan/relational/sql/ast/CreateFunction.java | 13 + .../plan/relational/sql/ast/CreateIndex.java | 13 + .../plan/relational/sql/ast/CreateModel.java | 13 + .../relational/sql/ast/CreateOrUpdateDevice.java | 16 + .../plan/relational/sql/ast/CreatePipe.java | 52 +- .../plan/relational/sql/ast/CreatePipePlugin.java | 14 + .../plan/relational/sql/ast/CreateTable.java | 26 + .../plan/relational/sql/ast/CreateTopic.java | 13 + .../plan/relational/sql/ast/CreateTraining.java | 15 + .../plan/relational/sql/ast/CreateView.java | 14 + .../plan/relational/sql/ast/CurrentDatabase.java | 11 + .../plan/relational/sql/ast/CurrentTime.java | 14 + .../plan/relational/sql/ast/CurrentUser.java | 11 + .../plan/relational/sql/ast/DatabaseStatement.java | 9 + .../plan/relational/sql/ast/Deallocate.java | 12 + .../plan/relational/sql/ast/DecimalLiteral.java | 11 + .../plan/relational/sql/ast/Delete.java | 12 + .../plan/relational/sql/ast/DeleteDevice.java | 19 +- .../relational/sql/ast/DereferenceExpression.java | 13 + .../plan/relational/sql/ast/DescribeTable.java | 11 + .../plan/relational/sql/ast/DoubleLiteral.java | 10 + .../plan/relational/sql/ast/DropColumn.java | 12 + .../plan/relational/sql/ast/DropDB.java | 11 + .../plan/relational/sql/ast/DropFunction.java | 11 + .../plan/relational/sql/ast/DropIndex.java | 12 + .../plan/relational/sql/ast/DropModel.java | 12 + .../plan/relational/sql/ast/DropPipe.java | 11 + .../plan/relational/sql/ast/DropPipePlugin.java | 12 + .../plan/relational/sql/ast/DropSubscription.java | 12 + .../plan/relational/sql/ast/DropTable.java | 12 + .../plan/relational/sql/ast/DropTopic.java | 12 + .../plan/relational/sql/ast/EmptyPattern.java | 11 + .../plan/relational/sql/ast/Except.java | 12 + .../plan/relational/sql/ast/ExcludedPattern.java | 12 + .../plan/relational/sql/ast/Execute.java | 12 + .../plan/relational/sql/ast/ExecuteImmediate.java | 13 + .../plan/relational/sql/ast/ExistsPredicate.java | 12 + .../plan/relational/sql/ast/Explain.java | 11 + .../plan/relational/sql/ast/ExplainAnalyze.java | 12 + .../plan/relational/sql/ast/ExtendRegion.java | 11 + .../plan/relational/sql/ast/Extract.java | 11 + .../plan/relational/sql/ast/FetchDevice.java | 19 + .../plan/relational/sql/ast/FieldReference.java | 11 + .../queryengine/plan/relational/sql/ast/Fill.java | 16 + .../plan/relational/sql/ast/FrameBound.java | 13 + .../plan/relational/sql/ast/FunctionCall.java | 23 + .../plan/relational/sql/ast/GenericDataType.java | 12 + .../plan/relational/sql/ast/GenericLiteral.java | 12 + .../plan/relational/sql/ast/GroupBy.java | 11 + .../plan/relational/sql/ast/GroupingSets.java | 14 + .../plan/relational/sql/ast/Identifier.java | 11 + .../plan/relational/sql/ast/IfExpression.java | 14 + .../plan/relational/sql/ast/InListExpression.java | 12 + .../plan/relational/sql/ast/InPredicate.java | 12 + .../plan/relational/sql/ast/Insert.java | 13 + .../plan/relational/sql/ast/InsertRow.java | 11 +- .../plan/relational/sql/ast/InsertRows.java | 14 +- .../plan/relational/sql/ast/InsertTablet.java | 9 + .../plan/relational/sql/ast/Intersect.java | 12 + .../relational/sql/ast/IsNotNullPredicate.java | 11 + .../plan/relational/sql/ast/IsNullPredicate.java | 11 + .../queryengine/plan/relational/sql/ast/Join.java | 13 + .../plan/relational/sql/ast/JoinCriteria.java | 4 +- .../plan/relational/sql/ast/JoinOn.java | 13 + .../plan/relational/sql/ast/JoinUsing.java | 11 + .../plan/relational/sql/ast/KillQuery.java | 12 + .../plan/relational/sql/ast/LikePredicate.java | 13 + .../queryengine/plan/relational/sql/ast/Limit.java | 11 + .../plan/relational/sql/ast/LoadModel.java | 14 + .../plan/relational/sql/ast/LoadTsFile.java | 36 + .../plan/relational/sql/ast/LogicalExpression.java | 11 + .../plan/relational/sql/ast/LongLiteral.java | 13 +- .../plan/relational/sql/ast/MeasureDefinition.java | 13 + .../plan/relational/sql/ast/MigrateRegion.java | 10 + .../plan/relational/sql/ast/NaturalJoin.java | 9 + .../queryengine/plan/relational/sql/ast/Node.java | 9 +- .../plan/relational/sql/ast/NotExpression.java | 11 + .../plan/relational/sql/ast/NullIfExpression.java | 13 + .../plan/relational/sql/ast/NullLiteral.java | 11 + .../plan/relational/sql/ast/NumericParameter.java | 11 + .../plan/relational/sql/ast/Offset.java | 11 + .../relational/sql/ast/OneOrMoreQuantifier.java | 12 + .../plan/relational/sql/ast/OrderBy.java | 11 + .../plan/relational/sql/ast/Parameter.java | 11 + .../relational/sql/ast/PatternAlternation.java | 12 + .../relational/sql/ast/PatternConcatenation.java | 12 + .../relational/sql/ast/PatternPermutation.java | 12 + .../sql/ast/PatternRecognitionRelation.java | 22 + .../plan/relational/sql/ast/PatternVariable.java | 12 + .../plan/relational/sql/ast/PipeEnriched.java | 13 + .../plan/relational/sql/ast/Prepare.java | 12 + .../plan/relational/sql/ast/ProcessingMode.java | 11 + .../plan/relational/sql/ast/Property.java | 12 + .../plan/relational/sql/ast/QualifiedName.java | 20 +- .../sql/ast/QuantifiedComparisonExpression.java | 13 + .../plan/relational/sql/ast/QuantifiedPattern.java | 13 + .../queryengine/plan/relational/sql/ast/Query.java | 53 + .../relational/sql/ast/QuerySpecification.java | 22 + .../plan/relational/sql/ast/RangeQuantifier.java | 14 + .../plan/relational/sql/ast/ReconstructRegion.java | 11 + .../sql/ast/RelationalAuthorStatement.java | 19 + .../plan/relational/sql/ast/RemoveAINode.java | 10 + .../plan/relational/sql/ast/RemoveConfigNode.java | 10 + .../plan/relational/sql/ast/RemoveDataNode.java | 11 + .../plan/relational/sql/ast/RemoveRegion.java | 11 + .../plan/relational/sql/ast/RenameColumn.java | 14 + .../plan/relational/sql/ast/RenameTable.java | 13 + .../queryengine/plan/relational/sql/ast/Row.java | 11 + .../relational/sql/ast/SearchedCaseExpression.java | 13 + .../plan/relational/sql/ast/Select.java | 10 + .../plan/relational/sql/ast/SetColumnComment.java | 13 + .../plan/relational/sql/ast/SetProperties.java | 12 + .../plan/relational/sql/ast/SetSqlDialect.java | 10 + .../plan/relational/sql/ast/SetTableComment.java | 12 + .../plan/relational/sql/ast/ShowAIDevices.java | 12 + .../plan/relational/sql/ast/ShowAINodes.java | 11 + .../plan/relational/sql/ast/ShowAvailableUrls.java | 11 + .../plan/relational/sql/ast/ShowCluster.java | 11 + .../plan/relational/sql/ast/ShowClusterId.java | 11 + .../plan/relational/sql/ast/ShowConfigNodes.java | 11 + .../plan/relational/sql/ast/ShowConfiguration.java | 1 - .../relational/sql/ast/ShowCurrentDatabase.java | 11 + .../relational/sql/ast/ShowCurrentSqlDialect.java | 11 + .../relational/sql/ast/ShowCurrentTimestamp.java | 11 + .../plan/relational/sql/ast/ShowCurrentUser.java | 11 + .../plan/relational/sql/ast/ShowDB.java | 10 + .../plan/relational/sql/ast/ShowDataNodes.java | 11 + .../plan/relational/sql/ast/ShowDevice.java | 12 + .../plan/relational/sql/ast/ShowFunctions.java | 11 + .../plan/relational/sql/ast/ShowIndex.java | 12 + .../plan/relational/sql/ast/ShowLoadedModels.java | 13 + .../plan/relational/sql/ast/ShowModels.java | 12 + .../plan/relational/sql/ast/ShowPipePlugins.java | 12 + .../plan/relational/sql/ast/ShowPipes.java | 13 + .../plan/relational/sql/ast/ShowRegions.java | 13 + .../plan/relational/sql/ast/ShowStatement.java | 17 + .../plan/relational/sql/ast/ShowSubscriptions.java | 13 + .../plan/relational/sql/ast/ShowTables.java | 12 + .../plan/relational/sql/ast/ShowTopics.java | 13 + .../plan/relational/sql/ast/ShowVariables.java | 11 + .../plan/relational/sql/ast/ShowVersion.java | 11 + .../relational/sql/ast/SimpleCaseExpression.java | 14 + .../plan/relational/sql/ast/SimpleGroupBy.java | 12 + .../plan/relational/sql/ast/SingleColumn.java | 15 + .../plan/relational/sql/ast/SkipTo.java | 12 + .../plan/relational/sql/ast/SortItem.java | 11 + .../plan/relational/sql/ast/StartPipe.java | 12 + .../plan/relational/sql/ast/StopPipe.java | 11 + .../plan/relational/sql/ast/StringLiteral.java | 11 + .../relational/sql/ast/SubqueryExpression.java | 12 + .../plan/relational/sql/ast/SubsetDefinition.java | 14 + .../plan/relational/sql/ast/SymbolReference.java | 11 + .../queryengine/plan/relational/sql/ast/Table.java | 11 + .../relational/sql/ast/TableFunctionArgument.java | 14 + .../sql/ast/TableFunctionInvocation.java | 14 + .../sql/ast/TableFunctionTableArgument.java | 15 + .../plan/relational/sql/ast/TableSubquery.java | 12 + .../plan/relational/sql/ast/TimeRange.java | 12 + .../queryengine/plan/relational/sql/ast/Trim.java | 12 + .../plan/relational/sql/ast/TypeParameter.java | 12 + .../queryengine/plan/relational/sql/ast/Union.java | 11 + .../plan/relational/sql/ast/UnloadModel.java | 14 + .../plan/relational/sql/ast/Update.java | 11 + .../plan/relational/sql/ast/UpdateAssignment.java | 13 + .../queryengine/plan/relational/sql/ast/Use.java | 11 + .../plan/relational/sql/ast/Values.java | 11 + .../relational/sql/ast/VariableDefinition.java | 13 + .../plan/relational/sql/ast/WhenClause.java | 12 + .../plan/relational/sql/ast/WindowDefinition.java | 13 + .../plan/relational/sql/ast/WindowFrame.java | 14 + .../plan/relational/sql/ast/WindowReference.java | 12 + .../relational/sql/ast/WindowSpecification.java | 18 + .../queryengine/plan/relational/sql/ast/With.java | 11 + .../plan/relational/sql/ast/WithQuery.java | 42 +- .../plan/relational/sql/ast/WrappedStatement.java | 5 + .../relational/sql/ast/ZeroOrMoreQuantifier.java | 12 + .../relational/sql/ast/ZeroOrOneQuantifier.java | 12 + .../plan/relational/sql/parser/AstBuilder.java | 31 +- .../plan/relational/sql/util/AstUtil.java | 30 + .../plan/relational/sql/util/SqlFormatter.java | 8 +- .../plan/relational/type/CompatibleResolver.java | 5 + .../plan/relational/type/InternalTypeManager.java | 6 + .../relational/utils/ResultColumnAppender.java | 145 ++ .../plan/relational/utils/TypeUtil.java | 13 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 2 +- .../plan/statement/StatementVisitor.java | 6 + .../plan/statement/crud/InsertRowStatement.java | 7 + .../InternalBatchActivateTemplateStatement.java | 2 +- .../internal/SeriesSchemaFetchStatement.java | 2 +- .../AlterTimeSeriesDataTypeStatement.java} | 35 +- .../metadata/AlterTimeSeriesStatement.java | 15 +- .../metadata/pipe/AlterPipeStatement.java | 40 +- .../metadata/pipe/CreatePipeStatement.java | 28 +- .../template/ActivateTemplateStatement.java | 11 +- .../template/BatchActivateTemplateStatement.java | 2 +- .../template/DeactivateTemplateStatement.java | 2 +- .../db/queryengine/plan/udf/UDTFForecast.java | 2 + .../FragmentInstanceStatisticsDrawer.java | 20 +- .../binary/ReadObject2ColumnTransformer.java | 78 + .../ternary/ReadObject3ColumnTransformer.java | 99 + .../udf/UserDefineScalarFunctionTransformer.java | 2 +- .../AbstractCastFunctionColumnTransformer.java | 10 + ...r.java => AbstractLengthColumnTransformer.java} | 15 +- .../unary/scalar/BlobLengthColumnTransformer.java | 25 +- .../scalar/CastFunctionColumnTransformer.java | 3 + .../unary/scalar/LengthColumnTransformer.java | 30 +- .../scalar/ObjectLengthColumnTransformer.java} | 23 +- .../unary/scalar/ReadObjectColumnTransformer.java | 126 + .../scalar/TryCastFunctionColumnTransformer.java | 3 + .../dag/input/ConstantInputReader.java | 1 + .../unary/ArithmeticNegationTransformer.java | 1 + .../dag/transformer/unary/InTransformer.java | 2 + .../unary/scalar/CastFunctionTransformer.java | 7 + .../unary/scalar/DiffFunctionTransformer.java | 1 + .../unary/scalar/RoundFunctionTransformer.java | 1 + .../transformation/dag/util/TransformUtils.java | 2 + .../transformation/dag/util/TypeUtils.java | 4 +- .../datastructure/row/SerializableRowList.java | 2 + .../datastructure/tv/SerializableTVList.java | 1 + .../rescon/MemSchemaRegionStatistics.java | 2 +- .../schemaengine/schemaregion/ISchemaRegion.java | 12 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 26 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 19 +- .../schemaregion/mtree/IMTreeStore.java | 2 +- .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 47 +- .../schemaregion/mtree/impl/mem/MemMTreeStore.java | 2 +- .../mtree/impl/mem/mnode/info/LogicalViewInfo.java | 3 + .../mtree/impl/mem/mnode/info/MeasurementInfo.java | 3 + .../mnode/iterator/AbstractTraverserIterator.java | 2 +- .../mnode/iterator/MemoryTraverserIterator.java | 2 +- .../mtree/impl/pbtree/CachedMTreeStore.java | 2 +- .../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 2 +- .../pbtree/ReentrantReadOnlyCachedMTreeStore.java | 2 +- .../mnode/iterator/CachedTraverserIterator.java | 2 +- .../schemaregion/mtree/traverser/Traverser.java | 2 +- .../schemaregion/read/req/IShowTimeSeriesPlan.java | 2 +- .../read/req/SchemaRegionReadPlanFactory.java | 2 +- .../read/req/impl/ShowTimeSeriesPlanImpl.java | 2 +- .../read/resp/info/impl/ShowDevicesResult.java | 24 +- .../schemaregion/utils/MNodeUtils.java | 2 +- .../schemaregion/utils/ResourceByPathUtils.java | 25 +- .../template/ClusterTemplateManager.java | 1 + .../db/schemaengine/template/ITemplateManager.java | 1 + .../template/TemplateInternalRPCUtil.java | 2 + .../db/service/metrics/CompactionMetrics.java | 14 +- .../iotdb/db/service/metrics/FileMetrics.java | 20 + .../db/service/metrics/file/ObjectFileMetrics.java | 86 + .../metrics/memory/QueryEngineMemoryMetrics.java | 12 +- .../iotdb/db/storageengine/StorageEngine.java | 47 + .../storageengine/dataregion/Base32ObjectPath.java | 169 ++ .../db/storageengine/dataregion/DataRegion.java | 373 ++- .../db/storageengine/dataregion/IObjectPath.java | 70 + .../storageengine/dataregion/PlainObjectPath.java | 126 + .../performer/impl/FastCompactionPerformer.java | 27 +- .../impl/ReadChunkCompactionPerformer.java | 7 +- .../execute/task/SettleCompactionTask.java | 6 +- .../subtask/FastCompactionPerformerSubTask.java | 33 +- .../execute/utils/CompactionSeriesContext.java | 60 + .../compaction/execute/utils/CompactionUtils.java | 143 + .../execute/utils/MultiTsFileDeviceIterator.java | 75 +- .../fast/FastAlignedSeriesCompactionExecutor.java | 31 +- .../FastNonAlignedSeriesCompactionExecutor.java | 42 +- .../fast/element/ChunkMetadataElement.java | 10 +- .../fast/reader/CompactionAlignedChunkReader.java | 8 +- .../ReadChunkAlignedSeriesCompactionExecutor.java | 31 +- .../readchunk/SingleSeriesCompactionExecutor.java | 13 +- .../utils/writer/AbstractCompactionWriter.java | 1 + .../schedule/CompactionScheduleContext.java | 27 + .../compaction/schedule/TTLScheduleTask.java | 12 + .../selector/utils/TsFileResourceCandidate.java | 7 +- .../dataregion/memtable/AbstractMemTable.java | 12 +- .../memtable/AbstractWritableMemChunk.java | 4 +- .../memtable/AlignedReadOnlyMemChunk.java | 39 +- .../memtable/AlignedWritableMemChunk.java | 133 +- .../memtable/AlignedWritableMemChunkGroup.java | 7 + .../dataregion/memtable/IMemTable.java | 4 + .../memtable/IWritableMemChunkGroup.java | 4 + .../dataregion/memtable/ReadOnlyMemChunk.java | 37 +- .../dataregion/memtable/TsFileProcessor.java | 4 + .../dataregion/memtable/WritableMemChunk.java | 45 +- .../dataregion/memtable/WritableMemChunkGroup.java | 18 + .../dataregion/modification/DeletionPredicate.java | 4 + .../modification/TableDeletionEntry.java | 9 + .../read/reader/chunk/DiskAlignedChunkLoader.java | 16 +- .../read/reader/chunk/DiskChunkLoader.java | 10 + .../read/reader/chunk/MemAlignedPageReader.java | 10 +- .../read/reader/chunk/MemPageReader.java | 2 + .../dataregion/tsfile/TsFileResource.java | 13 +- .../tsfile/timeindex/ArrayDeviceTimeIndex.java | 5 +- .../dataregion/tsfile/timeindex/FileTimeIndex.java | 3 +- .../dataregion/tsfile/timeindex/ITimeIndex.java | 10 +- .../tsfile/timeindex/PlainDeviceTimeIndex.java | 3 +- .../allocation/AbstractNodeAllocationStrategy.java | 21 +- .../dataregion/wal/buffer/WALBuffer.java | 3 + .../dataregion/wal/buffer/WALEntry.java | 6 + .../dataregion/wal/buffer/WALEntryType.java | 4 +- .../dataregion/wal/buffer/WALInfoEntry.java | 4 + .../dataregion/wal/node/IWALNode.java | 3 + .../dataregion/wal/node/WALFakeNode.java | 6 + .../storageengine/dataregion/wal/node/WALNode.java | 52 +- .../file/UnsealedTsFileRecoverPerformer.java | 2 + ...leStatementDataTypeConvertExecutionVisitor.java | 2 +- .../load/memory/LoadTsFileMemoryManager.java | 19 +- .../splitter/BatchedAlignedValueChunkData.java | 2 + .../db/storageengine/rescon/disk/TierManager.java | 99 + .../rescon/memory/PrimitiveArrayManager.java | 1 + .../iotdb/db/tools/TsFileSplitByPartitionTool.java | 1 + .../org/apache/iotdb/db/utils/CommonUtils.java | 23 + .../iotdb/db/utils/EncodingInferenceUtils.java | 1 + .../java/org/apache/iotdb/db/utils/MemUtils.java | 4 +- .../org/apache/iotdb/db/utils/ObjectTypeUtils.java | 335 +++ .../org/apache/iotdb/db/utils/ObjectWriter.java | 82 + .../apache/iotdb/db/utils/QueryDataSetUtils.java | 11 +- .../org/apache/iotdb/db/utils/SchemaUtils.java | 266 ++ .../org/apache/iotdb/db/utils/TabletDecoder.java | 1 + .../apache/iotdb/db/utils/TimeValuePairUtils.java | 2 + .../apache/iotdb/db/utils/TypeInferenceUtils.java | 1 + .../apache/iotdb/db/utils/cte/CteDataReader.java | 52 + .../apache/iotdb/db/utils/cte/CteDataStore.java | 109 + .../apache/iotdb/db/utils/cte/MemoryReader.java | 75 + .../db/utils/datastructure/AlignedTVList.java | 162 +- .../db/utils/datastructure/BackIntTVList.java | 7 + .../iotdb/db/utils/datastructure/IntTVList.java | 35 +- .../MergeSortMultiAlignedTVListIterator.java | 1 + .../MergeSortMultiTVListIterator.java | 1 + .../datastructure/MultiAlignedTVListIterator.java | 1 + .../utils/datastructure/MultiTVListIterator.java | 1 + .../db/utils/datastructure/QuickIntTVList.java | 7 + .../iotdb/db/utils/datastructure/TVList.java | 41 +- .../iotdb/db/utils/datastructure/TimIntTVList.java | 7 + .../db/utils/windowing/window/WindowImpl.java | 1 + .../schemaregion/SchemaExecutionVisitorTest.java | 277 ++ .../cache/TreeDeviceSchemaCacheManagerTest.java | 2 +- .../schemaRegion/SchemaRegionManagementTest.java | 2 +- .../SchemaRegionSimpleRecoverTest.java | 2 +- .../schemaRegion/SchemaRegionTemplateTest.java | 2 +- .../schemaRegion/SchemaRegionTestUtil.java | 2 +- .../schemaRegion/SchemaStatisticsTest.java | 2 +- .../pipe/event/PipeTabletInsertionEventTest.java | 91 +- .../pipe/event/PipeTsFileInsertionEventTest.java | 298 +++ .../pipe/event/TsFileInsertionEventParserTest.java | 41 +- .../db/pipe/pattern/IoTDBTreePatternTest.java | 35 + .../db/pipe/pattern/TreePatternPruningTest.java | 156 ++ .../apache/iotdb/db/pipe/sink/PipeSinkTest.java | 102 + .../source/PipeTreeStatementToPlanVisitorTest.java | 87 + .../common/schematree/ClusterSchemaTreeTest.java | 2 +- .../fragment/FragmentInstanceExecutionTest.java | 243 +- .../execution/operator/CteScanOperatorTest.java | 246 ++ .../operator/MergeTreeSortOperatorTest.java | 5 + .../predicate/TreePredicateConversionTest.java | 572 ++++ .../plan/analyze/FakeSchemaFetcherImpl.java | 2 +- .../plan/function/RecordObjectTypeTest.java | 156 ++ .../plan/parser/StatementGeneratorTest.java | 34 +- .../plan/planner/distribution/LastQueryTest.java | 57 + .../plan/planner/distribution/Util.java | 2 +- .../plan/planner/distribution/Util2.java | 2 +- .../planner/node/source/SourceNodeSerdeTest.java | 16 + .../statement/sys/pipe/PipeStatementTest.java | 8 +- .../plan/relational/analyzer/AnalyzerTest.java | 7 + .../relational/analyzer/InsertIntoQueryTest.java | 7 + .../analyzer/LimitOffsetPushDownTest.java | 7 + .../analyzer/PushAggregationThroughUnionTest.java | 6 + .../plan/relational/analyzer/SortTest.java | 7 + .../plan/relational/analyzer/SubQueryTest.java | 7 + .../relational/analyzer/TableFunctionTest.java | 4 +- .../relational/planner/CteMaterializerTest.java | 344 +++ .../plan/relational/planner/CteSubqueryTest.java | 168 ++ .../plan/relational/planner/PlanTester.java | 39 +- .../planner/UncorrelatedSubqueryTest.java | 261 +- .../planner/assertions/AliasMatcher.java | 8 + .../planner/assertions/ColumnReference.java | 23 +- .../planner/assertions/CteScanMatcher.java | 86 + .../planner/assertions/DeviceTableScanMatcher.java | 11 + .../planner/assertions/PlanMatchPattern.java | 48 +- .../planner/assertions/RvalueMatcher.java | 8 + .../planner/assertions/TableScanMatcher.java | 17 + .../informationschema/CurrentQueriesTest.java | 184 ++ .../informationschema}/ShowQueriesTest.java | 8 +- .../predicate/TablePredicateConversionTest.java | 506 ++++ .../relational/sql/AstMemoryEstimatorTest.java | 892 +++++++ .../scalar/BlobLengthColumnTransformerTest.java | 22 +- .../unary/scalar/ObjectTypeFunctionTest.java | 186 ++ .../compaction/CompactionDataTypeAlterTest.java | 322 +++ ...ctionDataTypeNotMatchAlterableDataTypeTest.java | 221 ++ .../compaction/CompactionSchedulerTest.java | 35 + .../FastNonAlignedCrossCompactionTest.java | 5 + .../ReadPointNonAlignedCrossCompactionTest.java | 4 + .../object/ObjectTypeCompactionTest.java | 459 ++++ .../memtable/WritableMemChunkRegionScanTest.java | 86 + .../wal/allocation/FirstCreateStrategyTest.java | 81 + .../db/utils/SchemaRegionSnapshotParserTest.java | 2 +- .../org/apache/iotdb/db/utils/SchemaUtilsTest.java | 39 + .../db/utils/datastructure/AlignedTVListTest.java | 10 +- .../db/utils/datastructure/IntTVListTest.java | 11 +- .../datastructure/PrimitiveArrayManagerTest.java | 1 + iotdb-core/metrics/core/pom.xml | 4 +- iotdb-core/metrics/interface/pom.xml | 8 +- iotdb-core/metrics/pom.xml | 2 +- iotdb-core/node-commons/pom.xml | 24 +- .../conf/iotdb-system.properties.template | 31 + .../iotdb/commons/audit/AbstractAuditLogger.java | 89 +- .../iotdb/commons/auth/entity/PrivilegeUnion.java | 2 +- .../iotdb/commons/client/ClientPoolFactory.java | 1 + .../iotdb/commons/concurrent/ThreadName.java | 1 + .../apache/iotdb/commons/conf/CommonConfig.java | 22 + .../iotdb/commons/conf/CommonDescriptor.java | 6 + .../apache/iotdb/commons/conf/IoTDBConstant.java | 6 + .../commons/exception/ObjectFileNotExist.java} | 16 +- .../apache/iotdb/commons/path/MeasurementPath.java | 31 + .../iotdb/commons/pipe/config/PipeDescriptor.java | 5 + .../pipe/config/constant/PipeSinkConstant.java | 49 +- .../datastructure/pattern/IoTDBTreePattern.java | 26 + .../pipe/datastructure/pattern/TreePattern.java | 490 +++- .../pattern/UnionIoTDBTreePattern.java | 6 + .../pattern/WithExclusionIoTDBTreePattern.java | 2 - .../pattern/WithExclusionTreePattern.java | 2 - .../commons/pipe/receiver/IoTDBFileReceiver.java | 11 +- .../pipe/source/IoTDBNonDataRegionSource.java | 14 +- .../iotdb/commons/pipe/source/IoTDBSource.java | 2 + .../iotdb/commons/schema/column/ColumnHeader.java | 14 +- .../schema/column/ColumnHeaderConstant.java | 10 +- .../iotdb/commons/schema/filter/SchemaFilter.java | 3 +- .../commons/schema/filter/impl/DataTypeFilter.java | 9 + .../schema/filter/impl/PathContainsFilter.java | 10 + .../commons/schema/filter/impl/TagFilter.java | 9 + .../commons/schema/filter/impl/TemplateFilter.java | 10 + .../commons/schema/filter/impl/ViewTypeFilter.java | 10 + .../multichildren/AbstractMultiChildrenFilter.java | 25 + .../filter/impl/multichildren/AndFilter.java | 9 + .../schema/filter/impl/multichildren/OrFilter.java | 9 + .../singlechild/AbstractSingleChildFilter.java | 16 + .../filter/impl/singlechild/AttributeFilter.java | 10 + .../schema/filter/impl/singlechild/NotFilter.java | 9 + .../schema/filter/impl/singlechild/TagFilter.java | 10 + .../filter/impl/values/ComparisonFilter.java | 10 + .../schema/filter/impl/values/InFilter.java | 8 + .../schema/filter/impl/values/LikeFilter.java | 11 + .../schema/filter/impl/values/PreciseFilter.java | 10 + .../table/AlterOrDropTableOperationType.java | 5 +- .../commons/schema/table/InformationSchema.java | 43 +- .../apache/iotdb/commons/schema/table/TsTable.java | 66 +- .../schema/table/column/AttributeColumnSchema.java | 7 + .../schema/table/column/FieldColumnSchema.java | 14 +- .../schema/table/column/TagColumnSchema.java | 6 + .../schema/table/column/TimeColumnSchema.java | 6 + .../schema/table/column/TsTableColumnSchema.java | 6 + .../table/column/TsTableColumnSchemaUtil.java | 12 + .../iotdb/commons/schema}/template/Template.java | 2 +- .../schema/tree/AlterTimeSeriesOperationType.java} | 29 +- .../apache/iotdb/commons/udf/builtin/UDTFAbs.java | 3 + .../iotdb/commons/udf/builtin/UDTFBottomK.java | 1 + .../commons/udf/builtin/UDTFCommonDerivative.java | 1 + .../udf/builtin/UDTFCommonValueDifference.java | 1 + .../iotdb/commons/udf/builtin/UDTFConst.java | 5 + .../udf/builtin/UDTFContinuouslySatisfy.java | 2 + .../udf/builtin/UDTFEqualSizeBucketAggSample.java | 1 + .../udf/builtin/UDTFEqualSizeBucketM4Sample.java | 1 + .../builtin/UDTFEqualSizeBucketOutlierSample.java | 1 + .../builtin/UDTFEqualSizeBucketRandomSample.java | 1 + .../iotdb/commons/udf/builtin/UDTFInRange.java | 3 + .../apache/iotdb/commons/udf/builtin/UDTFJexl.java | 3 + .../apache/iotdb/commons/udf/builtin/UDTFM4.java | 1 + .../apache/iotdb/commons/udf/builtin/UDTFMath.java | 3 + .../udf/builtin/UDTFNonNegativeDerivative.java | 1 + .../builtin/UDTFNonNegativeValueDifference.java | 1 + .../iotdb/commons/udf/builtin/UDTFOnOff.java | 1 + .../iotdb/commons/udf/builtin/UDTFSelectK.java | 2 + .../apache/iotdb/commons/udf/builtin/UDTFTopK.java | 1 + .../iotdb/commons/udf/builtin/UDTFValueTrend.java | 1 + .../relational/TableBuiltinScalarFunction.java | 1 + .../commons/udf/utils/UDFDataTypeTransformer.java | 7 + .../apache/iotdb/commons/utils/MetadataUtils.java} | 15 +- .../apache/iotdb/commons/utils/SerializeUtils.java | 6 + .../apache/iotdb/commons/utils/WindowsOSUtils.java | 64 + .../iotdb/commons/utils/WindowsOSUtilsTest.java} | 40 +- iotdb-core/pom.xml | 2 +- iotdb-core/relational-grammar/pom.xml | 2 +- .../db/relational/grammar/sql/RelationalSql.g4 | 3 +- iotdb-protocol/openapi/pom.xml | 2 +- iotdb-protocol/pom.xml | 2 +- iotdb-protocol/thrift-ainode/pom.xml | 4 +- .../thrift-ainode/src/main/thrift/ainode.thrift | 25 +- iotdb-protocol/thrift-commons/pom.xml | 2 +- iotdb-protocol/thrift-confignode/pom.xml | 4 +- .../src/main/thrift/confignode.thrift | 15 + iotdb-protocol/thrift-consensus/pom.xml | 4 +- iotdb-protocol/thrift-datanode/pom.xml | 4 +- .../thrift-datanode/src/main/thrift/client.thrift | 2 +- .../src/main/thrift/datanode.thrift | 22 +- library-udf/pom.xml | 4 +- pom.xml | 14 +- 1015 files changed, 42757 insertions(+), 6263 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index 280d18df76e,b8fd3a094db..cbb50c6b293 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@@ -70,7 -68,8 +70,9 @@@ import org.apache.iotdb.db.protocol.cli import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.queryengine.common.ConnectionInfo; + import org.apache.iotdb.db.queryengine.common.QueryId; + import org.apache.iotdb.db.queryengine.execution.QueryState; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; import org.apache.iotdb.db.queryengine.plan.Coordinator; import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowCreateViewTask; @@@ -140,13 -133,11 +145,14 @@@ public class InformationSchemaContentSu private InformationSchemaContentSupplierFactory() {} - public static Iterator<TsBlock> getSupplier( + public static IInformationSchemaContentSupplier getSupplier( + final OperatorContext context, final String tableName, final List<TSDataType> dataTypes, - Expression predicate, - final UserEntity userEntity) { ++ final Expression predicate, + final UserEntity userEntity, + final Filter pushDownFilter, + final PaginationController paginationController) { try { switch (tableName) { case InformationSchema.QUERIES: @@@ -181,11 -172,12 +187,15 @@@ return new ConfigNodesSupplier(dataTypes, userEntity); case InformationSchema.DATA_NODES: return new DataNodesSupplier(dataTypes, userEntity); + case InformationSchema.TABLE_DISK_USAGE: + return new TableDiskUsageSupplier( + dataTypes, userEntity, pushDownFilter, paginationController, context); case InformationSchema.CONNECTIONS: return new ConnectionsSupplier(dataTypes, userEntity); + case InformationSchema.CURRENT_QUERIES: + return new CurrentQueriesSupplier(dataTypes, predicate, userEntity); + case InformationSchema.QUERIES_COSTS_HISTOGRAM: + return new QueriesCostsHistogramSupplier(dataTypes, userEntity); default: throw new UnsupportedOperationException("Unknown table: " + tableName); } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java index 72a7afdee3b,37603ee2dbc..92462e45f3c --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java @@@ -57,9 -58,7 +58,8 @@@ import org.apache.iotdb.db.queryengine. import org.apache.iotdb.db.queryengine.plan.statement.component.SortItem; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; - import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 8ff0372b200,4cae2c0f288..fa5b5c22f9b --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@@ -135,10 -137,8 +137,9 @@@ import org.apache.iotdb.db.queryengine. import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; - import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index a4ceaf29274,2274869c355..7cf4be93c89 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@@ -85,9 -87,7 +87,8 @@@ import org.apache.iotdb.db.queryengine. import org.apache.iotdb.db.queryengine.plan.statement.metadata.view.ShowLogicalViewStatement; import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; - import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index ba157772439,407b3154b1f..021c123ff66 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@@ -1312,9 -1312,9 +1332,10 @@@ public class TableOperatorGenerator ext operatorContext, node.getPlanNodeId(), getSupplier( + operatorContext, node.getQualifiedObjectName().getObjectName(), dataTypes, + node.getPushDownPredicate(), context .getDriverContext() .getFragmentInstanceContext() diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index bfe1b8737cf,d442d51a214..33173336fa4 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@@ -313,27 -385,17 +385,21 @@@ public class RelationPlanner extends As outerContext); } - TableScanNode tableScanNode = - qualifiedObjectName.getDatabaseName().equals(INFORMATION_DATABASE) - ? new InformationSchemaTableScanNode( - idAllocator.genPlanNodeId(), qualifiedObjectName, outputSymbols, tableColumnSchema) - : new DeviceTableScanNode( - idAllocator.genPlanNodeId(), - qualifiedObjectName, - outputSymbols, - tableColumnSchema, - tagAndAttributeIndexMap); + TableScanNode tableScanNode; + if (qualifiedObjectName.getDatabaseName().equals(INFORMATION_DATABASE)) { + tableScanNode = + new InformationSchemaTableScanNode( + idAllocator.genPlanNodeId(), qualifiedObjectName, outputSymbols, tableColumnSchema); + } else { + tableScanNode = + new DeviceTableScanNode( + idAllocator.genPlanNodeId(), + qualifiedObjectName, + outputSymbols, + tableColumnSchema, + tagAndAttributeIndexMap); + } return new RelationPlan(tableScanNode, scope, outputSymbols, outerContext); - - // Collection<Field> fields = analysis.getMaterializedViewStorageTableFields(node); - // Query namedQuery = analysis.getNamedQuery(node); - // Collection<Field> fields = analysis.getMaterializedViewStorageTableFields(node); - // plan = addRowFilters(node, plan); - // plan = addColumnMasks(node, plan); } @Override diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java index 3c88590b98f,d7d755ddc1d..14ea965349a --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java @@@ -85,8 -85,9 +85,10 @@@ public class DataNodeLocationSupplierFa public List<TDataNodeLocation> getDataNodeLocations(final String tableName) { switch (tableName) { case InformationSchema.QUERIES: + case InformationSchema.TABLE_DISK_USAGE: case InformationSchema.CONNECTIONS: + case InformationSchema.CURRENT_QUERIES: + case InformationSchema.QUERIES_COSTS_HISTOGRAM: return getReadableDataNodeLocations(); case InformationSchema.DATABASES: case InformationSchema.TABLES: diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 363d8f7da87,55905f67682..fc6c47090fb --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@@ -449,9 -450,96 +452,91 @@@ public class PushPredicateIntoTableSca if (TRUE_LITERAL.equals(context.inheritedPredicate)) { return node; } + + // push-down for CURRENT_QUERIES + if (CURRENT_QUERIES.equals(node.getQualifiedObjectName().getObjectName())) { + SplitExpression splitExpression = splitCurrentQueriesPredicate(context.inheritedPredicate); + // exist expressions can push down to scan operator + if (!splitExpression.getExpressionsCanPushDown().isEmpty()) { + List<Expression> expressions = splitExpression.getExpressionsCanPushDown(); + checkState(expressions.size() == 1, "Unexpected number of expressions in table scan"); + node.setPushDownPredicate(expressions.get(0)); + } + + // exist expressions cannot push down + if (!splitExpression.getExpressionsCannotPushDown().isEmpty()) { + List<Expression> expressions = splitExpression.getExpressionsCannotPushDown(); + return new FilterNode( + queryId.genPlanNodeId(), + node, + expressions.size() == 1 + ? expressions.get(0) + : new LogicalExpression(LogicalExpression.Operator.AND, expressions)); + } + return node; + } - - FilterNode filterNode = - new FilterNode(queryId.genPlanNodeId(), node, context.inheritedPredicate); - context.inheritedPredicate = TRUE_LITERAL; - return filterNode; + return combineFilterAndScan(node, context.inheritedPredicate); } + private SplitExpression splitCurrentQueriesPredicate(Expression predicate) { + List<Expression> expressionsCanPushDown = new ArrayList<>(); + List<Expression> expressionsCannotPushDown = new ArrayList<>(); + + if (predicate instanceof LogicalExpression + && ((LogicalExpression) predicate).getOperator() == LogicalExpression.Operator.AND) { + + // predicate like state = 'xxx' can be push down + // Note: the optimizer CanonicalizeExpressionRewriter will ensure the predicate like 'xxx' = + // state will be canonicalized to state = 'xxx' + boolean hasExpressionPushDown = false; + for (Expression expression : ((LogicalExpression) predicate).getTerms()) { + if (isStateComparedWithConstant(expression) && !hasExpressionPushDown) { + // if there are more than one state = 'xxx' terms, only add first to push-down candidate + expressionsCanPushDown.add(expression); + hasExpressionPushDown = true; + } else { + expressionsCannotPushDown.add(expression); + } + } + + return new SplitExpression( + Collections.emptyList(), expressionsCanPushDown, expressionsCannotPushDown, null); + } + + if (isStateComparedWithConstant(predicate)) { + expressionsCanPushDown.add(predicate); + } else { + expressionsCannotPushDown.add(predicate); + } + + return new SplitExpression( + Collections.emptyList(), expressionsCanPushDown, expressionsCannotPushDown, null); + } + + private boolean isStateComparedWithConstant(Expression expression) { + if (!(expression instanceof ComparisonExpression)) { + return false; + } + + ComparisonExpression comparisonExpression = (ComparisonExpression) expression; + + if (ComparisonExpression.Operator.EQUAL != comparisonExpression.getOperator()) { + return false; + } + + if (!(comparisonExpression.getLeft() instanceof SymbolReference) + || !STATE_TABLE_MODEL.equals( + ((SymbolReference) comparisonExpression.getLeft()).getName())) { + return false; + } + + if (!(comparisonExpression.getRight() instanceof StringLiteral)) { + return false; + } + + return true; + } + @Override public PlanNode visitDeviceTableScan( DeviceTableScanNode tableScanNode, RewriteContext context) { diff --cc iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java index 93ebba65222,4d27792e532..168923c7a42 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java @@@ -63,7 -63,7 +63,8 @@@ import org.apache.iotdb.db.queryengine. import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.UnsetSchemaTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.view.CreateLogicalViewStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.AuthorStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowDiskUsageStatement; + import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.isession.template.TemplateNode; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.service.rpc.thrift.TSAggregationQueryReq; @@@ -120,35 -120,35 +121,66 @@@ import static org.junit.Assert.assertTr public class StatementGeneratorTest { + @Test - public void testShowQueries() { ++ public void testShowDiskUsage() { ++ + Statement showDiskUsage = + StatementGenerator.createStatement( + "show disk_usage from root.test.** order by database, datanodeid, regionid, timepartition, sizeinbytes", + ZonedDateTime.now().getOffset()); + Assert.assertTrue(showDiskUsage instanceof ShowDiskUsageStatement); + Assert.assertEquals( + ((ShowDiskUsageStatement) showDiskUsage).getSortItemList().get(0), + new SortItem(OrderByKey.DATABASE, Ordering.ASC)); + Assert.assertEquals( + ((ShowDiskUsageStatement) showDiskUsage).getSortItemList().get(1), + new SortItem(OrderByKey.DATANODEID, Ordering.ASC)); + Assert.assertEquals( + ((ShowDiskUsageStatement) showDiskUsage).getSortItemList().get(2), + new SortItem(OrderByKey.REGIONID, Ordering.ASC)); + Assert.assertEquals( + ((ShowDiskUsageStatement) showDiskUsage).getSortItemList().get(3), + new SortItem(OrderByKey.TIMEPARTITION, Ordering.ASC)); + Assert.assertEquals( + ((ShowDiskUsageStatement) showDiskUsage).getSortItemList().get(4), + new SortItem(OrderByKey.SIZEINBYTES, Ordering.ASC)); ++ + Assert.assertThrows( + SemanticException.class, + () -> + StatementGenerator.createStatement( + "show disk_usage from root.test.** order by a", ZonedDateTime.now().getOffset())); + } + + @Test + public void testShowQueries() { + Statement showQueries = + StatementGenerator.createStatement( + "show queries order by time, queryid, datanodeid, elapsedtime, statement", + ZonedDateTime.now().getOffset()); + Assert.assertTrue(showQueries instanceof ShowQueriesStatement); + Assert.assertEquals( + ((ShowQueriesStatement) showQueries).getSortItemList().get(0), + new SortItem(OrderByKey.TIME, Ordering.ASC)); + Assert.assertEquals( + ((ShowQueriesStatement) showQueries).getSortItemList().get(1), + new SortItem(OrderByKey.QUERYID, Ordering.ASC)); + Assert.assertEquals( + ((ShowQueriesStatement) showQueries).getSortItemList().get(2), + new SortItem(OrderByKey.DATANODEID, Ordering.ASC)); + Assert.assertEquals( + ((ShowQueriesStatement) showQueries).getSortItemList().get(3), + new SortItem(OrderByKey.ELAPSEDTIME, Ordering.ASC)); + Assert.assertEquals( + ((ShowQueriesStatement) showQueries).getSortItemList().get(4), + new SortItem(OrderByKey.STATEMENT, Ordering.ASC)); + Assert.assertThrows( + SemanticException.class, + () -> + StatementGenerator.createStatement( + "show queries order by a", ZonedDateTime.now().getOffset())); + } + @Test public void testRawDataQuery() throws IllegalPathException { TSRawDataQueryReq req = diff --cc iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/source/SourceNodeSerdeTest.java index 4d0ce6e362d,068b2bbc4b6..b61f4c24cd8 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/source/SourceNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/source/SourceNodeSerdeTest.java @@@ -24,7 -24,7 +24,8 @@@ import org.apache.iotdb.commons.path.Pa import org.apache.iotdb.db.queryengine.plan.planner.node.PlanNodeDeserializeHelper; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.LastQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowDiskUsageNode; + import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.ShowQueriesNode; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.schema.MeasurementSchema; @@@ -71,19 -71,18 +72,34 @@@ public class SourceNodeSerdeTest assertEquals(PlanNodeDeserializeHelper.deserialize(byteBuffer), node); } + @Test + public void testShowDiskUsageNode() throws IllegalPathException { + ShowDiskUsageNode node = + new ShowDiskUsageNode(new PlanNodeId("test"), null, new PartialPath("root.test.d1")); + + ByteBuffer byteBuffer = ByteBuffer.allocate(2048); + node.serialize(byteBuffer); + byteBuffer.flip(); + assertEquals(PlanNodeDeserializeHelper.deserialize(byteBuffer), node); + node = new ShowDiskUsageNode(new PlanNodeId("test"), null, new PartialPath("root.test.d1")); + byteBuffer = ByteBuffer.allocate(2048); + node.serialize(byteBuffer); + byteBuffer.flip(); + assertEquals(PlanNodeDeserializeHelper.deserialize(byteBuffer), node); + } ++ + @Test + public void testShowQueriesNode() throws IllegalPathException { + ShowQueriesNode node = new ShowQueriesNode(new PlanNodeId("test"), null, "root"); + + ByteBuffer byteBuffer = ByteBuffer.allocate(2048); + node.serialize(byteBuffer); + byteBuffer.flip(); + assertEquals(PlanNodeDeserializeHelper.deserialize(byteBuffer), node); + node = new ShowQueriesNode(new PlanNodeId("test"), null, "root"); + byteBuffer = ByteBuffer.allocate(2048); + node.serialize(byteBuffer); + byteBuffer.flip(); + assertEquals(PlanNodeDeserializeHelper.deserialize(byteBuffer), node); + } } diff --cc iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java index 58733c7d468,4f458d34e05..63f95170a07 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java @@@ -57,8 -49,9 +57,10 @@@ public class InformationSchema public static final String NODES = "nodes"; public static final String CONFIG_NODES = "config_nodes"; public static final String DATA_NODES = "data_nodes"; + public static final String TABLE_DISK_USAGE = "table_disk_usage"; public static final String CONNECTIONS = "connections"; + public static final String CURRENT_QUERIES = "current_queries"; + public static final String QUERIES_COSTS_HISTOGRAM = "queries_costs_histogram"; static { final TsTable queriesTable = new TsTable(QUERIES); @@@ -386,21 -361,41 +386,54 @@@ new AttributeColumnSchema(ColumnHeaderConstant.CLIENT_IP, TSDataType.STRING)); connectionsTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(CONNECTIONS, connectionsTable); + + final TsTable currentQueriesTable = new TsTable(CURRENT_QUERIES); + currentQueriesTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.QUERY_ID_TABLE_MODEL, TSDataType.STRING)); + currentQueriesTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.STATE_TABLE_MODEL, TSDataType.STRING)); + currentQueriesTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.START_TIME_TABLE_MODEL, TSDataType.TIMESTAMP)); + currentQueriesTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.END_TIME_TABLE_MODEL, TSDataType.TIMESTAMP)); + currentQueriesTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.DATA_NODE_ID_TABLE_MODEL, TSDataType.INT32)); + currentQueriesTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.COST_TIME, TSDataType.FLOAT)); + currentQueriesTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.STATEMENT_TABLE_MODEL, TSDataType.STRING)); + currentQueriesTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.USER_TABLE_MODEL, TSDataType.STRING)); + currentQueriesTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.CLIENT_IP, TSDataType.STRING)); + currentQueriesTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); + schemaTables.put(CURRENT_QUERIES, currentQueriesTable); + + final TsTable queriesCostsHistogramTable = new TsTable(QUERIES_COSTS_HISTOGRAM); + queriesCostsHistogramTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.BIN, TSDataType.STRING)); + queriesCostsHistogramTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.NUMS, TSDataType.INT32)); + queriesCostsHistogramTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.DATANODE_ID, TSDataType.INT32)); + queriesCostsHistogramTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); + schemaTables.put(QUERIES_COSTS_HISTOGRAM, queriesCostsHistogramTable); } + static { + columnsThatSupportPushDownPredicate.put( + TABLE_DISK_USAGE, + ImmutableSet.of( + ColumnHeaderConstant.DATABASE, + ColumnHeaderConstant.TABLE_NAME_TABLE_MODEL, + ColumnHeaderConstant.DATA_NODE_ID_TABLE_MODEL, + ColumnHeaderConstant.REGION_ID_TABLE_MODEL, + ColumnHeaderConstant.TIME_PARTITION_TABLE_MODEL)); + + tablesThatSupportPushDownLimitOffset.add(TABLE_DISK_USAGE); + } + public static Map<String, TsTable> getSchemaTables() { return schemaTables; }
