This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch table_disk_usage_statistics_with_cache in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 03b0f2cf1b54dd45bb3e1a13e5d2c56f7197a12d Merge: 880718f393a 5d1efef31da Author: shuwenwei <[email protected]> AuthorDate: Wed Dec 10 18:57:50 2025 +0800 Merge branch 'iotdb_master' into table_disk_usage_statistics .github/workflows/cluster-it-1c1d1a.yml | 3 - LICENSE-binary | 4 +- dependencies.json | 16 +- .../iotdb/it/env/cluster/node/AINodeWrapper.java | 2 +- .../iotdb/itbase/runtime/ClusterTestStatement.java | 7 + .../iotdb/ainode/it/AINodeCallInferenceIT.java | 117 + .../ainode/it/AINodeConcurrentForecastIT.java | 113 + .../ainode/it/AINodeConcurrentInferenceIT.java | 187 -- .../apache/iotdb/ainode/it/AINodeForecastIT.java | 98 + .../iotdb/ainode/it/AINodeInferenceSQLIT.java | 344 --- .../ainode/it/AINodeInstanceManagementIT.java | 79 +- .../iotdb/ainode/it/AINodeModelManageIT.java | 53 +- .../apache/iotdb/ainode/utils/AINodeTestUtils.java | 119 +- .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 145 ++ .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 16 + .../it/schema/IoTDBCreateAlignedTimeseriesIT.java | 22 + .../db/it/schema/IoTDBCreateTimeseriesIT.java | 8 + .../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 13 +- .../iotdb/db/it/utils/TSDataTypeTestUtils.java | 113 + .../pipe/it/dual/tablemodel/TableModelUtils.java | 8 +- .../treemodel/auto/basic/IoTDBPipeSyntaxIT.java | 61 + .../pipe/it/single/IoTDBPipePermissionIT.java | 27 +- .../relational/it/db/it/IoTDBDeletionTableIT.java | 6 +- .../relational/it/db/it/IoTDBInsertTableIT.java | 6 + .../it/db/it/IoTDBPreparedStatementIT.java | 385 ++++ .../scalar/IoTDBScalarFunctionTableIT.java | 23 +- .../it/query/recent/IoTDBExceptTableIT.java | 159 ++ .../it/query/recent/IoTDBLengthFunctionIT.java | 119 + .../relational/it/schema/IoTDBDatabaseIT.java | 22 +- .../iotdb/relational/it/schema/IoTDBDeviceIT.java | 44 +- .../iotdb/relational/it/schema/IoTDBTableIT.java | 16 + .../it/session/IoTDBSessionRelationalIT.java | 12 +- .../iotdb/session/it/IoTDBConnectionsIT.java | 364 +++ .../src/test/resources/ainode-example/config.yaml | 5 - .../src/test/resources/ainode-example/model.pt | Bin 1906 -> 0 bytes .../apache/iotdb/rpc/TElasticFramedTransport.java | 5 + .../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 - iotdb-client/session/pom.xml | 14 + .../org/apache/iotdb/session/NodesSupplier.java | 49 +- iotdb-core/ainode/.gitignore | 3 + iotdb-core/ainode/ainode.spec | 138 +- iotdb-core/ainode/build_binary.py | 18 +- iotdb-core/ainode/iotdb/ainode/core/config.py | 33 +- iotdb-core/ainode/iotdb/ainode/core/constant.py | 173 +- iotdb-core/ainode/iotdb/ainode/core/exception.py | 2 +- .../ainode/core/inference/inference_request.py | 22 +- .../core/inference/inference_request_pool.py | 116 +- .../inference/{strategy => pipeline}/__init__.py | 0 .../core/inference/pipeline/basic_pipeline.py | 87 + .../core/inference/pipeline/pipeline_loader.py | 56 + .../iotdb/ainode/core/inference/pool_controller.py | 83 +- .../pool_scheduler/basic_pool_scheduler.py | 2 +- .../strategy/abstract_inference_pipeline.py | 60 - .../ainode/iotdb/ainode/core/inference/utils.py | 45 +- .../iotdb/ainode/core/manager/inference_manager.py | 172 +- .../iotdb/ainode/core/manager/model_manager.py | 162 +- .../ainode/iotdb/ainode/core/manager/utils.py | 7 +- .../ainode/core/model/built_in_model_factory.py | 1238 ----------- .../__init__.py => model/model_constants.py} | 31 + .../ainode/iotdb/ainode/core/model/model_enums.py | 70 - .../iotdb/ainode/core/model/model_factory.py | 291 --- .../ainode/iotdb/ainode/core/model/model_info.py | 130 +- .../ainode/iotdb/ainode/core/model/model_loader.py | 156 ++ .../iotdb/ainode/core/model/model_storage.py | 808 ++++--- .../core/model/{timerxl => sktime}/__init__.py | 0 .../ainode/core/model/sktime/arima/config.json | 25 + .../core/model/sktime/configuration_sktime.py | 409 ++++ .../model/sktime/exponential_smoothing/config.json | 11 + .../core/model/sktime/gaussian_hmm/config.json | 22 + .../ainode/core/model/sktime/gmm_hmm/config.json | 24 + .../ainode/core/model/sktime/modeling_sktime.py | 180 ++ .../core/model/sktime/naive_forecaster/config.json | 9 + .../ainode/core/model/sktime/pipeline_sktime.py | 68 + .../core/model/sktime/stl_forecaster/config.json | 22 + .../ainode/core/model/sktime/stray/config.json | 11 + .../ainode/core/model/sundial/modeling_sundial.py | 8 +- .../sundial/pipeline_sundial.py} | 42 +- .../strategy => model/timer_xl}/__init__.py | 0 .../{timerxl => timer_xl}/configuration_timer.py | 0 .../model/{timerxl => timer_xl}/modeling_timer.py | 9 +- .../timer_xl/pipeline_timer.py} | 36 +- .../{timerxl => timer_xl}/ts_generation_mixin.py | 0 .../ainode/iotdb/ainode/core/model/uri_utils.py | 137 -- iotdb-core/ainode/iotdb/ainode/core/model/utils.py | 98 + iotdb-core/ainode/iotdb/ainode/core/rpc/client.py | 39 - iotdb-core/ainode/iotdb/ainode/core/rpc/handler.py | 47 +- iotdb-core/ainode/poetry.lock | 2331 -------------------- iotdb-core/ainode/pyproject.toml | 9 +- .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 2 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 7 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 8 + iotdb-core/confignode/pom.xml | 2 +- .../async/AsyncAINodeHeartbeatClientPool.java | 14 +- .../async/AsyncDataNodeHeartbeatClientPool.java | 1 - .../consensus/request/ConfigPhysicalPlan.java | 24 - .../request/read/model/GetModelInfoPlan.java | 64 - .../request/read/model/ShowModelPlan.java | 70 - .../request/write/model/CreateModelPlan.java | 79 - .../request/write/model/DropModelInNodePlan.java | 70 - .../request/write/model/DropModelPlan.java | 79 - .../request/write/model/UpdateModelInfoPlan.java | 122 - .../consensus/response/model/GetModelInfoResp.java | 63 - .../consensus/response/model/ModelTableResp.java | 62 - .../iotdb/confignode/manager/ConfigManager.java | 179 -- .../apache/iotdb/confignode/manager/IManager.java | 42 - .../iotdb/confignode/manager/ModelManager.java | 244 -- .../iotdb/confignode/manager/ProcedureManager.java | 20 - .../pipe/agent/task/PipeConfigNodeSubtask.java | 6 +- .../sink/protocol/IoTDBConfigRegionAirGapSink.java | 6 +- .../pipe/sink/protocol/IoTDBConfigRegionSink.java | 6 +- .../iotdb/confignode/persistence/ModelInfo.java | 378 ---- .../persistence/executor/ConfigPlanExecutor.java | 25 - .../persistence/pipe/PipePluginInfo.java | 78 +- .../procedure/CompletedProcedureRecycler.java | 21 +- .../iotdb/confignode/procedure/Procedure.java | 27 +- .../confignode/procedure/ProcedureExecutor.java | 88 +- .../procedure/TimeoutExecutorThread.java | 12 +- .../procedure/impl/model/CreateModelProcedure.java | 250 --- .../procedure/impl/model/DropModelProcedure.java | 199 -- .../procedure/impl/node/RemoveAINodeProcedure.java | 17 +- .../impl/pipe/task/AlterPipeProcedureV2.java | 1 + .../procedure/state/RemoveAINodeState.java | 1 - .../procedure/store/ConfigProcedureStore.java | 26 +- .../procedure/store/IProcedureStore.java | 10 +- .../procedure/store/ProcedureFactory.java | 12 - .../confignode/procedure/store/ProcedureType.java | 2 + .../thrift/ConfigNodeRPCServiceProcessor.java | 94 +- iotdb-core/consensus/pom.xml | 14 +- iotdb-core/datanode/pom.xml | 4 +- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 9 +- .../apache/iotdb/db/conf/DataNodeMemoryConfig.java | 10 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 67 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 30 +- .../db/consensus/DataRegionConsensusImpl.java | 6 + .../dataregion/DataRegionStateMachine.java | 15 +- .../schemaregion/SchemaExecutionVisitor.java | 44 +- .../subtask/processor/PipeProcessorSubtask.java | 7 + .../agent/task/subtask/sink/PipeSinkSubtask.java | 11 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 73 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 2 +- .../protocol/airgap/IoTDBAirGapReceiverAgent.java | 5 +- .../protocol/thrift/IoTDBDataNodeReceiver.java | 8 +- .../visitor/PipeStatementExceptionVisitor.java | 8 +- .../visitor/PipeStatementTSStatusVisitor.java | 3 +- ...leStatementDataTypeConvertExecutionVisitor.java | 4 +- ...eeStatementDataTypeConvertExecutionVisitor.java | 4 +- .../resource/memory/InsertNodeMemoryEstimator.java | 55 +- .../pipe/resource/memory/PipeMemoryWeightUtil.java | 62 +- .../request/PipeTransferTabletBatchReqV2.java | 7 +- .../request/PipeTransferTabletRawReq.java | 110 +- .../request/PipeTransferTabletRawReqV2.java | 50 +- .../airgap/IoTDBSchemaRegionAirGapSink.java | 6 +- .../handler/PipeConsensusDeleteEventHandler.java | 2 +- .../handler/PipeTransferTrackableHandler.java | 16 +- .../async/handler/PipeTransferTsFileHandler.java | 15 +- .../thrift/sync/IoTDBDataRegionSyncSink.java | 3 +- .../thrift/sync/IoTDBSchemaRegionSink.java | 6 +- .../pipe/sink/util/TabletStatementConverter.java | 476 ++++ .../sink/util/sorter/InsertEventDataAdapter.java | 127 ++ .../util/sorter/InsertTabletStatementAdapter.java | 118 + ...EventSorter.java => PipeInsertEventSorter.java} | 99 +- .../sorter/PipeTableModelTabletEventSorter.java | 67 +- .../sorter/PipeTreeModelTabletEventSorter.java | 48 +- .../db/pipe/sink/util/sorter/TabletAdapter.java | 113 + .../iotdb/db/protocol/client/ConfigNodeClient.java | 79 +- .../protocol/client/DataNodeClientPoolFactory.java | 42 +- .../iotdb/db/protocol/client/an/AINodeClient.java | 321 +++ .../protocol/client/an}/AINodeClientManager.java | 25 +- .../iotdb/db/protocol/session/ClientSession.java | 23 + .../iotdb/db/protocol/session/IClientSession.java | 47 + .../db/protocol/session/InternalClientSession.java | 24 + .../db/protocol/session/MqttClientSession.java | 24 + .../db/protocol/session/PreparedStatementInfo.java | 99 + .../db/protocol/session/RestClientSession.java | 25 + .../iotdb/db/protocol/session/SessionManager.java | 55 +- .../thrift/handler/BaseServerContextHandler.java | 1 + .../protocol/thrift/impl/ClientRPCServiceImpl.java | 320 ++- .../db/queryengine/common/ConnectionInfo.java | 64 + .../common/header/DatasetHeaderFactory.java | 4 + .../execution/executor/RegionWriteExecutor.java | 4 +- .../fragment/FragmentInstanceContext.java | 2 +- .../fragment/FragmentInstanceExecution.java | 2 +- .../operator/process/ai/InferenceOperator.java | 82 +- .../process/window/TableWindowOperator.java | 2 + .../InformationSchemaContentSupplierFactory.java | 159 +- .../relational/ColumnTransformerBuilder.java | 8 +- .../iotdb/db/queryengine/plan/Coordinator.java | 108 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 134 +- .../db/queryengine/plan/analyze/IModelFetcher.java | 4 - .../db/queryengine/plan/analyze/ModelFetcher.java | 51 +- .../analyze/cache/partition/PartitionCache.java | 2 +- .../dualkeycache/impl/CacheEntryGroupImpl.java | 6 + .../schema/dualkeycache/impl/DualKeyCacheImpl.java | 61 +- .../schema/dualkeycache/impl/ICacheEntryGroup.java | 3 + .../plan/analyze/load/LoadTsFileAnalyzer.java | 5 +- .../load/TreeSchemaAutoCreatorAndVerifier.java | 4 + .../analyze/schema/AutoCreateSchemaExecutor.java | 51 +- .../execution/config/TableConfigTaskVisitor.java | 39 +- .../execution/config/TreeConfigTaskVisitor.java | 8 + .../config/executor/ClusterConfigTaskExecutor.java | 169 +- .../config/executor/IConfigTaskExecutor.java | 2 + ...dModelsTask.java => ShowAvailableUrlsTask.java} | 40 +- .../config/metadata/ai/ShowAIDevicesTask.java | 2 +- .../config/metadata/ai/ShowLoadedModelsTask.java | 4 +- .../config/metadata/ai/ShowModelsTask.java | 4 +- .../execution/config/session/DeallocateTask.java | 72 + .../plan/execution/config/session/PrepareTask.java | 85 + .../session/PreparedStatementMemoryManager.java | 157 ++ .../db/queryengine/plan/parser/ASTVisitor.java | 6 + .../plan/planner/plan/node/PlanGraphPrinter.java | 10 + .../plan/planner/plan/node/PlanNodeType.java | 4 + .../plan/planner/plan/node/PlanVisitor.java | 5 + .../plan/node/load/LoadSingleTsFileNode.java | 6 +- .../plan/node/process/AI/InferenceNode.java | 3 +- .../parameter/model/ModelInferenceDescriptor.java | 61 +- .../relational/analyzer/StatementAnalyzer.java | 146 +- .../analyzer/StatementAnalyzerFactory.java | 17 +- .../function/tvf/ForecastTableFunction.java | 37 +- .../plan/relational/metadata/Metadata.java | 16 +- .../relational/metadata/TableMetadataImpl.java | 29 +- .../fetcher/TableHeaderSchemaValidator.java | 378 ++++ .../fetcher/cache/TableDeviceCacheEntry.java | 8 +- .../fetcher/cache/TableDeviceLastCache.java | 56 +- .../fetcher/cache/TableDeviceSchemaCache.java | 8 +- .../cache/TableDeviceSchemaCacheMetrics.java | 9 +- .../fetcher/cache/TreeDeviceNormalSchema.java | 6 +- .../plan/relational/planner/RelationPlanner.java | 24 +- .../plan/relational/planner/TableModelPlanner.java | 25 +- .../iterative/rule/EvaluateEmptyIntersect.java | 63 + ...istinctAsUnion.java => ImplementExceptAll.java} | 78 +- ...on.java => ImplementExceptDistinctAsUnion.java} | 52 +- .../rule/ImplementIntersectDistinctAsUnion.java | 3 +- .../planner/iterative/rule/MergeExcept.java | 47 + .../planner/iterative/rule/MergeIntersect.java | 47 + .../iterative/rule/PruneDistinctAggregation.java | 27 +- .../iterative/rule/PruneExceptSourceColumns.java | 54 + .../rule/PruneIntersectSourceColumns.java | 55 + .../iterative/rule/RemoveEmptyExceptBranches.java | 139 ++ .../planner/iterative/rule/SetOperationMerge.java | 123 +- .../iterative/rule/SetOperationNodeTranslator.java | 4 +- .../plan/relational/planner/node/ExceptNode.java | 99 + .../plan/relational/planner/node/Patterns.java | 20 +- .../DataNodeLocationSupplierFactory.java | 2 +- .../optimizations/LogicalOptimizeFactory.java | 43 +- .../optimizations/UnaliasSymbolReferences.java | 31 + .../security/TreeAccessCheckVisitor.java | 24 +- .../plan/relational/sql/AstMemoryEstimator.java | 67 + .../plan/relational/sql/ParameterExtractor.java | 121 + .../plan/relational/sql/ast/AstVisitor.java | 20 + .../plan/relational/sql/ast/Deallocate.java | 79 + .../plan/relational/sql/ast/Execute.java | 96 + .../plan/relational/sql/ast/ExecuteImmediate.java | 99 + .../plan/relational/sql/ast/InsertRows.java | 19 +- .../plan/relational/sql/ast/LoadTsFile.java | 61 +- .../plan/relational/sql/ast/Prepare.java | 87 + .../plan/relational/sql/ast/RenameColumn.java | 5 +- .../plan/relational/sql/ast/RenameTable.java | 5 +- .../ast/{Statement.java => ShowAvailableUrls.java} | 39 +- .../plan/relational/sql/ast/Statement.java | 25 + .../relational/sql/ast/WrappedInsertStatement.java | 414 +++- .../plan/relational/sql/parser/AstBuilder.java | 45 + .../scheduler/load/LoadTsFileDispatcherImpl.java | 12 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 6 +- .../db/queryengine/plan/statement/Statement.java | 23 + .../plan/statement/StatementVisitor.java | 6 + .../plan/statement/crud/InsertBaseStatement.java | 157 +- .../crud/InsertMultiTabletsStatement.java | 6 + .../plan/statement/crud/InsertRowStatement.java | 42 + .../crud/InsertRowsOfOneDeviceStatement.java | 6 + .../plan/statement/crud/InsertRowsStatement.java | 6 + .../plan/statement/crud/InsertTabletStatement.java | 265 ++- .../plan/statement/crud/LoadTsFileStatement.java | 50 +- .../metadata/ShowAvailableUrlsStatement.java} | 23 +- .../db/queryengine/plan/udf/UDTFForecast.java | 24 +- .../unary/scalar/BlobLengthColumnTransformer.java | 56 + .../rescon/MemSchemaEngineStatistics.java | 3 +- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 10 +- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 10 +- .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 18 +- .../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 21 +- .../req/impl/CreateAlignedTimeSeriesPlanImpl.java | 10 + .../write/req/impl/CreateTimeSeriesPlanImpl.java | 10 + .../db/schemaengine/table/DataNodeTableCache.java | 17 +- .../java/org/apache/iotdb/db/service/DataNode.java | 3 + .../iotdb/db/service/metrics/WritingMetrics.java | 3 +- .../iotdb/db/storageengine/StorageEngine.java | 13 +- .../db/storageengine/dataregion/DataRegion.java | 295 ++- .../dataregion/IDataRegionForQuery.java | 2 +- .../dataregion/VirtualDataRegion.java | 2 +- .../execute/task/InnerSpaceCompactionTask.java | 3 + .../execute/task/SettleCompactionTask.java | 5 + .../compaction/repair/RepairTimePartition.java | 2 +- .../dataregion/memtable/AbstractMemTable.java | 86 +- .../memtable/AlignedWritableMemChunk.java | 132 +- .../dataregion/memtable/IMemTable.java | 6 +- .../dataregion/memtable/TsFileProcessor.java | 36 +- .../dataregion/memtable/WritableMemChunk.java | 52 +- .../dataregion/snapshot/SnapshotTaker.java | 14 +- .../dataregion/tsfile/TsFileResource.java | 43 +- .../file/UnsealedTsFileRecoverPerformer.java | 6 +- .../db/storageengine/load/LoadTsFileManager.java | 40 +- .../storageengine/load/active/ActiveLoadAgent.java | 90 + .../load/active/ActiveLoadDirScanner.java | 20 +- .../load/active/ActiveLoadTsFileLoader.java | 6 +- ...leStatementDataTypeConvertExecutionVisitor.java | 10 +- ...eeStatementDataTypeConvertExecutionVisitor.java | 10 +- .../db/storageengine/load/disk/MinIOSelector.java | 10 +- .../ActiveLoadUtil.java => util/LoadUtil.java} | 49 +- .../rescon/memory/PrimitiveArrayManager.java | 1 + .../db/utils/datastructure/AlignedTVList.java | 20 +- .../apache/iotdb/db/auth/AuthorityCheckerTest.java | 51 + .../org/apache/iotdb/db/auth/TreeAccessTest.java | 84 + .../cache/TreeDeviceSchemaCacheManagerTest.java | 14 + .../db/metadata/path/MeasurementPathTest.java | 4 +- .../pipe/sink/PipeDataNodeThriftRequestTest.java | 4 +- .../db/pipe/sink/PipeStatementEventSorterTest.java | 313 +++ .../sink/util/TabletStatementConverterTest.java | 607 +++++ .../process/window/TableWindowOperatorTest.java | 51 + .../plan/relational/analyzer/AnalyzerTest.java | 40 +- .../plan/relational/analyzer/AuthTest.java | 10 +- .../analyzer/EvaluateEmptyIntersectTest.java | 48 + .../plan/relational/analyzer/ExceptTest.java | 132 ++ .../plan/relational/analyzer/MergeExceptTest.java | 147 ++ .../relational/analyzer/MergeIntersectTest.java | 112 + .../analyzer/RemoveEmptyExceptBranchesTest.java | 101 + .../analyzer/RowPatternRecognitionTest.java | 4 +- .../plan/relational/analyzer/TSBSMetadata.java | 19 +- .../relational/analyzer/TableFunctionTest.java | 3 - .../plan/relational/analyzer/TestMetadata.java | 44 +- .../fetcher/cache/TableDeviceSchemaCacheTest.java | 9 + .../plan/relational/planner/PlanTester.java | 4 +- .../plan/statement/InsertStatementTest.java | 97 +- .../scalar/BlobLengthColumnTransformerTest.java | 148 ++ .../DataNodeInternalRPCServiceImplTest.java | 59 +- .../iotdb/db/storageengine/StorageEngineTest.java | 4 +- .../storageengine/dataregion/DataRegionTest.java | 4 +- .../inner/InnerSpaceCompactionExceptionTest.java | 16 + .../repair/RepairUnsortedFileSchedulerTest.java | 10 +- .../memtable/AlignedTVListIteratorTest.java | 57 + .../memtable/WritableMemChunkRegionScanTest.java | 359 +++ .../dataregion/snapshot/IoTDBSnapshotTest.java | 2 +- .../apache/iotdb/db/utils/TSDataTypeTestUtils.java | 111 + .../datastructure/PrimitiveArrayManagerTest.java | 12 +- iotdb-core/node-commons/pom.xml | 13 +- .../apache/iotdb/commons/client/ClientManager.java | 25 + .../iotdb/commons/client/ClientPoolFactory.java | 80 +- .../iotdb/commons/client/ainode/AINodeClient.java | 369 ---- .../AsyncAINodeInternalServiceClient.java} | 25 +- .../async/AsyncPipeDataTransferServiceClient.java | 21 +- .../apache/iotdb/commons/conf/CommonConfig.java | 38 + .../iotdb/commons/conf/CommonDescriptor.java | 5 + .../commons/file/SystemPropertiesHandler.java | 13 +- .../iotdb/commons/model/ModelInformation.java | 43 +- .../org/apache/iotdb/commons/model/ModelTable.java | 4 +- .../apache/iotdb/commons/path/MeasurementPath.java | 26 - .../org/apache/iotdb/commons/path/PartialPath.java | 26 + .../service/PipePluginExecutableManager.java | 9 + .../task/subtask/PipeAbstractSinkSubtask.java | 13 + .../iotdb/commons/pipe/config/PipeConfig.java | 10 + .../iotdb/commons/pipe/config/PipeDescriptor.java | 11 + .../pipe/receiver/PipeReceiverStatusHandler.java | 39 +- .../schema/column/ColumnHeaderConstant.java | 13 + .../commons/schema/table/InformationSchema.java | 35 +- .../schema/table/InsertNodeMeasurementInfo.java | 244 ++ .../schema/table/TsFileTableSchemaUtil.java | 207 ++ .../apache/iotdb/commons/schema/table/TsTable.java | 226 +- .../schema/table/column/FieldColumnSchema.java | 1 + .../schema/table/column/TsTableColumnSchema.java | 6 + .../org/apache/iotdb/commons/utils/FileUtils.java | 92 +- .../apache/iotdb/commons/utils/JVMCommonUtils.java | 7 +- .../org/apache/iotdb/commons/utils/RetryUtils.java | 59 +- .../PipeReceiverStatusHandlerTest.java | 302 +++ .../commons/pipe/task/PipeSleepIntervalTest.java | 83 + .../schema/table/TsFileTableSchemaUtilTest.java | 748 +++++++ .../apache/iotdb/commons/utils/FileUtilsTest.java | 79 + .../db/relational/grammar/sql/RelationalSql.g4 | 34 +- .../thrift-ainode/src/main/thrift/ainode.thrift | 8 +- .../src/main/thrift/confignode.thrift | 142 +- .../thrift-datanode/src/main/thrift/client.thrift | 1 + pom.xml | 34 +- scripts/sbin/windows/start-confignode.bat | 28 - scripts/sbin/windows/start-datanode.bat | 48 - 382 files changed, 16390 insertions(+), 10907 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index 08a35c9744d,daceffce6b7..280d18df76e --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@@ -70,7 -66,8 +68,9 @@@ import org.apache.iotdb.db.protocol.cli import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.session.IClientSession; + import org.apache.iotdb.db.protocol.session.SessionManager; + import org.apache.iotdb.db.queryengine.common.ConnectionInfo; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; import org.apache.iotdb.db.queryengine.plan.Coordinator; import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowCreateViewTask; @@@ -135,15 -123,13 +135,18 @@@ import static org.apache.iotdb.db.query import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowPipePluginsTask.PIPE_PLUGIN_TYPE_EXTERNAL; public class InformationSchemaContentSupplierFactory { + + private static final SessionManager sessionManager = SessionManager.getInstance(); + private InformationSchemaContentSupplierFactory() {} - public static Iterator<TsBlock> getSupplier( - final String tableName, final List<TSDataType> dataTypes, final UserEntity userEntity) { + public static IInformationSchemaContentSupplier getSupplier( + final OperatorContext context, + final String tableName, + final List<TSDataType> dataTypes, + final UserEntity userEntity, + final Filter pushDownFilter, + final PaginationController paginationController) { try { switch (tableName) { case InformationSchema.QUERIES: @@@ -180,9 -164,8 +181,11 @@@ return new ConfigNodesSupplier(dataTypes, userEntity); case InformationSchema.DATA_NODES: return new DataNodesSupplier(dataTypes, userEntity); + case InformationSchema.TABLE_DISK_USAGE: + return new TableDiskUsageSupplier( + dataTypes, userEntity, pushDownFilter, paginationController, context); + case InformationSchema.CONNECTIONS: + return new ConnectionsSupplier(dataTypes, userEntity); default: throw new UnsupportedOperationException("Unknown table: " + tableName); } @@@ -1242,178 -1119,7 +1141,178 @@@ } } - private abstract static class TsBlockSupplier implements Iterator<TsBlock> { + private static class TableDiskUsageSupplier implements IInformationSchemaContentSupplier { + private final List<TSDataType> dataTypes; + private final Map<String, List<TTableInfo>> databaseTableInfoMap; + private final Filter pushDownFilter; + private final PaginationController paginationController; + private final OperatorContext operatorContext; + + private DataRegion currentDataRegion; + private long currentTimePartition; + private List<String> currentTablesToScan; + private boolean currentDatabaseOnlyHasOneTable; + private TableDiskUsageStatisticUtil statisticUtil; + + private final StorageEngineTimePartitionIterator timePartitionIterator; + + private TableDiskUsageSupplier( + final List<TSDataType> dataTypes, + final UserEntity userEntity, + final Filter pushDownFilter, + final PaginationController paginationController, + final OperatorContext operatorContext) + throws TException, ClientManagerException { + this.dataTypes = dataTypes; + this.pushDownFilter = pushDownFilter; + this.paginationController = paginationController; + this.operatorContext = operatorContext; + AuthorityChecker.getAccessControl().checkUserGlobalSysPrivilege(userEntity); + try (final ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + this.databaseTableInfoMap = client.showTables4InformationSchema().getDatabaseTableInfoMap(); + } + this.timePartitionIterator = + new StorageEngineTimePartitionIterator( + Optional.of( + dataRegion -> { + List<TTableInfo> tTableInfos = + databaseTableInfoMap.get(dataRegion.getDatabaseName()); + if (tTableInfos == null || tTableInfos.isEmpty()) { + return false; + } + return PathUtils.isTableModelDatabase(dataRegion.getDatabaseName()); + }), + Optional.of( + (dataRegion, timePartition) -> { + currentTablesToScan = getTablesToScan(dataRegion, timePartition); + return !currentTablesToScan.isEmpty(); + })); + } + + @Override + public boolean hasNext() { + if (statisticUtil != null) { + return true; + } + if (!paginationController.hasCurLimit()) { + return false; + } + try { + if (timePartitionIterator.next()) { + currentDataRegion = timePartitionIterator.currentDataRegion(); + currentTimePartition = timePartitionIterator.currentTimePartition(); + statisticUtil = + new TableDiskUsageStatisticUtil( + currentDataRegion.getTsFileManager(), + currentTimePartition, + currentTablesToScan, + currentDatabaseOnlyHasOneTable, + Optional.ofNullable(operatorContext.getInstanceContext())); + return true; + } + return false; + } catch (Exception e) { + closeStatisticUtil(); + throw new RuntimeException(e.getMessage(), e); + } + } + + private List<String> getTablesToScan(DataRegion dataRegion, long timePartition) { + String databaseName = dataRegion.getDatabaseName(); + List<TTableInfo> tTableInfos = databaseTableInfoMap.get(databaseName); + if (tTableInfos == null || tTableInfos.isEmpty()) { + return Collections.emptyList(); + } + + List<String> tablesToScan = new ArrayList<>(tTableInfos.size()); + int totalValidTableCount = 0; + for (TTableInfo tTableInfo : tTableInfos) { + if (tTableInfo.getType() != TableType.BASE_TABLE.ordinal()) { + continue; + } + totalValidTableCount++; + if (pushDownFilter != null) { + Object[] row = new Object[5]; + row[0] = new Binary(dataRegion.getDatabaseName(), TSFileConfig.STRING_CHARSET); + row[1] = new Binary(tTableInfo.getTableName(), TSFileConfig.STRING_CHARSET); + row[2] = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); - row[3] = Integer.parseInt(dataRegion.getDataRegionId()); ++ row[3] = Integer.parseInt(dataRegion.getDataRegionIdString()); + row[4] = timePartition; + if (!pushDownFilter.satisfyRow(0, row)) { + continue; + } + } + if (!paginationController.hasCurLimit()) { + break; + } + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + continue; + } + paginationController.consumeLimit(); + tablesToScan.add(tTableInfo.getTableName()); + } + currentDatabaseOnlyHasOneTable = totalValidTableCount == 1; + return tablesToScan; + } + + @Override + public TsBlock next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + long maxRuntime = OperatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS); + long start = System.nanoTime(); + + if (statisticUtil.hasNextFile()) { + do { + statisticUtil.calculateNextFile(); + } while (System.nanoTime() - start < maxRuntime && statisticUtil.hasNextFile()); + if (statisticUtil.hasNextFile()) { + return null; + } + } + + TsBlockBuilder builder = new TsBlockBuilder(dataTypes); + long[] resultArr = statisticUtil.getResult(); + + for (int i = 0; i < currentTablesToScan.size(); i++) { + builder.getTimeColumnBuilder().writeLong(0); + ColumnBuilder[] columns = builder.getValueColumnBuilders(); + + columns[0].writeBinary( + new Binary(currentDataRegion.getDatabaseName(), TSFileConfig.STRING_CHARSET)); + columns[1].writeBinary(new Binary(currentTablesToScan.get(i), TSFileConfig.STRING_CHARSET)); + columns[2].writeInt(IoTDBDescriptor.getInstance().getConfig().getDataNodeId()); - columns[3].writeInt(Integer.parseInt(currentDataRegion.getDataRegionId())); ++ columns[3].writeInt(Integer.parseInt(currentDataRegion.getDataRegionIdString())); + columns[4].writeLong(currentTimePartition); + columns[5].writeLong(resultArr[i]); + builder.declarePosition(); + } + closeStatisticUtil(); + return builder.build(); + } + + @Override + public void close() throws IOException { + closeStatisticUtil(); + } + + private void closeStatisticUtil() { + if (statisticUtil == null) { + return; + } + try { + statisticUtil.close(); + statisticUtil = null; + } catch (IOException ignored) { + } + } + } + + private abstract static class TsBlockSupplier implements IInformationSchemaContentSupplier { protected final TsBlockBuilder resultBuilder; protected final ColumnBuilder[] columnBuilders; @@@ -1442,10 -1148,37 +1341,42 @@@ } protected abstract void constructLine(); + + @Override + public void close() throws IOException { + // do nothing + } } + + private static class ConnectionsSupplier extends TsBlockSupplier { + private Iterator<ConnectionInfo> sessionConnectionIterator; + + private ConnectionsSupplier(final List<TSDataType> dataTypes, final UserEntity userEntity) { + super(dataTypes); + accessControl.checkUserGlobalSysPrivilege(userEntity); + sessionConnectionIterator = sessionManager.getAllSessionConnectionInfo().iterator(); + } + + @Override + protected void constructLine() { + ConnectionInfo connectionInfo = sessionConnectionIterator.next(); + columnBuilders[0].writeBinary( + new Binary(String.valueOf(connectionInfo.getDataNodeId()), TSFileConfig.STRING_CHARSET)); + columnBuilders[1].writeBinary( + new Binary(String.valueOf(connectionInfo.getUserId()), TSFileConfig.STRING_CHARSET)); + columnBuilders[2].writeBinary( + new Binary(String.valueOf(connectionInfo.getSessionId()), TSFileConfig.STRING_CHARSET)); + columnBuilders[3].writeBinary( + new Binary(connectionInfo.getUserName(), TSFileConfig.STRING_CHARSET)); + columnBuilders[4].writeLong(connectionInfo.getLastActiveTime()); + columnBuilders[5].writeBinary( + new Binary(connectionInfo.getClientAddress(), TSFileConfig.STRING_CHARSET)); + resultBuilder.declarePosition(); + } + + @Override + public boolean hasNext() { + return sessionConnectionIterator.hasNext(); + } + } } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java index 2044852fa27,f8cf497546e..3c88590b98f --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java @@@ -85,7 -85,7 +85,8 @@@ public class DataNodeLocationSupplierFa public List<TDataNodeLocation> getDataNodeLocations(final String tableName) { switch (tableName) { case InformationSchema.QUERIES: + case InformationSchema.TABLE_DISK_USAGE: + case InformationSchema.CONNECTIONS: return getReadableDataNodeLocations(); case InformationSchema.DATABASES: case InformationSchema.TABLES: diff --cc iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java index ff37aecd2d3,243bc41c40c..7a4de8d8573 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java @@@ -58,7 -49,7 +57,8 @@@ public class InformationSchema public static final String NODES = "nodes"; public static final String CONFIG_NODES = "config_nodes"; public static final String DATA_NODES = "data_nodes"; + public static final String TABLE_DISK_USAGE = "table_disk_usage"; + public static final String CONNECTIONS = "connections"; static { final TsTable queriesTable = new TsTable(QUERIES); @@@ -372,36 -346,23 +355,52 @@@ dataNodesTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(DATA_NODES, dataNodesTable); + final TsTable tableDiskUsageTable = new TsTable(TABLE_DISK_USAGE); + tableDiskUsageTable.addColumnSchema( + new FieldColumnSchema(ColumnHeaderConstant.DATABASE, TSDataType.STRING)); + tableDiskUsageTable.addColumnSchema( + new FieldColumnSchema(ColumnHeaderConstant.TABLE_NAME_TABLE_MODEL, TSDataType.STRING)); + tableDiskUsageTable.addColumnSchema( + new FieldColumnSchema(ColumnHeaderConstant.NODE_ID_TABLE_MODEL, TSDataType.INT32)); + tableDiskUsageTable.addColumnSchema( + new FieldColumnSchema(ColumnHeaderConstant.REGION_ID_TABLE_MODEL, TSDataType.INT32)); + tableDiskUsageTable.addColumnSchema( + new FieldColumnSchema(ColumnHeaderConstant.TIME_PARTITION_TABLE_MODEL, TSDataType.INT64)); + tableDiskUsageTable.addColumnSchema( + new FieldColumnSchema(ColumnHeaderConstant.SIZE_IN_BYTES_TABLE_MODEL, TSDataType.INT64)); + tableDiskUsageTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); + schemaTables.put(TABLE_DISK_USAGE, tableDiskUsageTable); ++ + final TsTable connectionsTable = new TsTable(CONNECTIONS); + connectionsTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.DATANODE_ID, TSDataType.STRING)); + connectionsTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.USERID, TSDataType.STRING)); + connectionsTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.SESSION_ID, TSDataType.STRING)); + connectionsTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.USER_NAME, TSDataType.STRING)); + connectionsTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.LAST_ACTIVE_TIME, TSDataType.TIMESTAMP)); + connectionsTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.CLIENT_IP, TSDataType.STRING)); + connectionsTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); + schemaTables.put(CONNECTIONS, connectionsTable); } + static { + columnsThatSupportPushDownPredicate.put( + TABLE_DISK_USAGE, + ImmutableSet.of( + ColumnHeaderConstant.DATABASE, + ColumnHeaderConstant.TABLE_NAME_TABLE_MODEL, + ColumnHeaderConstant.NODE_ID_TABLE_MODEL, + ColumnHeaderConstant.REGION_ID_TABLE_MODEL, + ColumnHeaderConstant.TIME_PARTITION_TABLE_MODEL)); + + tablesThatSupportPushDownLimitOffset.add(TABLE_DISK_USAGE); + } + public static Map<String, TsTable> getSchemaTables() { return schemaTables; }
