This is an automated email from the ASF dual-hosted git repository. chenyz pushed a commit to branch udtf-optimize in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7723e8d427b578633c3047e287053a82202c126b Merge: 4ba48079a1b 22238fc880b Author: Chen YZ <[email protected]> AuthorDate: Sat Feb 22 23:02:00 2025 +0800 Merge branch 'udtf' into udtf-optimize .github/workflows/cluster-it-1c1d.yml | 9 +- .github/workflows/cluster-it-1c1d1a.yml | 9 +- .github/workflows/cluster-it-1c3d.yml | 9 +- .github/workflows/compile-check.yml | 9 +- .github/workflows/dependency-check.yml | 1 - .github/workflows/multi-language-client.yml | 1 - .../{pipe-it-2cluster.yml => pipe-it.yml} | 314 +- .github/workflows/sonar-codecov.yml | 1 - .github/workflows/table-cluster-it-1c1d.yml | 9 +- .github/workflows/table-cluster-it-1c3d.yml | 9 +- .github/workflows/todos-check.yml | 27 +- .github/workflows/unit-test.yml | 1 - docker/src/main/DockerCompose/do-docker-build.sh | 4 +- docker/src/main/DockerCompose/entrypoint.sh | 5 +- .../main/DockerCompose/replace-conf-from-env.sh | 33 +- docker/src/main/Dockerfile-1.0.0-confignode | 2 +- docker/src/main/Dockerfile-1.0.0-datanode | 2 +- docker/src/main/Dockerfile-1.0.0-standalone | 2 +- docker/src/main/Dockerfile-1c1d | 2 +- .../org/apache/iotdb/TableModelJDBCExample.java | 4 +- .../server/CustomizedJsonPayloadFormatter.java | 2 +- .../org/apache/iotdb/mqtt/server/MyMessage.java | 31 +- .../java/org/apache/iotdb/mqtt/MQTTClient.java | 39 +- .../java/org/apache/iotdb/CountPointProcessor.java | 3 +- .../apache/iotdb/SubscriptionSessionExample.java | 2 +- .../TableModelSubscriptionSessionExample.java | 16 +- integration-test/pom.xml | 46 +- .../iotdb/it/env/cluster/ClusterConstant.java | 1 + .../it/env/cluster/config/MppCommonConfig.java | 10 +- .../it/env/cluster/config/MppDataNodeConfig.java | 12 + .../env/cluster/config/MppSharedCommonConfig.java | 13 +- .../iotdb/it/env/cluster/env/AbstractEnv.java | 12 + .../iotdb/it/env/cluster/node/DataNodeWrapper.java | 11 + .../it/env/remote/config/RemoteCommonConfig.java | 7 +- .../it/env/remote/config/RemoteDataNodeConfig.java | 10 + .../iotdb/it/env/remote/env/RemoteServerEnv.java | 10 + ...va => MultiClusterIT2DualTableManualBasic.java} | 2 +- .../MultiClusterIT2DualTableManualEnhanced.java | 16 +- ....java => MultiClusterIT2DualTreeAutoBasic.java} | 2 +- ...va => MultiClusterIT2DualTreeAutoEnhanced.java} | 2 +- ...del.java => MultiClusterIT2DualTreeManual.java} | 2 +- .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 3 + .../org/apache/iotdb/itbase/env/CommonConfig.java | 4 +- .../apache/iotdb/itbase/env/DataNodeConfig.java | 4 + .../iotdb/itbase/runtime/ClusterTestStatement.java | 5 +- .../{AbstractScript.java => AbstractScriptIT.java} | 2 +- .../apache/iotdb/cli/it/StartClientScriptIT.java | 6 +- ....java => IoTDBAutoRegionGroupExtension2IT.java} | 4 +- ...st.java => IoTDBPartitionTableAutoCleanIT.java} | 2 +- ...IoTDBRegionOperationReliabilityITFramework.java | 72 - .../IoTDBRegionGroupExpandAndShrinkForIoTV1IT.java | 20 +- ...IoTDBRegionMigrateNormalITForIoTV2BatchIT.java} | 2 +- ...=> IoTDBRegionMigrateOtherForIoTV2BatchIT.java} | 2 +- ... IoTDBRegionMigrateNormalForIoTV2StreamIT.java} | 2 +- ...IoTDBRegionMigrateOtherITForIoTV2StreamIT.java} | 2 +- ...rCrashWhenRemoveRemotePeerForIoTV2BatchIT.java} | 2 +- ...RegionMigrateDataNodeCrashForIoTV2BatchIT.java} | 2 +- ...alCrashWhenDeleteLocalPeerForIoTV2BatchIT.java} | 2 +- ...lCrashWhenRemoveRemotePeerForIoTV2BatchIT.java} | 2 +- ...CrashWhenRemoveRemotePeerForIoTV2StreamIT.java} | 2 +- ...egionMigrateDataNodeCrashForIoTV2StreamIT.java} | 2 +- ...lCrashWhenDeleteLocalPeerForIoTV2StreamIT.java} | 2 +- ...CrashWhenRemoveRemotePeerForIoTV2StreamIT.java} | 2 +- ...oTDBRegionMigrateClusterCrashIoTV2BatchIT.java} | 2 +- ...BRegionMigrateConfigNodeCrashIoTV2BatchIT.java} | 2 +- ...TDBRegionMigrateClusterCrashIoTV2StreamIT.java} | 2 +- ...RegionMigrateConfigNodeCrashIoTV2StreamIT.java} | 2 +- .../IoTDBRemoveConfigNodeITFramework.java | 206 + .../IoTDBRemoveConfigNodeNormalIT.java} | 16 +- .../IoTDBRemoveDataNodeITFramework.java | 2 +- .../apache/iotdb/db/it/IoTDBDuplicateTimeIT.java | 5 +- .../apache/iotdb/db/it/IoTDBFloatPrecisionIT.java | 45 + .../apache/iotdb/db/it/IoTDBInsertWithQueryIT.java | 1 + .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 73 + .../org/apache/iotdb/db/it/IoTDBRestartIT.java | 4 +- .../org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java | 38 + .../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 38 + .../iotdb/db/it/auth/IoTDBRelationalAuthIT.java | 183 + .../iotdb/db/it/auth/IoTDBSystemPermissionIT.java | 2 +- .../iotdb/db/it/groupby/IoTDBGroupByUnseqIT.java | 2 +- .../apache/iotdb/db/it/query/IoTDBQueryDemoIT.java | 25 + .../org/apache/iotdb/db/it/utils/TestUtils.java | 227 +- .../it/{ => dual}/tablemodel/TableModelUtils.java | 2 +- .../AbstractPipeTableModelDualManualIT.java} | 4 +- .../tablemodel/manual/basic}/IoTDBPipeAlterIT.java | 17 +- .../manual/basic}/IoTDBPipeDataSinkIT.java | 21 +- .../manual/basic}/IoTDBPipeExtractorIT.java | 10 +- .../manual/basic}/IoTDBPipeIsolationIT.java | 17 +- .../manual/basic}/IoTDBPipeLifeCycleIT.java | 165 +- .../manual/basic}/IoTDBPipeProtocolIT.java | 11 +- .../manual/basic}/IoTDBPipeSwitchStatusIT.java | 18 +- .../manual/basic}/IoTDBPipeSyntaxIT.java | 17 +- .../manual/basic}/IoTDBPipeWithLoadIT.java | 10 +- .../manual/basic}/IoTDBTablePatternFormatIT.java | 17 +- .../manual/enhanced}/IoTDBPipeAutoConflictIT.java | 11 +- .../manual/enhanced}/IoTDBPipeAutoDropIT.java | 17 +- .../manual/enhanced}/IoTDBPipeClusterIT.java | 10 +- .../enhanced}/IoTDBPipeConnectorCompressionIT.java | 10 +- .../enhanced}/IoTDBPipeConnectorParallelIT.java | 18 +- .../manual/enhanced}/IoTDBPipeDoubleLivingIT.java | 20 +- .../manual/enhanced/IoTDBPipeIdempotentIT.java | 205 + .../manual/enhanced/IoTDBPipeMetaIT.java} | 95 +- .../manual/enhanced}/IoTDBPipeNullValueIT.java | 17 +- .../IoTDBPipeTypeConversionISessionIT.java | 26 +- .../enhanced}/IoTDBPipeTypeConversionIT.java | 25 +- .../auto/AbstractPipeDualTreeModelAutoIT.java} | 4 +- .../treemodel/auto/basic}/IoTDBPipeAlterIT.java | 16 +- .../auto/basic}/IoTDBPipeConnectorParallelIT.java | 17 +- .../treemodel/auto/basic}/IoTDBPipeDataSinkIT.java | 17 +- .../auto/basic}/IoTDBPipeExtractorIT.java | 9 +- .../auto/basic}/IoTDBPipeLifeCycleIT.java | 30 +- .../auto/basic}/IoTDBPipeProcessorIT.java | 10 +- .../treemodel/auto/basic}/IoTDBPipeProtocolIT.java | 10 +- .../auto/basic}/IoTDBPipeSwitchStatusIT.java | 17 +- .../treemodel/auto/basic}/IoTDBPipeSyntaxIT.java | 17 +- .../auto/basic}/IoTDBTreePatternFormatIT.java | 17 +- .../auto/enhanced}/IoTDBPipeAutoConflictIT.java | 10 +- .../auto/enhanced}/IoTDBPipeAutoDropIT.java | 16 +- .../auto/enhanced}/IoTDBPipeClusterIT.java | 11 +- .../IoTDBPipeConditionalOperationsIT.java | 16 +- .../enhanced}/IoTDBPipeConnectorCompressionIT.java | 9 +- .../auto/enhanced}/IoTDBPipeIdempotentIT.java | 117 +- .../auto/enhanced}/IoTDBPipeNullValueIT.java | 9 +- .../auto/enhanced}/IoTDBPipeWithLoadIT.java | 9 +- .../auto/enhanced}/PipeNowFunctionIT.java | 16 +- .../manual/AbstractPipeDualTreeModelManualIT.java} | 4 +- .../treemodel}/manual/IoTDBPipeInclusionIT.java | 8 +- .../manual/IoTDBPipeManualConflictIT.java | 8 +- .../manual/IoTDBPipeMetaHistoricalIT.java | 8 +- .../manual/IoTDBPipeMetaLeaderChangeIT.java | 8 +- .../treemodel}/manual/IoTDBPipeMetaRestartIT.java | 8 +- .../manual/IoTDBPipeMultiSchemaRegionIT.java | 8 +- .../treemodel}/manual/IoTDBPipePermissionIT.java | 8 +- .../treemodel}/manual/IoTDBPipeReqAutoSliceIT.java | 8 +- .../manual/IoTDBPipeTypeConversionISessionIT.java | 8 +- .../manual/IoTDBPipeTypeConversionIT.java | 8 +- .../iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java | 2 +- .../it/db/it/IoTDBAuthenticationTableIT.java | 1001 +++++ .../relational/it/db/it/IoTDBDeletionTableIT.java | 115 + .../relational/it/db/it/IoTDBInsertTableIT.java | 28 +- .../it/db/it/IoTDBLoadConfigurationTableIT.java | 85 + .../relational/it/db/it/IoTDBRestartTableIT.java | 3 - .../it/db/it/IoTDBSetSystemStatusTableIT.java | 104 + .../relational/it/mqtt/IoTDBMQTTServiceIT.java | 132 + .../query/old/query/IoTDBGreatestLeastTableIT.java | 305 ++ .../it/query/recent/IoTDBMaintainAuthIT.java | 289 ++ .../it/query/recent/IoTDBNullIdQueryIT.java | 33 + .../it/query/recent/IoTDBQueryAuthIT.java | 320 ++ .../relational/it/schema/IoTDBDatabaseIT.java | 145 +- .../iotdb/relational/it/schema/IoTDBTableIT.java | 57 + .../it/session/IoTDBSessionRelationalIT.java | 268 ++ .../iotdb/session/it/IoTDBSessionSimpleIT.java | 258 +- .../it/local/IoTDBSubscriptionBasicIT.java | 67 + .../regression/param/IoTDBTestParamTopicIT.java | 2 +- .../apache/iotdb/tools/it/ExportDataTestIT.java | 4 +- .../apache/iotdb/tools/it/ExportSchemaTestIT.java | 4 +- .../apache/iotdb/tools/it/ExportTsFileTestIT.java | 36 +- .../apache/iotdb/tools/it/ImportDataTestIT.java | 4 +- .../apache/iotdb/tools/it/ImportSchemaTestIT.java | 4 +- .../src/test/resources/logback-test.xml | 2 +- iotdb-client/cli/pom.xml | 4 + .../src/main/java/org/apache/iotdb/cli/Cli.java | 4 +- .../org/apache/iotdb/tool/common/Constants.java | 30 +- .../org/apache/iotdb/tool/common/OptionsUtil.java | 112 + .../org/apache/iotdb/tool/tsfile/ExportTsFile.java | 539 +-- .../subscription/AbstractSubscriptionTsFile.java | 74 + .../tool/tsfile/subscription/CommonParam.java | 212 + .../subscription/SubscriptionTableTsFile.java | 186 + .../subscription/SubscriptionTreeTsFile.java | 183 + iotdb-client/client-py/iotdb/Session.py | 51 +- iotdb-client/client-py/iotdb/SessionPool.py | 12 + iotdb-client/client-py/iotdb/table_session.py | 9 + iotdb-client/client-py/iotdb/table_session_pool.py | 6 + iotdb-client/client-py/session_ssl_example.py | 85 + iotdb-client/jdbc/pom.xml | 55 +- .../main/java/org/apache/iotdb/jdbc/Constant.java | 7 +- ...ata.java => IoTDBAbstractDatabaseMetadata.java} | 4330 +++++++++----------- .../org/apache/iotdb/jdbc/IoTDBConnection.java | 30 +- .../apache/iotdb/jdbc/IoTDBConnectionParams.java | 4 +- .../apache/iotdb/jdbc/IoTDBDataSourceFactory.java | 3 - .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 3555 ++-------------- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 7 + .../IoTDBRelationalDatabaseMetadata.java | 405 ++ .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 14 + .../iotdb/rpc/subscription/config/TopicConfig.java | 7 +- .../subscription/payload/poll/TabletsPayload.java | 4 +- .../java/org/apache/iotdb/rpc/RpcUtilsTest.java | 10 + .../java/org/apache/iotdb/session/Session.java | 34 +- .../apache/iotdb/session/SessionConnection.java | 1031 ++--- .../apache/iotdb/session/TableSessionBuilder.java | 4 +- .../org/apache/iotdb/session/pool/SessionPool.java | 5 +- .../iotdb/session/pool/TableSessionWrapper.java | 15 + .../consumer/ISubscriptionTablePullConsumer.java | 18 + .../consumer/ISubscriptionTablePushConsumer.java | 18 + .../consumer/ISubscriptionTreePullConsumer.java | 18 + .../consumer/ISubscriptionTreePushConsumer.java | 18 + .../base/AbstractSubscriptionConsumer.java | 6 +- .../base/AbstractSubscriptionProviders.java | 48 +- .../consumer/table/SubscriptionTableProvider.java | 18 +- .../table/SubscriptionTablePullConsumer.java | 34 +- .../table/SubscriptionTablePushConsumer.java | 36 +- .../consumer/tree/SubscriptionTreeProvider.java | 18 +- .../tree/SubscriptionTreePullConsumer.java | 34 +- .../tree/SubscriptionTreePushConsumer.java | 34 +- iotdb-core/ainode/pyproject.toml | 14 +- iotdb-core/ainode/resources/conf/ainode-env.sh | 2 +- .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 6 + .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 23 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 24 + .../assembly/resources/sbin/remove-confignode.bat | 135 - .../assembly/resources/sbin/remove-confignode.sh | 69 - .../confignode/conf/ConfigNodeRemoveCheck.java | 5 +- .../request/ConfigPhysicalPlanVisitor.java | 221 +- .../pipe/payload/PipeDeleteLogicalViewPlan.java | 5 +- .../pipe/payload/PipeDeleteTimeSeriesPlan.java | 5 +- .../statemachine/ConfigRegionStateMachine.java | 6 +- .../iotdb/confignode/manager/ConfigManager.java | 3 +- .../confignode/manager/PermissionManager.java | 7 +- .../client/IoTDBConfigNodeSyncClientManager.java | 6 +- .../protocol/IoTDBConfigRegionAirGapConnector.java | 3 + .../protocol/IoTDBConfigRegionConnector.java | 6 +- .../pipe/event/PipeConfigRegionSnapshotEvent.java | 17 +- .../extractor/ConfigRegionListeningFilter.java | 84 +- .../pipe/extractor/IoTDBConfigRegionExtractor.java | 23 +- ...ConfigPhysicalPlanTablePatternParseVisitor.java | 62 + ...peConfigPhysicalPlanTableScopeParseVisitor.java | 86 + ...eConfigPhysicalPlanTreePatternParseVisitor.java | 30 +- ...ipeConfigPhysicalPlanTreeScopeParseVisitor.java | 80 + .../receiver/protocol/IoTDBConfigNodeReceiver.java | 187 +- .../PipeConfigPhysicalPlanTSStatusVisitor.java | 207 +- .../iotdb/confignode/persistence/AuthorInfo.java | 70 +- .../schema/CNPhysicalPlanGenerator.java | 15 +- ...otParser.java => ConfigNodeSnapshotParser.java} | 24 +- .../procedure/impl/StateMachineProcedure.java | 7 +- .../impl/schema/AlterLogicalViewProcedure.java | 4 +- ...ipeConfigPhysicalPlanScopeParseVisitorTest.java | 116 + ...igPhysicalPlanTablePatternParseVisitorTest.java | 25 + .../org/apache/iotdb/consensus/common/Peer.java | 13 +- .../apache/iotdb/consensus/iot/IoTConsensus.java | 30 +- .../consensus/iot/IoTConsensusServerImpl.java | 258 +- .../consensus/iot/logdispatcher/LogDispatcher.java | 9 +- .../service/IoTConsensusRPCServiceProcessor.java | 2 +- .../apache/iotdb/consensus/pipe/PipeConsensus.java | 90 +- .../consensus/pipe/PipeConsensusPeerManager.java | 89 +- .../consensus/pipe/PipeConsensusServerImpl.java | 33 +- .../apache/iotdb/consensus/iot/ReplicateTest.java | 63 +- .../assembly/resources/sbin/remove-datanode.bat | 148 - .../src/assembly/resources/sbin/remove-datanode.sh | 91 - .../src/assembly/resources/sbin/start-datanode.bat | 2 +- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 3 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 34 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 25 +- .../db/pipe/agent/runtime/PipeAgentLauncher.java | 3 +- .../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 63 +- .../task/builder/PipeDataNodeTaskBuilder.java | 12 +- .../agent/task/connection/PipeEventCollector.java | 11 +- .../agent/task/stage/PipeTaskProcessorStage.java | 9 +- .../PipeRealtimePriorityBlockingQueue.java | 10 +- .../subtask/processor/PipeProcessorSubtask.java | 5 +- .../processor/PipeProcessorSubtaskWorker.java | 11 +- .../client/IoTDBDataNodeAsyncClientManager.java | 9 +- .../client/IoTDBDataNodeSyncClientManager.java | 6 +- .../airgap/IoTDBDataNodeAirGapConnector.java | 3 + .../async/IoTDBDataRegionAsyncConnector.java | 18 +- .../handler/PipeTransferTrackableHandler.java | 33 +- .../async/handler/PipeTransferTsFileHandler.java | 70 +- .../thrift/sync/IoTDBDataNodeSyncConnector.java | 6 +- .../db/pipe/event/common/PipeInsertionEvent.java | 64 +- .../db/pipe/event/common/row/PipeRowCollector.java | 27 +- .../schema/PipeSchemaRegionSnapshotEvent.java | 7 +- .../tablet/PipeInsertNodeTabletInsertionEvent.java | 56 +- .../common/tablet/PipeRawTabletInsertionEvent.java | 26 +- .../common/tsfile/PipeTsFileInsertionEvent.java | 86 +- .../query/TsFileInsertionEventQueryParser.java | 70 +- .../scan/TsFileInsertionEventScanParser.java | 35 +- .../table/TsFileInsertionEventTableParser.java | 70 +- .../db/pipe/event/realtime/PipeRealtimeEvent.java | 6 + .../event/realtime/PipeRealtimeEventFactory.java | 12 +- ...oricalDataRegionTsFileAndDeletionExtractor.java | 10 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 26 +- .../PipeRealtimeDataRegionHybridExtractor.java | 139 +- .../realtime/assigner/PipeDataRegionAssigner.java | 19 + .../listener/PipeInsertionDataNodeListener.java | 4 +- .../schemaregion/SchemaRegionListeningFilter.java | 8 +- .../iotdb/db/pipe/metric/PipeResourceMetrics.java | 34 + .../processor/aggregate/AggregateProcessor.java | 18 +- .../twostage/plugin/TwoStageCountProcessor.java | 14 +- .../pipeconsensus/PipeConsensusReceiver.java | 55 +- .../protocol/thrift/IoTDBDataNodeReceiver.java | 10 +- .../visitor/PipePlanToStatementVisitor.java | 18 +- .../db/pipe/resource/memory/PipeMemoryManager.java | 137 +- .../resource/tsfile/PipeTsFileResourceManager.java | 15 + .../iotdb/db/protocol/client/ConfigNodeClient.java | 3 +- .../db/protocol/mqtt/JSONPayloadFormatter.java | 6 +- .../db/protocol/mqtt/LinePayloadFormatter.java | 252 ++ .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 239 +- .../org/apache/iotdb/db/protocol/mqtt/Message.java | 58 +- .../iotdb/db/protocol/mqtt/TableMessage.java | 144 + .../mqtt/{Message.java => TreeMessage.java} | 16 +- .../db/protocol/rest/impl/PingApiServiceImpl.java | 4 +- .../protocol/thrift/impl/ClientRPCServiceImpl.java | 29 +- .../execution/executor/RegionWriteExecutor.java | 33 +- .../fragment/FragmentInstanceContext.java | 78 +- .../fragment/FragmentInstanceManager.java | 2 + .../execution/fragment/QueryContext.java | 9 + .../InformationSchemaContentSupplierFactory.java | 341 +- .../relational/aggregation/TableAggregator.java | 49 +- .../relational/ColumnTransformerBuilder.java | 15 + .../queryengine/metric/QueryPlanCostMetricSet.java | 68 +- .../iotdb/db/queryengine/plan/Coordinator.java | 8 + .../db/queryengine/plan/analyze/AnalyzeUtils.java | 9 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 16 +- .../db/queryengine/plan/analyze/Analyzer.java | 3 +- .../queryengine/plan/analyze/TemplatedAnalyze.java | 3 +- .../analyze/cache/partition/PartitionCache.java | 5 + .../schema/dualkeycache/IDualKeyCacheStats.java | 6 + .../cache/schema/dualkeycache/impl/CacheStats.java | 24 +- .../schema/dualkeycache/impl/DualKeyCacheImpl.java | 122 +- .../plan/analyze/load/LoadTsFileAnalyzer.java | 7 + .../analyze/load/LoadTsFileTableSchemaCache.java | 11 +- .../analyze/load/LoadTsFileTreeSchemaCache.java | 7 +- .../plan/analyze/schema/SchemaValidator.java | 14 +- .../plan/execution/IQueryExecution.java | 2 + .../queryengine/plan/execution/QueryExecution.java | 19 +- .../plan/execution/config/ConfigExecution.java | 9 +- .../execution/config/TableConfigTaskVisitor.java | 46 + .../execution/config/TreeConfigTaskVisitor.java | 31 + .../config/executor/ClusterConfigTaskExecutor.java | 75 +- .../config/executor/IConfigTaskExecutor.java | 6 + .../config/metadata/RemoveConfigNodeTask.java | 43 + .../config/metadata/ShowPipePluginsTask.java | 4 +- .../config/session/SetSqlDialectTask.java} | 41 +- .../db/queryengine/plan/parser/ASTVisitor.java | 57 +- .../db/queryengine/plan/planner/IPlanner.java | 3 +- .../queryengine/plan/planner/LogicalPlanner.java | 6 +- .../plan/planner/TableOperatorGenerator.java | 9 +- .../queryengine/plan/planner/TreeModelPlanner.java | 20 +- .../plan/node/load/LoadSingleTsFileNode.java | 12 +- .../plan/node/load/LoadTsFilePieceNode.java | 6 +- .../plan/planner/plan/node/write/InsertNode.java | 2 + .../planner/plan/node/write/InsertRowNode.java | 3 + .../plan/node/write/RelationalDeleteDataNode.java | 4 +- .../plan/node/write/RelationalInsertRowNode.java | 5 + .../node/write/RelationalInsertTabletNode.java | 8 + .../plan/relational/analyzer/Analyzer.java | 3 +- .../relational/analyzer/StatementAnalyzer.java | 11 +- .../relational/metadata/TableMetadataImpl.java | 20 + .../fetcher/TableHeaderSchemaValidator.java | 4 - .../fetcher/cache/TableDeviceSchemaCache.java | 19 +- .../cache/TableDeviceSchemaCacheMetrics.java | 34 + .../relational/planner/TableLogicalPlanner.java | 16 +- .../plan/relational/planner/TableModelPlanner.java | 32 +- .../distribute/TableDistributedPlanner.java | 11 +- .../DataNodeLocationSupplierFactory.java | 22 +- .../optimizations/PushPredicateIntoTableScan.java | 6 +- .../plan/relational/security/AccessControl.java | 8 + .../relational/security/AccessControlImpl.java | 62 +- .../relational/security/AllowAllAccessControl.java | 5 + .../plan/relational/sql/ast/AstVisitor.java | 16 + .../plan/relational/sql/ast/InsertRows.java | 5 +- .../plan/relational/sql/ast/LoadConfiguration.java | 16 +- .../plan/relational/sql/ast/RemoveConfigNode.java | 72 + .../plan/relational/sql/ast/SetSqlDialect.java | 79 + .../plan/relational/sql/ast/SetSystemStatus.java | 16 +- .../relational/sql/ast/WrappedInsertStatement.java | 78 +- .../plan/relational/sql/parser/AstBuilder.java | 145 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 47 +- .../plan/statement/StatementVisitor.java | 21 + .../plan/statement/crud/InsertBaseStatement.java | 78 +- .../crud/InsertMultiTabletsStatement.java | 11 + .../plan/statement/crud/InsertRowStatement.java | 10 + .../crud/InsertRowsOfOneDeviceStatement.java | 5 + .../plan/statement/crud/InsertRowsStatement.java | 11 + .../plan/statement/crud/InsertTabletStatement.java | 10 + .../metadata/RemoveConfigNodeStatement.java | 71 + .../plan/statement/sys/SetSqlDialectStatement.java | 76 + .../sys/ShowCurrentSqlDialectStatement.java | 34 +- .../statement/sys/ShowCurrentUserStatement.java | 34 +- .../AbstractGreatestLeastColumnTransformer.java | 109 + .../multi/BinaryGreatestColumnTransformer.java | 53 + .../column/multi/BinaryLeastColumnTransformer.java | 53 + .../multi/BooleanGreatestColumnTransformer.java | 52 + .../multi/BooleanLeastColumnTransformer.java | 49 + .../multi/DoubleGreatestColumnTransformer.java | 52 + .../column/multi/DoubleLeastColumnTransformer.java | 52 + .../multi/FloatGreatestColumnTransformer.java | 52 + .../column/multi/FloatLeastColumnTransformer.java | 52 + .../multi/Int32GreatestColumnTransformer.java | 52 + .../column/multi/Int32LeastColumnTransformer.java | 52 + .../multi/Int64GreatestColumnTransformer.java | 52 + .../column/multi/Int64LeastColumnTransformer.java | 52 + .../schemaregion/utils/ResourceByPathUtils.java | 146 +- .../java/org/apache/iotdb/db/service/DataNode.java | 2 +- .../iotdb/db/service/DataNodeShutdownHook.java | 22 +- .../{RPCService.java => ExternalRPCService.java} | 6 +- .../org/apache/iotdb/db/service/MQTTService.java | 2 + .../iotdb/db/service/metrics/WritingMetrics.java | 29 +- .../iotdb/db/storageengine/StorageEngine.java | 35 +- .../db/storageengine/dataregion/DataRegion.java | 23 +- .../performer/impl/FastCompactionPerformer.java | 4 +- .../execute/utils/MultiTsFileDeviceIterator.java | 7 +- .../executor/fast/SeriesCompactionExecutor.java | 20 +- .../readchunk/AlignedSeriesCompactionExecutor.java | 4 +- .../ReadChunkAlignedSeriesCompactionExecutor.java | 6 +- .../readchunk/SingleSeriesCompactionExecutor.java | 30 +- .../utils/writer/AbstractCompactionWriter.java | 5 +- .../writer/AbstractCrossCompactionWriter.java | 7 +- .../writer/AbstractInnerCompactionWriter.java | 1 + .../utils/writer/FastCrossCompactionWriter.java | 6 + .../utils/writer/FastInnerCompactionWriter.java | 6 + .../writer/ReadPointCrossCompactionWriter.java | 1 + .../compaction/io/CompactionDiskTSMIterator.java | 42 +- .../compaction/io/CompactionTsFileInput.java | 122 +- .../compaction/io/CompactionTsFileReader.java | 60 +- .../compaction/io/CompactionTsFileWriter.java | 9 + .../compaction/repair/RepairDataFileScanUtil.java | 56 +- .../estimator/AbstractCompactionEstimator.java | 9 +- .../selector/impl/SettleSelectorImpl.java | 24 +- .../selector/utils/TsFileResourceCandidate.java | 13 +- .../dataregion/flush/MemTableFlushTask.java | 16 +- .../dataregion/memtable/AbstractMemTable.java | 166 +- .../memtable/AlignedReadOnlyMemChunk.java | 588 ++- .../memtable/AlignedWritableMemChunk.java | 752 +++- .../memtable/AlignedWritableMemChunkGroup.java | 21 +- .../dataregion/memtable/IMemTable.java | 8 +- .../dataregion/memtable/IWritableMemChunk.java | 56 +- .../memtable/IWritableMemChunkGroup.java | 9 +- .../dataregion/memtable/ReadOnlyMemChunk.java | 274 +- .../dataregion/memtable/TsFileProcessor.java | 65 +- .../dataregion/memtable/WritableMemChunk.java | 595 ++- .../dataregion/memtable/WritableMemChunkGroup.java | 30 +- .../dataregion/modification/ModificationFile.java | 16 +- .../filescan/impl/MemAlignedChunkHandleImpl.java | 21 +- .../read/reader/chunk/MemAlignedChunkReader.java | 259 +- .../read/reader/chunk/MemAlignedPageReader.java | 137 +- .../read/reader/chunk/MemChunkReader.java | 139 +- .../read/reader/chunk/MemPageReader.java | 129 +- .../reader/chunk/metadata/AlignedPageMetadata.java | 74 + .../read/reader/chunk/metadata/PageMetadata.java | 90 + .../dataregion/tsfile/TsFileResource.java | 76 +- .../tsfile/timeindex/ArrayDeviceTimeIndex.java | 23 +- .../dataregion/tsfile/timeindex/FileTimeIndex.java | 9 +- .../dataregion/tsfile/timeindex/ITimeIndex.java | 5 +- .../utils/TsFileDeviceStartEndTimeIterator.java | 6 +- .../dataregion/utils/TsFileResourceUtils.java | 30 +- .../storageengine/dataregion/wal/io/WALReader.java | 2 +- .../wal/recover/file/TsFilePlanRedoer.java | 9 +- .../file/UnsealedTsFileRecoverPerformer.java | 10 +- ...leStatementDataTypeConvertExecutionVisitor.java | 4 + ...eeStatementDataTypeConvertExecutionVisitor.java | 4 + .../load/splitter/AlignedChunkData.java | 16 +- .../splitter/BatchedAlignedValueChunkData.java | 8 +- .../load/splitter/NonAlignedChunkData.java | 8 +- .../load/splitter/TsFileSplitter.java | 7 + .../broker/SubscriptionPrefetchingQueue.java | 14 +- .../broker/SubscriptionPrefetchingTabletQueue.java | 2 +- .../broker/SubscriptionPrefetchingTsFileQueue.java | 2 +- .../batch/SubscriptionPipeTabletEventBatch.java | 52 +- .../SubscriptionPipeTabletIterationSnapshot.java | 62 + .../pipe/SubscriptionPipeTabletBatchEvents.java | 29 +- .../response/SubscriptionEventTabletResponse.java | 5 - .../response/SubscriptionEventTsFileResponse.java | 7 +- .../apache/iotdb/db/tools/IoTDBDataDirViewer.java | 6 +- .../iotdb/db/tools/TsFileResourcePrinter.java | 10 +- .../iotdb/db/tools/utils/TsFileValidationScan.java | 10 +- .../TsFileOverlapValidationAndRepairTool.java | 8 +- .../org/apache/iotdb/db/utils/DateTimeUtils.java | 4 +- .../java/org/apache/iotdb/db/utils/MathUtils.java | 24 +- .../apache/iotdb/db/utils/ModificationUtils.java | 17 + .../iotdb/db/utils/TimestampPrecisionUtils.java | 20 +- .../db/utils/datastructure/AlignedTVList.java | 415 +- .../db/utils/datastructure/BackAlignedTVList.java | 66 +- .../db/utils/datastructure/BackBinaryTVList.java | 72 +- .../db/utils/datastructure/BackBooleanTVList.java | 71 +- .../db/utils/datastructure/BackDoubleTVList.java | 71 +- .../db/utils/datastructure/BackFloatTVList.java | 71 +- .../db/utils/datastructure/BackIntTVList.java | 72 +- .../db/utils/datastructure/BackLongTVList.java | 71 +- .../iotdb/db/utils/datastructure/BackwardSort.java | 66 +- .../iotdb/db/utils/datastructure/BinaryTVList.java | 128 +- .../db/utils/datastructure/BooleanTVList.java | 75 +- .../iotdb/db/utils/datastructure/DoubleTVList.java | 76 +- .../iotdb/db/utils/datastructure/FloatTVList.java | 76 +- .../iotdb/db/utils/datastructure/IntTVList.java | 75 +- .../iotdb/db/utils/datastructure/LongTVList.java | 75 +- .../MergeSortAlignedTVListIterator.java | 242 ++ .../datastructure/MergeSortTvListIterator.java | 162 + ...ickIntTVList.java => PageColumnAccessInfo.java} | 47 +- .../db/utils/datastructure/QuickAlignedTVList.java | 34 +- .../db/utils/datastructure/QuickBinaryTVList.java | 34 +- .../db/utils/datastructure/QuickBooleanTVList.java | 31 +- .../db/utils/datastructure/QuickDoubleTVList.java | 31 +- .../db/utils/datastructure/QuickFloatTVList.java | 31 +- .../db/utils/datastructure/QuickIntTVList.java | 37 +- .../db/utils/datastructure/QuickLongTVList.java | 31 +- .../iotdb/db/utils/datastructure/QuickSort.java | 28 +- .../iotdb/db/utils/datastructure/TVList.java | 363 +- .../db/utils/datastructure/TimAlignedTVList.java | 109 +- .../db/utils/datastructure/TimBinaryTVList.java | 113 +- .../db/utils/datastructure/TimBooleanTVList.java | 111 +- .../db/utils/datastructure/TimDoubleTVList.java | 111 +- .../db/utils/datastructure/TimFloatTVList.java | 112 +- .../iotdb/db/utils/datastructure/TimIntTVList.java | 112 +- .../db/utils/datastructure/TimLongTVList.java | 111 +- .../iotdb/db/utils/datastructure/TimSort.java | 105 +- ....apache.iotdb.db.protocol.mqtt.PayloadFormatter | 1 + .../org/apache/iotdb/db/auth/entity/UserTest.java | 4 +- .../db/protocol/mqtt/JSONPayloadFormatterTest.java | 8 +- .../db/protocol/mqtt/LinePayloadFormatterTest.java | 91 + .../db/protocol/mqtt/PayloadFormatManagerTest.java | 2 +- .../db/protocol/rest/IoTDBRestServiceTest.java | 6 +- .../operator/MergeTreeSortOperatorTest.java | 6 + .../plan/parser/StatementGeneratorTest.java | 3 + .../plan/relational/analyzer/AnalyzerTest.java | 7 +- .../plan/relational/analyzer/ShowQueriesTest.java | 7 +- .../storageengine/dataregion/DataRegionTest.java | 6 +- .../compaction/CompactionDataTypeNotMatchTest.java | 19 +- .../FastCompactionPerformerWithEmptyPageTest.java | 5 +- .../FastInnerCompactionPerformerTest.java | 134 +- .../ReadPointCompactionPerformerTest.java | 134 +- .../TsFileValidationCorrectnessTests.java | 14 +- ...eCompactionWithFastPerformerValidationTest.java | 14 +- ...actionWithReadPointPerformerValidationTest.java | 14 +- .../InsertionCrossSpaceCompactionRecoverTest.java | 4 +- .../InsertionCrossSpaceCompactionSelectorTest.java | 4 +- .../cross/RewriteCompactionFileSelectorTest.java | 6 +- .../CompactionTableModelTestFileWriter.java | 2 +- .../compaction/utils/CompactionCheckerUtils.java | 11 +- .../memtable/MemChunkDeserializeTest.java | 332 ++ .../dataregion/memtable/PrimitiveMemTableTest.java | 41 +- .../dataregion/memtable/TsFileProcessorTest.java | 451 +- .../read/control/FileReaderManagerTest.java | 5 +- .../reader/chunk/AlignedMemPageReaderTest.java | 32 +- .../reader/chunk/MemAlignedChunkLoaderTest.java | 138 +- .../read/reader/chunk/MemChunkLoaderTest.java | 260 +- .../read/reader/chunk/MemPageReaderTest.java | 12 +- .../dataregion/tsfile/TsFileResourceTest.java | 12 +- .../wal/recover/WALRecoverManagerTest.java | 21 +- .../file/SealedTsFileRecoverPerformerTest.java | 33 +- .../wal/recover/file/TsFilePlanRedoerTest.java | 18 +- .../file/UnsealedTsFileRecoverPerformerTest.java | 21 +- .../apache/iotdb/db/utils/DateTimeUtilsTest.java | 14 + .../apache/iotdb/db/utils/EnvironmentUtils.java | 4 - .../db/utils/TimestampPrecisionUtilsTest.java | 10 + .../db/utils/datastructure/AlignedTVListTest.java | 20 +- .../db/utils/datastructure/BinaryTVListTest.java | 30 - .../datanode/src/test/resources/logback-test.xml | 2 +- .../conf/iotdb-system.properties.template | 46 +- .../src/assembly/resources/sbin/health_check.bat | 2 +- .../commons/auth/authorizer/BasicAuthorizer.java | 21 + .../iotdb/commons/auth/authorizer/IAuthorizer.java | 4 + .../commons/auth/authorizer/OpenIdAuthorizer.java | 6 +- .../org/apache/iotdb/commons/auth/entity/Role.java | 8 + .../org/apache/iotdb/commons/auth/entity/User.java | 8 +- .../apache/iotdb/commons/conf/CommonConfig.java | 58 +- .../iotdb/commons/conf/CommonDescriptor.java | 43 +- .../apache/iotdb/commons/conf/IoTDBConstant.java | 1 + .../iotdb/commons/consensus/ConsensusGroupId.java | 7 +- .../commons/pipe/agent/task/PipeTaskAgent.java | 52 +- .../task/connection/BlockingPendingQueue.java | 14 + .../iotdb/commons/pipe/config/PipeConfig.java | 21 +- .../config/constant/PipeConnectorConstant.java | 4 + .../pipe/connector/client/IoTDBClientManager.java | 6 +- .../connector/client/IoTDBSyncClientManager.java | 9 +- .../common/PipeTransferHandshakeConstant.java | 1 + .../pipe/connector/protocol/IoTDBConnector.java | 12 +- .../connector/protocol/IoTDBSslSyncConnector.java | 6 +- .../options/PipeInclusionOptions.java | 15 +- .../commons/pipe/receiver/IoTDBFileReceiver.java | 12 +- .../schema/column/ColumnHeaderConstant.java | 30 + .../commons/schema/table/InformationSchema.java | 109 + .../iotdb/commons/service/metric/enums/Metric.java | 2 + .../relational/TableBuiltinScalarFunction.java | 2 + .../db/relational/grammar/sql/RelationalSql.g4 | 24 +- pom.xml | 19 +- 575 files changed, 22756 insertions(+), 12866 deletions(-) diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index ae88050e4a8,2ff675b0196..8fa597a9c6f --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@@ -433,20 -436,23 +436,23 @@@ public class IoTDBConfig /** The sort algorithm used in TVList */ private TVListSortAlgorithm tvListSortAlgorithm = TVListSortAlgorithm.TIM; - /** When average series point number reaches this, flush the memtable to disk */ - private int avgSeriesPointNumberThreshold = 100000; + /** + * the threshold when working TVList is sorted and added into immutable TVList list in the + * writable memtable + */ + private int tvListSortThreshold = 0; /** Enable inner space compaction for sequence files */ - private volatile boolean enableSeqSpaceCompaction = true; + private volatile boolean enableSeqSpaceCompaction = false; /** Enable inner space compaction for unsequence files */ - private volatile boolean enableUnseqSpaceCompaction = true; + private volatile boolean enableUnseqSpaceCompaction = false; /** Compact the unsequence files into the overlapped sequence files */ - private volatile boolean enableCrossSpaceCompaction = true; + private volatile boolean enableCrossSpaceCompaction = false; /** Enable auto repair compaction */ - private volatile boolean enableAutoRepairCompaction = true; + private volatile boolean enableAutoRepairCompaction = false; /** The buffer for sort operation */ private long sortBufferSize = 1024 * 1024L;
