This is an automated email from the ASF dual-hosted git repository. xingtanzjr pushed a commit to branch ml_add_peer_0914_test in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9fffaab3ae80933197161dadb07f22464c041dbe Merge: d6fe0579f3 bb2ab27369 Author: Jinrui.Zhang <[email protected]> AuthorDate: Wed Sep 14 17:39:25 2022 +0800 Merge branch 'beyyes/fix_procedure_bug' into ml_add_peer_0914_test README.md | 8 +- README_ZH.md | 8 +- RELEASE_NOTES.md | 25 + .../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 5 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 14 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 20 +- .../org/apache/iotdb/tool/AbstractCsvTool.java | 72 ++- .../main/java/org/apache/iotdb/tool/ExportCsv.java | 124 ++-- .../main/java/org/apache/iotdb/tool/ImportCsv.java | 191 ++++-- client-cpp/src/main/CMakeLists.txt | 2 +- client-cpp/src/main/Session.cpp | 57 +- client-cpp/src/main/Session.h | 27 +- .../resources/conf/iotdb-confignode.properties | 16 +- .../confignode/client/DataNodeRequestType.java | 4 +- .../sync/datanode/SyncDataNodeClientPool.java | 14 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 15 +- .../confignode/conf/ConfigNodeDescriptor.java | 13 +- .../confignode/conf/ConfigNodeStartupCheck.java | 7 + .../consensus/request/ConfigPhysicalPlan.java | 1 + .../request/write/CreateRegionGroupsPlan.java | 10 + .../consensus/response/DataNodeRegisterResp.java | 7 + .../statemachine/PartitionRegionStateMachine.java | 4 +- .../iotdb/confignode/manager/ConfigManager.java | 6 +- .../iotdb/confignode/manager/ConsensusManager.java | 32 +- .../apache/iotdb/confignode/manager/IManager.java | 4 +- .../iotdb/confignode/manager/NodeManager.java | 22 +- .../iotdb/confignode/manager/PartitionManager.java | 21 +- .../iotdb/confignode/manager/ProcedureManager.java | 7 +- .../iotdb/confignode/manager/load/LoadManager.java | 6 +- .../manager/load/LoadManagerMetrics.java | 288 +++++---- .../manager/load/balancer/RouteBalancer.java | 5 +- .../partition/GreedyPartitionAllocator.java | 93 ++- .../iotdb/confignode/persistence/NodeInfo.java | 30 - .../iotdb/confignode/persistence/UDFInfo.java | 4 +- .../persistence/metric/NodeInfoMetrics.java | 95 +++ .../persistence/metric/PartitionInfoMetrics.java | 198 ++++++ .../persistence/partition/PartitionInfo.java | 56 +- .../persistence/partition/RegionGroup.java | 68 +- .../partition/StorageGroupPartitionTable.java | 146 ++--- .../procedure/env/DataNodeRemoveHandler.java | 286 ++++++--- .../procedure/impl/AbstractNodeProcedure.java | 6 +- .../impl/CreateRegionGroupsProcedure.java | 85 ++- .../procedure/impl/RegionMigrateProcedure.java | 91 ++- .../procedure/state/RegionTransitionState.java | 4 +- .../procedure/state/RemoveConfigNodeState.java | 3 +- .../procedure/store/ProcedureFactory.java | 10 + .../iotdb/confignode/service/ConfigNode.java | 4 - .../service/thrift/ConfigNodeRPCService.java | 17 +- .../thrift/ConfigNodeRPCServiceHandler.java | 28 +- .../thrift/ConfigNodeRPCServiceHandlerMetrics.java | 69 +++ .../thrift/ConfigNodeRPCServiceMetrics.java | 70 +++ .../thrift/ConfigNodeRPCServiceProcessor.java | 16 +- .../impl/CreateRegionGroupsProcedureTest.java | 110 ++++ .../apache/iotdb/consensus/config/RatisConfig.java | 132 +++- .../consensus/exception/NodeReadOnlyException.java | 18 +- .../multileader/service/MultiLeaderRPCService.java | 4 +- .../ratis/ApplicationStateMachineProxy.java | 29 +- .../iotdb/consensus/ratis/RatisConsensus.java | 87 ++- .../org/apache/iotdb/consensus/ratis/Utils.java | 7 + .../tests/tools/importCsv/ExportCsvTestIT.java | 6 +- .../tests/tools/importCsv/ImportCsvTestIT.java | 2 +- docs/Development/HowtoContributeCode.md | 2 +- docs/Download/README.md | 42 +- docs/UserGuide/API/Status-Codes.md | 5 +- .../Administration-Management/Administration.md | 4 +- docs/UserGuide/Data-Concept/Data-Type.md | 2 +- docs/UserGuide/Data-Concept/Time-Partition.md | 2 +- docs/UserGuide/Data-Modeling/DataRegion.md | 2 +- .../DBeaver.md | 0 .../Flink-IoTDB.md} | 0 .../Flink-TsFile.md} | 0 .../Grafana-Connector.md} | 0 .../Grafana-Plugin.md} | 0 .../Hive-TsFile.md} | 0 .../MapReduce-TsFile.md} | 0 docs/UserGuide/Ecosystem-Integration/NiFi-IoTDB.md | 115 ++++ .../Spark-IoTDB.md} | 0 .../Spark-TsFile.md} | 0 .../Writing-Data-on-HDFS.md} | 0 .../Zeppelin-IoTDB.md | 0 .../{Monitor-and-Log-Tools.md => Log-Tool.md} | 55 +- .../Maintenance-Tools/Maintenance-Command.md | 34 +- docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 128 ++-- docs/UserGuide/Process-Data/Triggers.md | 2 +- docs/UserGuide/Query-Data/Fill-Null-Value.md | 6 +- docs/UserGuide/Query-Data/Last-Query.md | 18 +- docs/UserGuide/QuickStart/Files.md | 8 +- docs/UserGuide/QuickStart/QuickStart.md | 12 +- docs/UserGuide/Write-And-Delete-Data/CSV-Tool.md | 41 +- .../UserGuide/Write-And-Delete-Data/Delete-Data.md | 2 +- docs/zh/Development/HowtoContributeCode.md | 2 +- docs/zh/Download/README.md | 42 +- docs/zh/UserGuide/API/Status-Codes.md | 3 +- .../Administration-Management/Administration.md | 4 +- docs/zh/UserGuide/Data-Concept/Data-Type.md | 2 +- docs/zh/UserGuide/Data-Concept/Time-Partition.md | 2 +- docs/zh/UserGuide/Data-Modeling/DataRegion.md | 2 +- .../DBeaver.md | 0 .../Flink-IoTDB.md} | 0 .../Flink-TsFile.md} | 0 .../Grafana-Connector.md} | 0 .../Grafana-Plugin.md} | 0 .../Hive-TsFile.md} | 0 .../MapReduce-TsFile.md} | 0 .../UserGuide/Ecosystem-Integration/NiFi-IoTDB.md | 115 ++++ .../Spark-IoTDB.md} | 0 .../Spark-TsFile.md} | 0 .../Writing-Data-on-HDFS.md} | 0 .../Zeppelin-IoTDB.md | 0 .../{Monitor-and-Log-Tools.md => Log-Tool.md} | 55 +- .../Maintenance-Tools/Maintenance-Command.md | 34 +- docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 148 +++-- docs/zh/UserGuide/Process-Data/Triggers.md | 2 +- docs/zh/UserGuide/Query-Data/Fill-Null-Value.md | 6 +- docs/zh/UserGuide/Query-Data/Last-Query.md | 18 +- docs/zh/UserGuide/QuickStart/Files.md | 8 +- docs/zh/UserGuide/QuickStart/QuickStart.md | 8 +- .../zh/UserGuide/Write-And-Delete-Data/CSV-Tool.md | 34 +- .../UserGuide/Write-And-Delete-Data/Delete-Data.md | 2 +- .../pom.xml | 4 +- .../org/apache/iotdb/influxdb/InfluxDBExample.java | 0 example/pom.xml | 2 + .../main/java/org/apache/iotdb/HttpExample.java | 7 +- .../main/java/org/apache/iotdb/HttpsExample.java | 16 +- .../src/main/java/org/apache/iotdb/SSLClient.java | 65 +- .../org/apache/iotdb/trigger/AlertingExample.java | 2 +- .../org/apache/iotdb/trigger/TriggerExample.java | 2 +- .../iotdb/influxdb/session/InfluxDBSession.java | 4 +- .../org/apache/iotdb/it/env/ConfigFactory.java | 1 - .../java/org/apache/iotdb/db/it/IoTDBAuthIT.java | 25 +- .../org/apache/iotdb/db/it/IoTDBDeletionIT.java | 2 +- .../apache/iotdb/db/it/IoTDBFilterBetweenIT.java | 32 + .../java/org/apache/iotdb/db/it/IoTDBFilterIT.java | 119 ++++ .../org/apache/iotdb/db/it/IoTDBNestedQueryIT.java | 2 +- .../db/it/aligned/IoTDBAlignedDataDeletionIT.java | 2 +- .../IoTDBClusterPartitionTableTest.java | 160 +++-- .../db/it/{ => confignode}/IoTDBConfigNodeIT.java | 2 +- .../apache/iotdb/db/it/groupby/IoTDBHavingIT.java | 16 + .../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 62 +- .../{IoTDBPipeSinkIT.java => IoTDBPipeIT.java} | 82 ++- .../apache/iotdb/db/it/sync/IoTDBPipeSinkIT.java | 4 +- .../db/engine/trigger/example/Accumulator.java | 2 +- .../iotdb/db/engine/trigger/example/Counter.java | 2 +- .../iotdb/db/integration/IoTDBAuthorizationIT.java | 25 +- .../db/integration/sync/IoTDBSyncReceiverIT.java | 2 +- .../sync/IoTDBSyncReceiverLoaderIT.java | 2 +- .../db/integration/sync/IoTDBSyncSenderIT.java | 1 + .../IoTDBDeletionVersionAdaptionIT.java | 2 +- .../iotdb/session/IoTDBSessionInsertNullT.java | 298 +++++++++ jdbc/README.md | 2 +- .../reporter/DropwizardPrometheusReporter.java | 4 +- .../iotdb/metrics/AbstractMetricService.java | 65 +- .../iotdb/metrics/DoNothingMetricService.java | 2 +- .../apache/iotdb/metrics/config/MetricConfig.java | 4 +- .../{predefined => metricsets}/IMetricSet.java | 19 +- .../predefined/PredefinedMetric.java | 2 +- .../predefined/jvm/JvmClassLoaderMetrics.java | 20 +- .../predefined/jvm/JvmCompileMetrics.java | 20 +- .../predefined/jvm/JvmGcMetrics.java | 117 +++- .../predefined/jvm/JvmMemoryMetrics.java | 55 +- .../metricsets/predefined/jvm/JvmMetrics.java | 52 ++ .../predefined/jvm/JvmThreadMetrics.java | 43 +- .../{ => metricsets}/predefined/jvm/JvmUtils.java | 2 +- .../predefined/logback/LogbackMetrics.java | 104 ++++ .../predefined/logback/MetricsTurboFilter.java | 98 +++ .../iotdb/metrics/predefined/jvm/JvmMetrics.java | 49 -- .../metrics/predefined/logback/LogbackMetrics.java | 181 ------ .../reporter/MicrometerPrometheusReporter.java | 5 +- node-commons/pom.xml | 5 + .../iotdb/commons/client/ClientPoolFactory.java | 6 +- .../apache/iotdb/commons/cluster/NodeStatus.java | 8 +- .../iotdb/commons/concurrent/ThreadName.java | 62 +- .../apache/iotdb/commons/conf/CommonConfig.java | 68 ++ .../iotdb/commons/conf/CommonDescriptor.java | 9 + .../apache/iotdb/commons/conf/IoTDBConstant.java | 4 +- .../commons/enums}/HandleSystemErrorStrategy.java | 19 +- .../commons/exception/ConfigurationException.java | 23 +- .../commons/executable/ExecutableManager.java | 169 +++++ .../ExecutableResource.java} | 6 +- .../iotdb/commons/file/SystemFileFactory.java | 11 + .../commons/partition/DataPartitionTable.java | 14 +- .../commons/partition/SchemaPartitionTable.java | 18 +- .../commons/partition/SeriesPartitionTable.java | 19 +- .../org/apache/iotdb/commons/path/PartialPath.java | 120 +++- .../apache/iotdb/commons/service/ServiceType.java | 5 +- .../apache/iotdb/commons/sync/SyncConstant.java | 4 +- .../apache/iotdb/commons/sync/SyncPathUtil.java | 18 + .../iotdb/commons/trigger/TriggerInformation.java | 169 +++++ .../apache/iotdb/commons/trigger/TriggerTable.java | 76 +++ .../enums/TriggerEvent.java} | 41 +- .../enums/TriggerType.java} | 36 +- .../exception/TriggerExecutionException.java | 13 +- .../exception/TriggerJarToLargeException.java | 13 +- .../exception/TriggerManagementException.java | 13 +- .../trigger/service/TriggerClassLoader.java | 4 +- .../service/TriggerClassLoaderManager.java} | 74 +-- .../trigger/service/TriggerExecutableManager.java | 73 +++ .../commons/udf/service/UDFClassLoaderManager.java | 12 +- .../commons/udf/service/UDFExecutableManager.java | 108 +--- .../udf/service/UDFRegistrationService.java | 3 +- .../apache/iotdb/commons/utils/NodeUrlUtils.java | 7 + .../apache/iotdb/commons/utils/StatusUtils.java | 2 +- .../client/mock/MockInternalRPCService.java | 4 +- .../apache/iotdb/commons/path/PartialPathTest.java | 66 +- openapi/src/main/openapi3/iotdb-rest.yaml | 10 +- pom.xml | 1 + .../schemaregion/rocksdb/RSchemaRegion.java | 7 + .../resources/conf/iotdb-datanode.properties | 22 +- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 15 +- .../apache/iotdb/db/client/ConfigNodeClient.java | 70 ++- .../iotdb/db/client/DataNodeClientPoolFactory.java | 16 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 129 ++-- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 68 +- .../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 6 +- .../db/conf/directories/DirectoryManager.java | 5 +- .../iotdb/db/conf/directories/FolderManager.java | 6 +- .../directories/strategy/DirectoryStrategy.java | 4 +- .../db/consensus/DataRegionConsensusImpl.java | 5 + .../consensus/statemachine/BaseStateMachine.java | 2 +- .../statemachine/DataRegionStateMachine.java | 4 +- .../statemachine/SchemaRegionStateMachine.java | 4 +- .../statemachine/visitor/DataExecutionVisitor.java | 2 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 5 +- .../apache/iotdb/db/engine/StorageEngineV2.java | 9 +- .../apache/iotdb/db/engine/cache/ChunkCache.java | 16 +- .../iotdb/db/engine/cache/ChunkCacheMetrics.java | 67 ++ .../db/engine/cache/TimeSeriesMetadataCache.java | 35 +- .../cache/TimeSeriesMetadataCacheMetrics.java | 77 +++ .../compaction/task/CompactionRecoverTask.java | 3 +- .../apache/iotdb/db/engine/flush/FlushManager.java | 31 +- .../iotdb/db/engine/flush/FlushManagerMBean.java | 2 + .../iotdb/db/engine/flush/FlushManagerMetrics.java | 90 +++ .../engine/memtable/AlignedWritableMemChunk.java | 160 +++-- .../iotdb/db/engine/snapshot/SnapshotLoader.java | 8 + .../iotdb/db/engine/snapshot/SnapshotTaker.java | 3 + .../iotdb/db/engine/storagegroup/DataRegion.java | 86 ++- .../db/engine/storagegroup/DataRegionMetrics.java | 73 +++ .../db/engine/storagegroup/TsFileManager.java | 14 +- .../db/engine/storagegroup/TsFileProcessor.java | 39 +- .../engine/storagegroup/TsFileProcessorInfo.java | 32 +- .../storagegroup/TsFileProcessorInfoMetrics.java | 74 +++ .../dataregion/StorageGroupManager.java | 10 - .../iotdb/db/engine/trigger/api/Trigger.java | 1 + .../db/engine/trigger/builtin/ForwardTrigger.java | 2 +- .../engine/trigger/executor/TriggerExecutor.java | 4 +- .../trigger/service/TriggerClassLoaderManager.java | 1 + .../service/TriggerRegistrationService.java | 3 +- .../trigger/sink/local/LocalIoTDBHandler.java | 4 +- .../db/exception/sync/PipeServerException.java | 2 +- .../iotdb/db/localconfignode/LocalConfigNode.java | 68 +- .../db/metadata/cache/DataNodeSchemaCache.java | 16 +- .../metadata/cache/DataNodeSchemaCacheMetrics.java | 67 ++ .../iotdb/db/metadata/logfile/MLogWriter.java | 4 +- .../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 34 +- .../db/metadata/rescon/SchemaResourceManager.java | 4 +- .../metadata/rescon/SchemaStatisticsManager.java | 16 - .../rescon/SchemaStatisticsManagerMetrics.java | 67 ++ .../db/metadata/schemaregion/ISchemaRegion.java | 4 + .../schemaregion/SchemaRegionMemoryImpl.java | 18 +- .../schemaregion/SchemaRegionSchemaFileImpl.java | 18 +- .../iotdb/db/mpp/common/MPPQueryContext.java | 7 + .../org/apache/iotdb/db/mpp/common/NodeRef.java | 61 ++ .../db/mpp/common/header/ColumnHeaderConstant.java | 20 +- .../db/mpp/common/header/DatasetHeaderFactory.java | 4 + .../db/mpp/common/schematree/PathPatternTree.java | 12 +- .../execution/exchange/MPPDataExchangeManager.java | 2 +- .../execution/exchange/MPPDataExchangeService.java | 16 +- .../exchange/MPPDataExchangeServiceMetrics.java | 65 ++ .../MPPDataExchangeServiceThriftHandler.java | 27 +- ...ppDataExchangeServiceThriftHandlerMetrics.java} | 51 +- .../db/mpp/execution/exchange/SinkHandle.java | 6 +- .../fragment/FragmentInstanceContext.java | 5 + .../fragment/FragmentInstanceManager.java | 25 +- .../operator/process/TransformOperator.java | 11 +- .../operator/schema/TimeSeriesCountOperator.java | 9 +- .../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 31 +- .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 158 ++--- .../apache/iotdb/db/mpp/plan/analyze/Analyzer.java | 5 +- .../db/mpp/plan/analyze/ExpressionAnalyzer.java | 214 +++---- .../mpp/plan/analyze/ExpressionTypeAnalyzer.java | 310 ++++++++++ .../mpp/plan/analyze/GroupByLevelController.java | 4 +- .../iotdb/db/mpp/plan/analyze/TypeProvider.java | 15 +- .../iotdb/db/mpp/plan/constant/StatementType.java | 2 +- .../plan/execution/config/ConfigTaskVisitor.java | 26 +- .../config/executor/ClusterConfigTaskExecutor.java | 108 +++- .../config/executor/IConfigTaskExecutor.java | 26 +- .../executor/StandaloneConfigTaskExecutor.java | 152 +++-- .../CreateTriggerTask.java} | 14 +- .../DropTriggerTask.java} | 15 +- .../config/metadata/ShowConfigNodesTask.java | 1 + .../execution/config/sys/sync/CreatePipeTask.java | 4 +- .../execution/config/sys/sync/DropPipeTask.java | 4 +- .../execution/config/sys/sync/ShowPipeTask.java | 41 +- .../execution/config/sys/sync/StartPipeTask.java | 4 +- .../execution/config/sys/sync/StopPipeTask.java | 4 +- .../iotdb/db/mpp/plan/expression/Expression.java | 27 +- .../binary/ArithmeticBinaryExpression.java | 25 +- .../plan/expression/binary/BinaryExpression.java | 8 +- .../expression/binary/CompareBinaryExpression.java | 54 +- .../expression/binary/LogicBinaryExpression.java | 15 +- .../mpp/plan/expression/leaf/ConstantOperand.java | 7 - .../db/mpp/plan/expression/leaf/LeafOperand.java | 5 +- .../plan/expression/leaf/TimeSeriesOperand.java | 7 - .../mpp/plan/expression/leaf/TimestampOperand.java | 8 - .../plan/expression/multi/FunctionExpression.java | 44 +- .../plan/expression/ternary/BetweenExpression.java | 20 +- .../plan/expression/ternary/TernaryExpression.java | 11 +- .../db/mpp/plan/expression/unary/InExpression.java | 18 +- .../plan/expression/unary/IsNullExpression.java | 18 +- .../mpp/plan/expression/unary/LikeExpression.java | 20 +- .../plan/expression/unary/LogicNotExpression.java | 20 +- .../plan/expression/unary/NegationExpression.java | 26 +- .../plan/expression/unary/RegularExpression.java | 20 +- .../mpp/plan/expression/unary/UnaryExpression.java | 7 +- .../visitor/ColumnTransformerVisitor.java | 69 +-- .../plan/expression/visitor/ExpressionVisitor.java | 63 +- .../visitor/IntermediateLayerVisitor.java | 17 +- .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 57 +- .../db/mpp/plan/planner/LogicalPlanBuilder.java | 100 ++- .../db/mpp/plan/planner/LogicalPlanVisitor.java | 55 +- .../db/mpp/plan/planner/OperatorTreeGenerator.java | 34 +- .../db/mpp/plan/planner/SubPlanTypeExtractor.java | 110 ++++ .../SimpleFragmentParallelPlanner.java | 5 +- .../plan/planner/distribution/SourceRewriter.java | 18 +- .../db/mpp/plan/planner/plan/PlanFragment.java | 9 +- .../mpp/plan/planner/plan/node/PlanNodeType.java | 4 + .../plan/planner/plan/node/SimplePlanVisitor.java | 15 +- .../node/metedata/read/TimeSeriesCountNode.java | 37 +- .../planner/plan/node/write/DeleteDataNode.java | 86 ++- .../db/mpp/plan/scheduler/StandaloneScheduler.java | 4 +- .../db/mpp/plan/statement/StatementVisitor.java | 11 + .../db/mpp/plan/statement/crud/QueryStatement.java | 12 +- .../plan/statement/metadata/CountStatement.java | 16 +- .../statement/metadata/CreateTriggerStatement.java | 121 ++++ .../DropTriggerStatement.java} | 57 +- .../statement/sys/sync/CreatePipeStatement.java | 38 ++ .../dag/builder/EvaluationDAGBuilder.java | 16 +- .../binary/CompareBinaryColumnTransformer.java | 12 +- .../handler/StatementConstructionHandler.java | 17 +- .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 36 +- .../iotdb/db/protocol/mqtt/PublishHandler.java | 39 +- .../handler/PhysicalPlanConstructionHandler.java | 17 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 4 +- .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 7 +- .../iotdb/db/query/control/SessionManager.java | 21 +- .../db/query/pool/RawQueryReadTaskPoolManager.java | 32 +- .../pool/RawQueryReadTaskPoolManagerMetrics.java | 92 +++ .../java/org/apache/iotdb/db/service/DataNode.java | 38 +- .../db/service/DataNodeInternalRPCService.java | 16 +- .../service/DataNodeInternalRPCServiceMetrics.java | 72 +++ .../db/service/DataNodeServerCommandLine.java | 7 +- .../iotdb/db/service/InfluxDBRPCService.java | 4 +- .../java/org/apache/iotdb/db/service/IoTDB.java | 5 +- .../java/org/apache/iotdb/db/service/NewIoTDB.java | 5 +- .../org/apache/iotdb/db/service/RPCService.java | 19 +- .../apache/iotdb/db/service/RPCServiceMetrics.java | 71 +++ .../iotdb/db/service/RegionMigrateService.java | 163 +++-- .../service/basic/StandaloneServiceProvider.java | 4 +- .../iotdb/db/service/metrics/MetricService.java | 31 +- .../db/service/metrics/predefined/FileMetrics.java | 72 ++- .../service/metrics/predefined/ProcessMetrics.java | 80 ++- .../service/metrics/predefined/SystemMetrics.java | 116 +++- .../handler/InternalServiceThriftHandler.java | 22 +- .../InternalServiceThriftHandlerMetrics.java | 68 ++ .../thrift/handler/RPCServiceThriftHandler.java | 19 +- .../handler/RPCServiceThriftHandlerMetrics.java | 65 ++ .../service/thrift/impl/ClientRPCServiceImpl.java | 3 +- .../impl/DataNodeInternalRPCServiceImpl.java | 151 +++-- .../db/service/thrift/impl/TSServiceImpl.java | 8 +- .../java/org/apache/iotdb/db/sync/SyncService.java | 160 +++-- .../iotdb/db/sync/common/ISyncInfoFetcher.java | 5 +- .../iotdb/db/sync/common/LocalSyncInfoFetcher.java | 19 +- .../org/apache/iotdb/db/sync/common/SyncInfo.java | 35 +- .../db/sync/common/persistence/SyncLogReader.java | 8 +- .../db/sync/common/persistence/SyncLogWriter.java | 17 +- .../iotdb/db/sync/pipedata/TsFilePipeData.java | 6 +- .../sync/pipedata/queue/BufferedPipeDataQueue.java | 1 + .../db/sync/pipedata/queue/PipeDataQueue.java | 2 + .../db/sync/receiver/load/DeletionLoader.java | 4 +- .../iotdb/db/sync/receiver/load/TsFileLoader.java | 59 +- .../sender/manager/ISyncManager.java} | 40 +- .../db/sync/sender/manager/LocalSyncManager.java | 92 +++ .../db/sync/sender/manager/SchemaSyncManager.java | 149 ----- .../db/sync/sender/manager/TsFileSyncManager.java | 106 ---- .../org/apache/iotdb/db/sync/sender/pipe/Pipe.java | 24 +- .../iotdb/db/sync/sender/pipe/TsFilePipe.java | 290 +++++---- .../db/sync/transport/client/IoTDBSyncClient.java | 21 +- .../db/sync/transport/client/SenderManager.java | 99 ++- .../sync/transport/client/SyncClientFactory.java | 14 +- .../db/sync/transport/server/ReceiverManager.java | 4 +- .../iotdb/db/trigger/executor/TriggerExecutor.java | 57 ++ .../trigger/service/TriggerManagementService.java | 189 ++++++ .../apache/iotdb/db/utils/ErrorHandlingUtils.java | 8 +- .../org/apache/iotdb/db/utils/OpenFileNumUtil.java | 5 +- .../db/utils/datastructure/AlignedTVList.java | 267 +------- .../iotdb/db/utils/datastructure/BinaryTVList.java | 110 +--- .../db/utils/datastructure/BooleanTVList.java | 112 +--- .../iotdb/db/utils/datastructure/DoubleTVList.java | 110 +--- .../iotdb/db/utils/datastructure/FloatTVList.java | 110 +--- .../iotdb/db/utils/datastructure/IntTVList.java | 125 +--- .../iotdb/db/utils/datastructure/LongTVList.java | 110 +--- .../iotdb/db/utils/datastructure/TVList.java | 204 +----- .../db/utils/datastructure/TimAlignedTVList.java | 300 +++++++++ .../db/utils/datastructure/TimBinaryTVList.java | 163 +++++ .../db/utils/datastructure/TimBooleanTVList.java | 163 +++++ .../db/utils/datastructure/TimDoubleTVList.java | 162 +++++ .../db/utils/datastructure/TimFloatTVList.java | 163 +++++ .../iotdb/db/utils/datastructure/TimIntTVList.java | 147 +++++ .../db/utils/datastructure/TimLongTVList.java | 162 +++++ .../iotdb/db/utils/datastructure/TimSort.java | 190 ++++++ .../apache/iotdb/db/utils/sync/SyncPipeUtil.java | 47 ++ .../java/org/apache/iotdb/db/wal/WALManager.java | 7 +- .../allocation/AbstractNodeAllocationStrategy.java | 8 +- .../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 23 +- .../org/apache/iotdb/db/wal/buffer/WALEntry.java | 11 +- .../apache/iotdb/db/wal/buffer/WALEntryType.java | 9 +- .../apache/iotdb/db/wal/buffer/WALInfoEntry.java | 1 + .../iotdb/db/wal/checkpoint/CheckpointManager.java | 6 +- .../org/apache/iotdb/db/wal/node/IWALNode.java | 4 + .../org/apache/iotdb/db/wal/node/WALFakeNode.java | 6 + .../java/org/apache/iotdb/db/wal/node/WALNode.java | 17 +- .../iotdb/db/wal/recover/WALRecoverManager.java | 8 +- .../db/wal/recover/file/TsFilePlanRedoer.java | 21 + .../file/UnsealedTsFileRecoverPerformer.java | 4 + .../db/engine/snapshot/IoTDBSnapshotTest.java | 38 ++ .../db/engine/storagegroup/DataRegionTest.java | 3 +- .../storagegroup/StorageGroupProcessorTest.java | 3 +- .../engine/storagegroup/TsFileProcessorTest.java | 6 +- .../metadata/idtable/trigger_example/Counter.java | 2 +- .../apache/iotdb/db/metric/MetricServiceTest.java | 1 - .../mpp/common/schematree/PathPatternTreeTest.java | 33 + .../mpp/execution/operator/OperatorMemoryTest.java | 3 +- .../operator/schema/CountMergeOperatorTest.java | 7 +- .../operator/schema/SchemaCountOperatorTest.java | 7 +- .../mpp/plan/plan/distribution/LastQueryTest.java | 26 +- .../iotdb/db/protocol/mqtt/PublishHandlerTest.java | 7 +- .../db/sync/receiver/manager/SyncInfoTest.java | 8 +- .../db/sync/receiver/recovery/SyncLogTest.java | 8 +- .../iotdb/db/sync/transport/SyncTransportTest.java | 11 +- .../apache/iotdb/db/utils/EnvironmentUtils.java | 6 +- .../db/utils/datastructure/BinaryTVListTest.java | 8 +- .../db/utils/datastructure/BooleanTVListTest.java | 8 +- .../db/utils/datastructure/DoubleTVListTest.java | 12 +- .../db/utils/datastructure/FloatTVListTest.java | 10 +- .../db/utils/datastructure/IntTVListTest.java | 12 +- .../db/utils/datastructure/LongTVListTest.java | 12 +- .../db/utils/datastructure/VectorTVListTest.java | 10 +- .../org/apache/iotdb/db/wal/WALManagerTest.java | 9 +- .../db/wal/allocation/FirstCreateStrategyTest.java | 9 +- .../db/wal/allocation/RoundRobinStrategyTest.java | 9 +- .../org/apache/iotdb/db/wal/io/WALFileTest.java | 14 + .../iotdb/db/wal/node/ConsensusReqReaderTest.java | 99 ++- .../db/wal/recover/WALRecoverManagerTest.java | 5 +- .../db/wal/recover/file/TsFilePlanRedoerTest.java | 33 + .../apache/iotdb/rpc/NoValidValueException.java | 14 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 4 +- .../java/org/apache/iotdb/session/Session.java | 685 +++++++++++++++++++-- .../apache/iotdb/session/SessionConnection.java | 4 +- site/src/main/.vuepress/config.js | 58 +- .../apache/iotdb/spark/db/EnvironmentUtils.java | 26 +- thrift-commons/src/main/thrift/common.thrift | 11 + .../src/main/thrift/confignode.thrift | 72 ++- thrift/src/main/thrift/client.thrift | 1 + thrift/src/main/thrift/datanode.thrift | 118 ++-- trigger-api/pom.xml | 73 +++ .../java/org/apache/iotdb/trigger/api/Trigger.java | 68 ++ .../iotdb}/trigger/api/TriggerAttributes.java | 2 +- .../tsfile/write/chunk/AlignedChunkWriterImpl.java | 28 + 468 files changed, 12788 insertions(+), 5478 deletions(-) diff --cc server/src/main/java/org/apache/iotdb/db/consensus/statemachine/DataRegionStateMachine.java index bd0218d92e,f4e1988373..45b4c58fe7 --- a/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/DataRegionStateMachine.java +++ b/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/DataRegionStateMachine.java @@@ -27,12 -28,8 +28,11 @@@ import org.apache.iotdb.consensus.commo import org.apache.iotdb.consensus.common.request.IConsensusRequest; import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest; import org.apache.iotdb.consensus.multileader.wal.GetConsensusReqReaderPlan; - import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.consensus.statemachine.visitor.DataExecutionVisitor; import org.apache.iotdb.db.engine.StorageEngineV2; +import org.apache.iotdb.db.engine.cache.BloomFilterCache; +import org.apache.iotdb.db.engine.cache.ChunkCache; +import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.snapshot.SnapshotLoader; import org.apache.iotdb.db.engine.snapshot.SnapshotTaker; import org.apache.iotdb.db.engine.storagegroup.DataRegion; diff --cc server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLoader.java index 8c96b2b693,e3fdf3750a..97771dcc0b --- a/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLoader.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLoader.java @@@ -114,13 -114,14 +114,20 @@@ public class SnapshotLoader private DataRegion loadSnapshotWithoutLog() { try { + try { + deleteAllFilesInDataDirs(); + LOGGER.info("Remove all data files in original data dir"); + } catch (IOException e) { + LOGGER.error("Failed to remove origin data files", e); + return null; + } LOGGER.info("Moving snapshot file to data dirs"); + try { + deleteAllFilesInDataDirs(); + LOGGER.info("Remove all data files in original data dir"); + } catch (IOException e) { + return null; + } createLinksFromSnapshotDirToDataDirWithoutLog(new File(snapshotPath)); return loadSnapshot(); } catch (IOException | DiskSpaceInsufficientException e) { diff --cc server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotTaker.java index ab12778351,67b3df98f0..d90eb08a9e --- a/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotTaker.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotTaker.java @@@ -145,10 -145,10 +145,13 @@@ public class SnapshotTaker private boolean createSnapshot(List<TsFileResource> resources, String snapshotId) { try { for (TsFileResource resource : resources) { + if (!resource.isClosed()) { + continue; + } File tsFile = resource.getTsFile(); + if (!resource.isClosed()) { + continue; + } File snapshotTsFile = getSnapshotFilePathForTsFile(tsFile, snapshotId); // create hard link for tsfile, resource, mods createHardLink(snapshotTsFile, tsFile);
