This is an automated email from the ASF dual-hosted git repository. xingtanzjr pushed a commit to branch ml_0808_test_exp1_parallel in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8b31ab57055ffd0353c3e54b34ad445a5c5517ae Merge: 3a743f41ae a04a1eb62b Author: Jinrui.Zhang <[email protected]> AuthorDate: Tue Aug 9 15:50:32 2022 +0800 Merge branch 'master' into ml_0808_test_exp1_parallel .github/workflows/client-cpp.yml | 4 +- .github/workflows/influxdb-protocol.yml | 2 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 45 +- checkstyle.xml | 7 + client-cpp/src/main/Session.cpp | 12 + client-cpp/src/test/cpp/sessionIT.cpp | 28 ++ .../assembly/resources/sbin/remove-confignode.sh | 40 +- .../assembly/resources/sbin/start-confignode.sh | 36 +- .../src/assembly/resources/sbin/stop-confignode.sh | 11 +- .../confignode/client/DataNodeRequestType.java | 4 +- ...ava => AsyncConfigNodeHeartbeatClientPool.java} | 42 +- .../async/datanode/AsyncDataNodeClientPool.java | 40 +- .../datanode/AsyncDataNodeHeartbeatClientPool.java | 72 +++ .../async/handlers/ConfigNodeHeartbeatHandler.java | 16 +- .../async/handlers/DataNodeHeartbeatHandler.java | 10 +- .../client/async/handlers/MergeHandler.java | 83 ++++ .../sync/confignode/SyncConfigNodeClientPool.java | 22 +- .../sync/datanode/SyncDataNodeClientPool.java | 4 +- .../iotdb/confignode/conf/ConfigNodeConfig.java | 41 -- .../confignode/conf/ConfigNodeDescriptor.java | 17 - .../consensus/request/ConfigPhysicalPlan.java | 4 + .../consensus/request/ConfigPhysicalPlanType.java | 2 + .../consensus/request/auth/AuthorPlan.java | 25 +- .../consensus/request/write/SetTTLPlan.java | 29 +- .../exception/AddConsensusGroupException.java | 11 +- .../confignode/manager/ClusterSchemaManager.java | 83 ++-- .../iotdb/confignode/manager/ConfigManager.java | 101 ++-- .../apache/iotdb/confignode/manager/IManager.java | 10 +- .../iotdb/confignode/manager/NodeManager.java | 135 ++---- .../iotdb/confignode/manager/PartitionManager.java | 21 +- .../iotdb/confignode/manager/ProcedureManager.java | 29 +- .../iotdb/confignode/manager/load/LoadManager.java | 350 ++++++++++++-- .../manager/load/balancer/RouteBalancer.java | 60 ++- .../load/balancer/router/LazyGreedyRouter.java | 154 +++++++ .../load/heartbeat/ConfigNodeHeartbeatCache.java | 2 +- .../manager/load/heartbeat/IRegionGroupCache.java | 9 + .../manager/load/heartbeat/RegionGroupCache.java | 7 +- .../iotdb/confignode/persistence/AuthorInfo.java | 167 ++++--- .../iotdb/confignode/persistence/NodeInfo.java | 48 +- .../persistence/executor/ConfigPlanExecutor.java | 10 +- .../persistence/schema/ClusterSchemaInfo.java | 34 +- .../procedure/env/ConfigNodeProcedureEnv.java | 20 +- .../procedure/env/DataNodeRemoveHandler.java | 23 +- .../procedure/impl/RegionMigrateProcedure.java | 12 +- .../procedure/state/RegionTransitionState.java | 1 - .../iotdb/confignode/service/ConfigNode.java | 26 +- .../service/thrift/ConfigNodeRPCService.java | 15 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 41 +- .../request/ConfigPhysicalPlanSerDeTest.java | 123 ++++- .../load/balancer/router/LazyGreedyRouterTest.java | 166 +++++++ .../load/balancer/router/LeaderRouterTest.java | 2 +- .../confignode/persistence/AuthorInfoTest.java | 344 ++++++++++++-- .../thrift/ConfigNodeRPCServiceProcessorTest.java | 84 ++-- docker/src/main/Dockerfile-single-influxdb | 4 +- docs/Download/README.md | 28 +- .../Administration-Management/Administration.md | 97 ++-- .../UserGuide/Ecosystem Integration/Spark IoTDB.md | 2 +- .../Maintenance-Tools/Maintenance-Command.md | 14 +- docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 10 +- docs/UserGuide/Operate-Metadata/TTL.md | 21 +- docs/UserGuide/Reference/SQL-Reference.md | 4 +- docs/UserGuide/UDF-Library/Series-Processing.md | 70 +++ docs/UserGuide/Write-And-Delete-Data/CSV-Tool.md | 14 +- docs/zh/Development/VoteRelease.md | 4 +- docs/zh/Download/README.md | 28 +- .../Administration-Management/Administration.md | 97 ++-- .../UserGuide/Ecosystem Integration/Spark IoTDB.md | 2 +- .../Maintenance-Tools/Maintenance-Command.md | 16 +- docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 22 +- docs/zh/UserGuide/Operate-Metadata/TTL.md | 18 + .../Query-Data/Performance-Tracing-Tool.md | 30 +- docs/zh/UserGuide/Query-Data/Tracing-Tool.md | 30 +- docs/zh/UserGuide/QuickStart/WayToGetIoTDB.md | 2 +- docs/zh/UserGuide/Reference/SQL-Reference.md | 2 +- docs/zh/UserGuide/UDF-Library/Series-Processing.md | 69 +++ .../zh/UserGuide/Write-And-Delete-Data/CSV-Tool.md | 14 +- .../Apache IoTDB ConfigNode Dashboard v0.14.0.json | 275 ++++++++++- .../Apache IoTDB DataNode Dashboard v0.14.0.json | 289 +++++++++++- grafana-plugin/backend-compile.bat | 31 ++ grafana-plugin/go.mod | 2 +- grafana-plugin/go.sum | 15 +- grafana-plugin/pom.xml | 54 ++- integration-test/import-control.xml | 8 + .../java/org/apache/iotdb/it/env/AbstractEnv.java | 115 ++--- .../org/apache/iotdb/it/env/ConfigFactory.java | 1 + .../org/apache/iotdb/it/env/ConfigNodeWrapper.java | 4 + .../org/apache/iotdb/it/env/DataNodeWrapper.java | 16 + .../java/org/apache/iotdb/it/env/EnvUtils.java | 90 ++++ .../java/org/apache/iotdb/it/env/MppConfig.java | 22 + .../org/apache/iotdb/it/env/RemoteServerEnv.java | 28 ++ .../apache/iotdb/it/env/StandaloneOnMppEnv.java | 5 +- .../BuiltinTimeSeriesGeneratingFunctionEnum.java | 1 + .../org/apache/iotdb/itbase/env/BaseConfig.java | 25 + .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 15 + .../org/apache/iotdb/db/it/IoTDBConfigNodeIT.java | 209 +++++++++ .../apache/iotdb/db/it/IoTDBFlushQueryMergeIT.java | 193 ++++++++ .../java/org/apache/iotdb/db/it/IoTDBTtlIT.java | 269 +++++++++++ .../IoTDBAlignedSeriesQueryWithDeletionIT.java | 139 ++++++ .../db/it/aligned/IoTDBInsertAlignedValuesIT.java | 12 + .../org/apache/iotdb/db/it/env/StandaloneEnv.java | 30 ++ .../db/it/udf/IoTDBUDTFBuiltinFunctionIT.java | 89 +++- .../iotdb/db/integration/IoTDBAuthorizationIT.java | 20 +- .../apache/iotdb/db/integration/IoTDBTtlIT.java | 130 ++++++ library-udf/pom.xml | 6 - .../micrometer/MicrometerMetricManagerTest.java | 11 + .../apache/iotdb/commons/client/ClientManager.java | 13 +- .../iotdb/commons/client/ClientPoolFactory.java | 56 ++- .../iotdb/commons/client/IClientManager.java | 5 + .../AsyncConfigNodeHeartbeatServiceClient.java | 140 ++++++ .../async/AsyncDataNodeHeartbeatServiceClient.java | 140 ++++++ .../apache/iotdb/commons/cluster/NodeStatus.java | 4 + .../apache/iotdb/commons/conf/CommonConfig.java | 40 ++ .../iotdb/commons/conf/CommonDescriptor.java | 17 + .../iotdb/commons/partition/DataPartition.java | 1 + .../BuiltinTimeSeriesGeneratingFunction.java | 1 + .../commons/udf/builtin/UDTFDeDuplication.java | 139 ++++++ .../commons/utils/BasicStructureSerDeUtil.java | 31 ++ pom.xml | 8 +- .../resources/conf/iotdb-datanode.properties | 4 +- .../iotdb/db/auth/ClusterAuthorityFetcher.java | 9 +- .../apache/iotdb/db/client/ConfigNodeClient.java | 28 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 6 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 23 +- .../apache/iotdb/db/engine/StorageEngineV2.java | 30 +- .../db/engine/compaction/CompactionWorker.java | 34 +- .../RewriteCrossSpaceCompactionSelector.java | 7 - .../impl/ReadPointCompactionPerformer.java | 18 +- .../writer/AbstractCompactionWriter.java | 7 +- .../writer/CrossSpaceCompactionWriter.java | 19 +- .../writer/InnerSpaceCompactionWriter.java | 15 +- .../iotdb/db/localconfignode/LocalConfigNode.java | 173 ++++--- .../db/metadata/cache/DataNodeSchemaCache.java | 9 +- .../{SchemaTree.java => ClusterSchemaTree.java} | 22 +- .../db/mpp/common/schematree/ISchemaTree.java | 70 +++ .../iotdb/db/mpp/execution/driver/Driver.java | 3 +- .../db/mpp/execution/exchange/ISinkHandle.java | 2 +- .../db/mpp/execution/exchange/LocalSinkHandle.java | 10 +- .../db/mpp/execution/exchange/SinkHandle.java | 52 ++- .../iotdb/db/mpp/execution/memory/MemoryPool.java | 2 +- .../operator/process/DeviceMergeOperator.java | 2 +- .../operator/process/FilterAndProjectOperator.java | 2 +- .../RowBasedTimeJoinOperator.java} | 62 +-- .../process/{ => join}/TimeJoinOperator.java | 7 +- .../{ => join}/merge/AscTimeComparator.java | 2 +- .../process/{ => join}/merge/ColumnMerger.java | 24 +- .../{ => join}/merge/DescTimeComparator.java | 2 +- .../{ => join}/merge/MultiColumnMerger.java | 56 ++- .../merge/NonOverlappedMultiColumnMerger.java | 25 +- .../{ => join}/merge/SingleColumnMerger.java | 44 +- .../process/{ => join}/merge/TimeComparator.java | 2 +- .../operator/schema/SchemaFetchMergeOperator.java | 4 +- .../operator/schema/SchemaFetchScanOperator.java | 8 +- .../schema/TimeSeriesSchemaScanOperator.java | 25 +- .../org/apache/iotdb/db/mpp/plan/Coordinator.java | 5 +- .../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 8 +- .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 51 +- .../db/mpp/plan/analyze/ClusterSchemaFetcher.java | 41 +- .../db/mpp/plan/analyze/ExpressionAnalyzer.java | 10 +- .../db/mpp/plan/analyze/FakeSchemaFetcherImpl.java | 11 +- .../iotdb/db/mpp/plan/analyze/ISchemaFetcher.java | 8 +- .../iotdb/db/mpp/plan/analyze/SchemaValidator.java | 6 +- .../mpp/plan/analyze/StandaloneSchemaFetcher.java | 33 +- .../db/mpp/plan/execution/QueryExecution.java | 3 +- .../mpp/plan/execution/config/ConfigExecution.java | 6 +- .../plan/execution/config/ConfigTaskVisitor.java | 29 ++ .../config/executor/ClusterConfigTaskExecutor.java | 52 ++- .../config/executor/IConfigTaskExecutor.java | 5 +- .../executor/StandaloneConfigTaskExecutor.java | 36 +- .../{ => metadata}/CountStorageGroupTask.java | 4 +- .../config/{ => metadata}/CreateFunctionTask.java | 4 +- .../{ => metadata}/DeleteStorageGroupTask.java | 4 +- .../config/{ => metadata}/DropFunctionTask.java | 4 +- .../config/{ => metadata}/SetStorageGroupTask.java | 4 +- .../config/{ => metadata}/SetTTLTask.java | 4 +- .../config/{ => metadata}/ShowClusterTask.java | 4 +- .../config/{ => metadata}/ShowConfigNodesTask.java | 4 +- .../config/{ => metadata}/ShowDataNodesTask.java | 4 +- .../config/{ => metadata}/ShowFunctionsTask.java | 4 +- .../config/{ => metadata}/ShowRegionTask.java | 4 +- .../{ => metadata}/ShowStorageGroupTask.java | 4 +- .../config/{ => metadata}/ShowTTLTask.java | 4 +- .../config/{ => metadata}/UnSetTTLTask.java | 2 +- .../template}/CreateSchemaTemplateTask.java | 4 +- .../template}/SetSchemaTemplateTask.java | 4 +- .../template}/ShowNodesInSchemaTemplateTask.java | 4 +- .../template}/ShowPathSetTemplateTask.java | 4 +- .../template}/ShowSchemaTemplateTask.java | 4 +- .../execution/config/{ => sys}/AuthorizerTask.java | 4 +- .../execution/config/{ => sys}/ClearCacheTask.java | 6 +- .../plan/execution/config/{ => sys}/FlushTask.java | 4 +- .../{ClearCacheTask.java => sys/MergeTask.java} | 29 +- .../iotdb/db/mpp/plan/expression/Expression.java | 144 +++--- .../plan/expression/binary/AdditionExpression.java | 12 - .../plan/expression/binary/BinaryExpression.java | 74 +-- .../plan/expression/binary/DivisionExpression.java | 12 - .../plan/expression/binary/EqualToExpression.java | 11 - .../expression/binary/GreaterEqualExpression.java | 12 - .../expression/binary/GreaterThanExpression.java | 12 - .../expression/binary/LessEqualExpression.java | 12 - .../plan/expression/binary/LessThanExpression.java | 12 - .../plan/expression/binary/LogicAndExpression.java | 11 - .../plan/expression/binary/LogicOrExpression.java | 11 - .../plan/expression/binary/ModuloExpression.java | 12 - .../binary/MultiplicationExpression.java | 12 - .../plan/expression/binary/NonEqualExpression.java | 12 - .../expression/binary/SubtractionExpression.java | 12 - .../mpp/plan/expression/leaf/ConstantOperand.java | 37 +- .../plan/expression/leaf/TimeSeriesOperand.java | 36 +- .../mpp/plan/expression/leaf/TimestampOperand.java | 35 +- .../plan/expression/multi/FunctionExpression.java | 93 +--- .../plan/expression/ternary/BetweenExpression.java | 18 - .../plan/expression/ternary/TernaryExpression.java | 91 +--- .../db/mpp/plan/expression/unary/InExpression.java | 9 - .../plan/expression/unary/IsNullExpression.java | 9 - .../mpp/plan/expression/unary/LikeExpression.java | 9 - .../plan/expression/unary/LogicNotExpression.java | 9 - .../plan/expression/unary/NegationExpression.java | 9 - .../plan/expression/unary/RegularExpression.java | 9 - .../mpp/plan/expression/unary/UnaryExpression.java | 61 +-- .../visitor/ColumnTransformerVisitor.java | 478 +++++++++++++++++++ .../plan/expression/visitor/ExpressionVisitor.java | 73 +++ .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 125 ++--- .../db/mpp/plan/planner/LogicalPlanVisitor.java | 9 - .../db/mpp/plan/planner/OperatorTreeGenerator.java | 52 ++- .../mpp/plan/planner/plan/node/PlanNodeType.java | 4 + .../planner/plan/node/write/DeleteDataNode.java | 6 +- .../plan/node/write/InsertMultiTabletsNode.java | 7 +- .../plan/planner/plan/node/write/InsertNode.java | 4 +- .../planner/plan/node/write/InsertRowNode.java | 20 +- .../planner/plan/node/write/InsertRowsNode.java | 7 +- .../plan/node/write/InsertRowsOfOneDeviceNode.java | 7 +- .../planner/plan/node/write/InsertTabletNode.java | 6 +- .../scheduler/AbstractFragInsStateTracker.java | 28 +- .../db/mpp/plan/scheduler/ClusterScheduler.java | 3 +- .../scheduler/FixedRateFragInsStateTracker.java | 14 +- .../mpp/plan/scheduler/SimpleQueryTerminator.java | 4 - .../db/mpp/plan/scheduler/StandaloneScheduler.java | 28 +- .../db/mpp/plan/statement/StatementVisitor.java | 10 +- .../db/mpp/plan/statement/sys/AuthorStatement.java | 20 +- .../MergeStatement.java} | 36 +- .../iotdb/db/protocol/influxdb/dto/IoTDBPoint.java | 40 +- .../function/aggregator/InfluxMeanFunction.java | 4 +- .../function/selector/InfluxFirstFunction.java | 5 +- .../function/selector/InfluxLastFunction.java | 5 +- .../influxdb/handler/AbstractQueryHandler.java | 511 +++++++++++++++++++++ .../protocol/influxdb/handler/NewQueryHandler.java | 200 ++++++++ .../db/protocol/influxdb/handler/QueryHandler.java | 468 +------------------ .../influxdb/meta/AbstractInfluxDBMetaManager.java | 114 +++++ .../influxdb/meta/InfluxDBMetaManager.java | 96 +--- .../influxdb/meta/NewInfluxDBMetaManager.java | 129 ++++++ .../db/protocol/influxdb/meta/TagInfoRecords.java | 28 +- .../influxdb/util/InfluxReqAndRespUtils.java | 55 +++ .../protocol/influxdb/util/QueryResultUtils.java | 153 ++++++ .../db/protocol/influxdb/util/StringUtils.java | 11 + .../handler/PhysicalPlanValidationHandler.java | 4 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 92 +++- .../iotdb/db/qp/logical/sys/AuthorOperator.java | 26 +- .../iotdb/db/qp/physical/sys/AuthorPlan.java | 62 +-- .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 101 ++-- .../iotdb/db/service/InfluxDBRPCService.java | 26 +- .../java/org/apache/iotdb/db/service/NewIoTDB.java | 3 - .../iotdb/db/service/RegionMigrateService.java | 25 +- .../recorder/CompactionMetricsRecorder.java | 2 +- .../handler/InfluxDBServiceThriftHandler.java | 10 +- .../impl/DataNodeInternalRPCServiceImpl.java | 26 +- .../thrift/impl/IInfluxDBServiceWithHandler.java | 11 +- .../service/thrift/impl/InfluxDBServiceImpl.java | 20 +- .../thrift/impl/NewInfluxDBServiceImpl.java | 128 ++++++ .../iotdb/db/wal/recover/WALNodeRecoverTask.java | 3 +- .../iotdb/db/wal/recover/WALRecoverWriter.java | 26 +- .../db/metadata/cache/DataNodeSchemaCacheTest.java | 11 +- ...emaTreeTest.java => ClusterSchemaTreeTest.java} | 28 +- .../db/mpp/execution/ConfigExecutionTest.java | 18 +- .../iotdb/db/mpp/execution/DataDriverTest.java | 6 +- .../execution/exchange/LocalSinkHandleTest.java | 6 +- .../db/mpp/execution/exchange/SinkHandleTest.java | 70 +-- .../db/mpp/execution/exchange/StubSinkHandle.java | 4 +- .../iotdb/db/mpp/execution/exchange/Utils.java | 11 +- .../operator/AlignedSeriesScanOperatorTest.java | 8 +- .../operator/DeviceMergeOperatorTest.java | 2 +- .../mpp/execution/operator/LimitOperatorTest.java | 6 +- .../execution/operator/MultiColumnMergerTest.java | 2 +- .../NonOverlappedMultiColumnMergerTest.java | 4 +- .../mpp/execution/operator/OffsetOperatorTest.java | 6 +- .../operator/RawDataAggregationOperatorTest.java | 6 +- .../execution/operator/SingleColumnMergerTest.java | 6 +- .../execution/operator/TimeJoinOperatorTest.java | 8 +- .../schema/SchemaFetchScanOperatorTest.java | 5 +- .../write/InsertMultiTabletsNodeSerdeTest.java | 2 +- .../plan/node/write/InsertRowNodeSerdeTest.java | 8 +- .../plan/node/write/InsertRowsNodeSerdeTest.java | 2 +- .../write/InsertRowsOfOneDeviceNodeSerdeTest.java | 3 +- .../plan/node/write/InsertTabletNodeSerdeTest.java | 6 +- .../plan/scheduler/StandaloneSchedulerTest.java | 5 - .../java/org/apache/iotdb/db/qp/PlannerTest.java | 13 +- .../iotdb/db/qp/physical/PhysicalPlanTest.java | 2 +- .../iotdb/db/wal/recover/WALRecoverWriterTest.java | 198 ++++++++ .../org/apache/iotdb/session/pool/SessionPool.java | 3 + thrift-commons/src/main/thrift/common.thrift | 14 +- .../src/main/thrift/confignode.thrift | 229 +++++++-- thrift/src/main/thrift/datanode.thrift | 10 +- .../file/metadata/statistics/Statistics.java | 10 + .../file/metadata/statistics/TimeStatistics.java | 8 + .../iotdb/tsfile/read/common/block/TsBlock.java | 4 + .../tsfile/read/common/block/TsBlockBuilder.java | 16 + .../block/column/BinaryArrayColumnEncoder.java | 26 +- .../read/common/block/column/BinaryColumn.java | 16 + .../read/common/block/column/BooleanColumn.java | 16 + .../block/column/ByteArrayColumnEncoder.java | 19 +- .../tsfile/read/common/block/column/Column.java | 38 ++ .../read/common/block/column/DoubleColumn.java | 16 + .../read/common/block/column/FloatColumn.java | 16 + .../block/column/Int32ArrayColumnEncoder.java | 37 +- .../block/column/Int64ArrayColumnEncoder.java | 64 ++- .../tsfile/read/common/block/column/IntColumn.java | 16 + .../read/common/block/column/LongColumn.java | 16 + .../block/column/RunLengthEncodedColumn.java | 58 +++ .../read/common/block/column/TimeColumn.java | 11 + .../tsfile/read/reader/page/AlignedPageReader.java | 98 +++- .../tsfile/read/reader/page/ValuePageReader.java | 99 ++++ .../tsfile/write/chunk/AlignedChunkWriterImpl.java | 63 +++ .../iotdb/tsfile/write/chunk/TimeChunkWriter.java | 10 +- .../iotdb/tsfile/write/chunk/ValueChunkWriter.java | 24 +- .../iotdb/tsfile/write/page/TimePageWriter.java | 6 +- .../iotdb/tsfile/write/page/ValuePageWriter.java | 72 ++- .../tsfile/common/block/TsBlockSerdeTest.java | 139 ++++-- 326 files changed, 9437 insertions(+), 3494 deletions(-)
