This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a change to branch feature/linux-memory
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from caee5a46862 modify according to review
add 381948b52f5 Bump org.apache.zeppelin:zeppelin-interpreter from 0.10.1
to 0.11.1 (#12324)
add 3f3b4f78185 feature add health_check (#12325)
add f52d752bf84 Region migration related work (#12293)
add 199342da8df Pipe: clear event reference count instead of decreasing
when closing to improve idempotence & add creation time for committer key to
avoid outdated commit & detect re-increasing reference count to avoid NPE
(#12315)
add 9c9a70e14cd Implement DataNodeSchemaLockManager for distributed
concurrency control
add 44b263e2a93 Fix bug change the hasnext impl of session data set
add 214c49a2108 new iot_consensus_throttle_threshold_in_byte (#12333)
add f4804a41440 Thread-safety load balance infrastructure (#12330)
add 13560ee1eab [RatisConsensus] Bump Ratis snapshot version to 8eafc4f
(#12335)
add f91d3508530 Merge branch 'master' into feature/linux-memory
No new revisions were added by this update.
Summary of changes:
.../iotdb/it/env/cluster/env/AbstractEnv.java | 29 +-
.../it/env/cluster/node/AbstractNodeWrapper.java | 5 +
.../apache/iotdb/itbase/env/BaseNodeWrapper.java | 2 +
...IoTDBRegionMigrateDataNodeCrashITFramework.java | 22 +-
.../IoTDBRegionMigrateReliabilityITFramework.java | 412 +++++++++----
.../it/regionmigration/KillPointContext.java | 32 +-
.../pass/IoTDBRegionMigrateClusterCrashIT.java | 68 +++
.../pass/IoTDBRegionMigrateConfigNodeCrashIT.java | 92 ++-
.../pass/IoTDBRegionMigrateNormalIT.java | 8 +-
.../pass/IoTDBRegionMigrateOtherIT.java | 9 +-
...ateCoordinatorCrashWhenRemoveRemotePeerIT.java} | 24 +-
.../IoTDBRegionMigrateDataNodeCrashIT.java | 59 +-
...MigrateOriginalCrashWhenDeleteLocalPeerIT.java} | 19 +-
...igrateOriginalCrashWhenRemoveRemotePeerIT.java} | 19 +-
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 1 +
.../org/apache/iotdb/isession/SessionDataSet.java | 6 +-
.../apache/iotdb/rpc/TElasticFramedTransport.java | 3 +-
.../client/async/AsyncDataNodeClientPool.java | 10 +-
.../client/async/handlers/AsyncClientHandler.java | 1 +
.../heartbeat/ConfigNodeHeartbeatHandler.java | 22 +-
.../heartbeat/DataNodeHeartbeatHandler.java | 73 ++-
...CHandler.java => TransferLeaderRPCHandler.java} | 20 +-
.../client/sync/SyncDataNodeClientPool.java | 9 +-
.../iotdb/confignode/manager/ConfigManager.java | 33 +-
.../iotdb/confignode/manager/ProcedureManager.java | 9 +-
.../iotdb/confignode/manager/load/LoadManager.java | 139 ++---
.../manager/load/balancer/RouteBalancer.java | 416 ++++++++-----
.../router/priority/GreedyPriorityBalancer.java | 4 +-
.../router/priority/LeaderPriorityBalancer.java | 4 +-
.../load/cache/AbstractHeartbeatSample.java} | 33 +-
.../manager/load/cache/AbstractLoadCache.java | 86 +++
.../manager/load/cache/AbstractStatistics.java | 49 ++
.../confignode/manager/load/cache/LoadCache.java | 284 ++++-----
.../load/cache/consensus/ConsensusGroupCache.java | 61 ++
.../consensus/ConsensusGroupHeartbeatSample.java | 27 +-
.../cache/consensus/ConsensusGroupStatistics.java} | 34 +-
.../manager/load/cache/node/BaseNodeCache.java | 106 +---
.../load/cache/node/ConfigNodeHeartbeatCache.java | 39 +-
.../load/cache/node/DataNodeHeartbeatCache.java | 37 +-
.../load/cache/node/NodeHeartbeatSample.java | 70 +--
.../manager/load/cache/node/NodeStatistics.java | 94 +--
.../manager/load/cache/region/RegionCache.java | 56 +-
.../load/cache/region/RegionGroupCache.java | 96 +--
.../load/cache/region/RegionGroupStatistics.java | 80 +--
.../load/cache/region/RegionHeartbeatSample.java | 27 +-
.../load/cache/region/RegionStatistics.java | 38 +-
.../manager/load/cache/route/RegionRouteCache.java | 147 -----
.../manager/load/service/EventService.java | 272 +++++++++
.../manager/load/service/HeartbeatService.java | 26 +-
.../manager/load/service/StatisticsService.java | 237 +-------
...va => ConsensusGroupStatisticsChangeEvent.java} | 30 +-
.../load/subscriber/IClusterStatusSubscriber.java | 8 +-
...scriber.java => NodeStatisticsChangeEvent.java} | 25 +-
....java => RegionGroupStatisticsChangeEvent.java} | 24 +-
.../iotdb/confignode/manager/node/NodeManager.java | 13 +-
.../runtime/PipeLeaderChangeHandler.java | 41 +-
.../runtime/PipeRuntimeCoordinator.java | 23 +-
.../pipe/execution/PipeConfigNodeSubtask.java | 9 +-
.../iotdb/confignode/procedure/Procedure.java | 15 -
.../procedure/env/ConfigNodeProcedureEnv.java | 45 +-
.../procedure/env/RegionMaintainHandler.java | 112 ++--
.../procedure/impl/StateMachineProcedure.java | 18 +
.../impl/node/AddConfigNodeProcedure.java | 10 +-
.../impl/region/AddRegionPeerProcedure.java | 91 ++-
.../impl/region/CreateRegionGroupsProcedure.java | 78 +--
.../impl/region/RemoveRegionPeerProcedure.java | 23 +-
.../impl/schema/DeleteDatabaseProcedure.java | 5 +-
.../testonly/CreateManyDatabasesProcedure.java | 10 +-
.../router/priority/GreedyPriorityTest.java | 10 +-
.../priority/LeaderPriorityBalancerTest.java | 8 +-
...CacheTest.java => ConsensusGroupCacheTest.java} | 20 +-
.../manager/load/cache/NodeCacheTest.java | 37 +-
.../manager/load/cache/RegionGroupCacheTest.java | 116 ++--
.../persistence/node/NodeStatisticsTest.java | 66 ---
.../statistics/RegionGroupStatisticsTest.java | 76 ---
.../partition/statistics/RegionStatisticsTest.java | 66 ---
iotdb-core/consensus/pom.xml | 4 +
.../org/apache/iotdb/consensus/IConsensus.java | 27 +
.../consensus/exception/ConsensusException.java | 4 +
.../apache/iotdb/consensus/iot/IoTConsensus.java | 98 +++-
.../consensus/iot/IoTConsensusServerImpl.java | 130 +++--
.../service/IoTConsensusRPCServiceProcessor.java | 8 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 49 +-
.../iotdb/consensus/simple/SimpleConsensus.java | 16 +
.../apache/iotdb/consensus/iot/ReplicateTest.java | 21 +
.../apache/iotdb/consensus/iot/StabilityTest.java | 12 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 19 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 44 +-
.../dataregion/DataRegionStateMachine.java | 15 +-
.../builder/PipeTransferBatchReqBuilder.java | 37 +-
.../async/IoTDBDataRegionAsyncConnector.java | 26 +
.../PipeTransferTabletBatchEventHandler.java | 12 +-
.../PipeTransferTabletInsertionEventHandler.java | 5 -
.../PipeTransferTsFileInsertionEventHandler.java | 2 -
.../db/pipe/resource/memory/PipeMemoryBlock.java | 7 +-
.../pipe/task/connection/PipeEventCollector.java | 8 +-
.../db/pipe/task/stage/PipeTaskConnectorStage.java | 3 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 2 +-
.../subtask/connector/PipeConnectorSubtask.java | 6 +-
.../connector/PipeConnectorSubtaskManager.java | 10 +-
.../subtask/processor/PipeProcessorSubtask.java | 5 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 67 ++-
.../db/queryengine/common/MPPQueryContext.java | 17 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 13 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 35 +-
.../analyze/cache/schema/DataNodeSchemaCache.java | 22 -
.../analyze/lock/DataNodeSchemaLockManager.java | 59 ++
.../plan/analyze/lock/SchemaLockType.java | 51 ++
.../plan/analyze/schema/ClusterSchemaFetcher.java | 14 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 69 +--
.../iotdb/db/service/RegionMigrateService.java | 7 +-
.../dataregion/snapshot/SnapshotLoader.java | 6 -
.../task/stage/SubscriptionTaskConnectorStage.java | 2 +-
.../SubscriptionConnectorSubtaskManager.java | 10 +-
.../metrics/metricsets/system/SystemMetrics.java | 49 +-
.../resources/conf/iotdb-common.properties | 3 +-
.../src/assembly/resources/sbin/health_check.bat | 641 +++++++++++++++++++++
.../src/assembly/resources/sbin/health_check.sh | 388 +++++++++++++
.../iotdb/commons/concurrent/ThreadName.java | 1 +
.../iotdb/commons/pipe/event/EnrichedEvent.java | 24 +-
.../pipe/progress/PipeEventCommitManager.java | 26 +-
.../commons/pipe/progress/PipeEventCommitter.java | 15 +-
.../task/subtask/PipeAbstractConnectorSubtask.java | 2 +-
.../pipe/task/subtask/PipeReportableSubtask.java | 2 +-
.../commons/pipe/task/subtask/PipeSubtask.java | 18 +-
... => IoTConsensusDeleteLocalPeerKillPoints.java} | 5 +-
...a => IoTConsensusInactivatePeerKillPoints.java} | 5 +-
...TConsensusRemovePeerCoordinatorKillPoints.java} | 2 +-
...{NeverTriggeredKillPoint.java => KillNode.java} | 8 +-
.../src/main/thrift/iotconsensus.thrift | 1 +
.../src/main/thrift/datanode.thrift | 8 +-
pom.xml | 4 +-
132 files changed, 4145 insertions(+), 2427 deletions(-)
copy
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/TestOnlyPlan.java
=>
integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateDataNodeCrashITFramework.java
(66%)
copy
iotdb-connector/grafana-connector/src/main/java/org/apache/iotdb/web/grafana/bean/TimeValues.java
=>
integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/KillPointContext.java
(61%)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateClusterCrashIT.java
rename
integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/{notpass/datanodecrash/CoordinatorRemoveRemotePeerCrashIT.java
=>
pass/datanodecrash/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerIT.java}
(64%)
rename
integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/{notpass
=> pass}/datanodecrash/IoTDBRegionMigrateDataNodeCrashIT.java (58%)
copy
integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/{IoTDBRegionMigrateNormalIT.java
=> datanodecrash/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT.java}
(65%)
copy
integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/{IoTDBRegionMigrateNormalIT.java
=> datanodecrash/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerIT.java}
(64%)
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/{AsyncTSStatusRPCHandler.java
=> TransferLeaderRPCHandler.java} (80%)
rename
iotdb-core/confignode/src/{test/java/org/apache/iotdb/confignode/manager/load/cache/RegionRouteCacheTest.java
=>
main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractHeartbeatSample.java}
(51%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractLoadCache.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractStatistics.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupCache.java
copy
iotdb-connector/spark-tsfile/src/main/java/org/apache/iotdb/spark/tsfile/qp/common/Operator.java
=>
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupHeartbeatSample.java
(60%)
mode change 100755 => 100644
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/common/NodeRef.java
=>
confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupStatistics.java}
(54%)
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/route/RegionRouteCache.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/service/EventService.java
rename
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/subscriber/{RouteChangeEvent.java
=> ConsensusGroupStatisticsChangeEvent.java} (52%)
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/subscriber/{IClusterStatusSubscriber.java
=> NodeStatisticsChangeEvent.java} (54%)
rename
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/subscriber/{StatisticsChangeEvent.java
=> RegionGroupStatisticsChangeEvent.java} (65%)
rename
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/{LoadCacheTest.java
=> ConsensusGroupCacheTest.java} (58%)
delete mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/node/NodeStatisticsTest.java
delete mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/partition/statistics/RegionGroupStatisticsTest.java
delete mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/partition/statistics/RegionStatisticsTest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/DataNodeSchemaLockManager.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/SchemaLockType.java
create mode 100644
iotdb-core/node-commons/src/assembly/resources/sbin/health_check.bat
create mode 100644
iotdb-core/node-commons/src/assembly/resources/sbin/health_check.sh
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/KillPoint/{NeverTriggeredKillPoint.java
=> IoTConsensusDeleteLocalPeerKillPoints.java} (91%)
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/KillPoint/{NeverTriggeredKillPoint.java
=> IoTConsensusInactivatePeerKillPoints.java} (90%)
rename
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/KillPoint/{IoTConsensusRemovePeerKillPoints.java
=> IoTConsensusRemovePeerCoordinatorKillPoints.java} (94%)
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/KillPoint/{NeverTriggeredKillPoint.java
=> KillNode.java} (87%)
