This is an automated email from the ASF dual-hosted git repository.
HTHou pushed a change to branch codex/ipv6-endpoint-support
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 8f84beaa0b4 Support IPv6 endpoint URLs
add 079405cd9e4 Test IPv6 JDBC database parsing
add 4c1f833adf1 Pipe: order historical TsFiles by query priority (#18088)
add e123ada23ec Respect pre-set env vars and add cgroup memory detection
in env scripts (#17765)
add d02e6af593a A lease-based self-fencing framework to enable high
availability for table metadata operations
add 506f94519fb Extend region migration speed limit to transfers and
deletion (#18118)
add afdcf327e3e Update programming languages and release notes for version
2.0.10 (#18164)
add 449946d5268 Fix pipe event accounting and aligned tablet failure
reporting (#18150)
add 33a38b6194d Pipe: manage ConfigNode receiver temporary memory (#18144)
add c716365a30f Fix subscription consumer lifecycle state ordering (#18151)
add 5aad9323c71 Reduce pipe type conversion log frequency (#18161)
add 343edab9c00 Fix IT log directory test class names (#18131)
add 9112791a88c Fix: remove some procedure that should not be included in
lease frame (#18167)
add ec81de1f9f2 Reduce pipe load success log noise (#18106)
add 7ccdaa503e2 Pipe: use session context in legacy receiver loaders
(#18111)
add 4482e65468a Fix pipe drop stuck on blocking sink reconnect (#18145)
add e8061edf668 Optimize load logic (#18153)
add 350c6c7bcd2 Optimize pipe TsFile cleanup on drop (#18163)
add f4ebc944681 Merge remote-tracking branch 'origin/master' into
codex/ipv6-endpoint-support
No new revisions were added by this update.
Summary of changes:
RELEASE_NOTES.md | 38 +
.../org/apache/iotdb/it/env/MultiEnvFactory.java | 8 +-
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../iotdb/it/env/cluster/env/AbstractEnv.java | 15 +
.../iotdb/it/env/cluster/env/MultiClusterEnv.java | 9 +
.../it/env/remote/config/RemoteCommonConfig.java | 5 +
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 4 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../apache/iotdb/db/it/IoTDBLoadTsFileAuthIT.java | 285 +++++++
.../iotdb/db/it/IoTDBSetConfigurationIT.java | 24 +
.../apache/iotdb/it/framework/IoTDBTestRunner.java | 8 +
.../framework/IoTDBTestRunnerWithParameters.java | 15 +-
.../single/IoTDBLegacyPipeReceiverSecurityIT.java | 109 ++-
.../relational/it/schema/IoTDBTableDDLHAIT.java | 289 +++++++
.../test/java/org/apache/iotdb/jdbc/UtilsTest.java | 4 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../base/AbstractSubscriptionProviders.java | 6 +
.../base/AbstractSubscriptionPullConsumer.java | 10 +-
.../base/AbstractSubscriptionPushConsumer.java | 10 +-
.../base/SubscriptionConsumerLifecycleTest.java | 267 ++++++
.../iotdb/confignode/i18n/ConfigNodeMessages.java | 1 +
.../iotdb/confignode/i18n/ManagerMessages.java | 17 +
.../iotdb/confignode/i18n/ProcedureMessages.java | 337 +++++---
.../iotdb/confignode/i18n/ConfigNodeMessages.java | 416 ++++++----
.../iotdb/confignode/i18n/ManagerMessages.java | 17 +
.../iotdb/confignode/i18n/ProcedureMessages.java | 915 +++++++++++++--------
.../CnToCnInternalServiceAsyncRequestManager.java | 4 +-
.../client/async/CnToDnAsyncRequestType.java | 3 +-
.../CnToDnInternalServiceAsyncRequestManager.java | 13 +-
.../heartbeat/DataNodeHeartbeatHandler.java | 6 +
.../rpc/DataNodeAsyncRequestRPCHandler.java | 3 +-
.../handlers/rpc/DataNodeTSStatusRPCHandler.java | 13 +-
.../confignode/conf/ConfigNodeMemoryConfig.java | 4 +-
.../consensus/request/ConfigPhysicalPlan.java | 4 +
.../consensus/request/ConfigPhysicalPlanType.java | 1 +
.../request/read/table/FetchTablePlan.java | 10 +-
...lePlan.java => RollbackPreDeleteTablePlan.java} | 10 +-
.../statemachine/ConfigRegionStateMachine.java | 10 +
.../iotdb/confignode/manager/ClusterManager.java | 3 +-
.../iotdb/confignode/manager/ConfigManager.java | 43 +-
.../apache/iotdb/confignode/manager/IManager.java | 4 +-
.../iotdb/confignode/manager/ProcedureManager.java | 6 +
.../manager/lease/ClusterCachePropagator.java | 159 ++++
.../manager/lease/DataNodeContactTracker.java | 110 +++
.../manager/lease/MetadataBroadcastVerdict.java | 66 ++
.../iotdb/confignode/manager/node/NodeManager.java | 2 +-
.../manager/partition/PartitionManager.java | 26 +-
.../agent/runtime/PipeConfigNodeRuntimeAgent.java | 13 +-
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 47 +-
.../resource/PipeConfigNodeResourceManager.java | 17 +-
.../memory/PipeConfigNodeMemoryManager.java | 115 +++
.../manager/schema/ClusterSchemaManager.java | 65 +-
.../persistence/executor/ConfigPlanExecutor.java | 3 +
.../persistence/schema/ClusterSchemaInfo.java | 25 +-
.../confignode/persistence/schema/ConfigMTree.java | 48 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 97 +--
.../procedure/env/RemoveDataNodeHandler.java | 3 +
.../procedure/impl/cq/CreateCQProcedure.java | 2 +-
.../impl/schema/AlterLogicalViewProcedure.java | 33 +-
.../schema/AlterTimeSeriesDataTypeProcedure.java | 32 +-
.../impl/schema/DeactivateTemplateProcedure.java | 2 +-
.../impl/schema/DeleteLogicalViewProcedure.java | 32 +-
.../impl/schema/DeleteTimeSeriesProcedure.java | 35 +-
.../procedure/impl/schema/SchemaUtils.java | 110 ++-
.../impl/schema/SetTemplateProcedure.java | 2 +-
.../table/AbstractAlterOrDropTableProcedure.java | 36 +-
.../impl/schema/table/CreateTableProcedure.java | 36 +-
.../impl/schema/table/DeleteDevicesProcedure.java | 51 +-
.../schema/table/DropTableColumnProcedure.java | 66 +-
.../impl/schema/table/DropTableProcedure.java | 90 +-
.../impl/sync/AuthOperationProcedure.java | 2 +-
.../procedure/state/schema/DropTableState.java | 3 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 14 +-
.../manager/lease/ClusterCachePropagatorTest.java | 189 +++++
.../manager/lease/DataNodeContactTrackerTest.java | 73 ++
.../lease/MetadataBroadcastVerdictTest.java | 77 ++
.../apache/iotdb/consensus/iot/IoTConsensus.java | 5 +-
.../iot/snapshot/IoTConsensusRateLimiter.java | 22 +-
.../iotdb/consensus/pipe/IoTConsensusV2.java | 5 +-
.../consensus/ratis/RateLimitedGrpcFactory.java} | 29 +-
.../ratis/RateLimitedGrpcLogAppender.java | 76 ++
.../consensus/ratis/RateLimitedGrpcRpcType.java} | 18 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 2 +
.../iotdb/consensus/simple/SimpleConsensus.java | 5 +-
.../ratis/RateLimitedGrpcLogAppenderTest.java | 66 ++
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 71 ++
.../iotdb/db/i18n/DataNodeQueryMessages.java | 2 +
.../iotdb/db/i18n/DataNodeSchemaMessages.java | 87 +-
.../iotdb/db/i18n/StorageEngineMessages.java | 3 +
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 64 ++
.../iotdb/db/i18n/DataNodeQueryMessages.java | 2 +
.../iotdb/db/i18n/DataNodeSchemaMessages.java | 65 +-
.../iotdb/db/i18n/StorageEngineMessages.java | 3 +
.../iotdb/db/auth/ClusterAuthorityFetcher.java | 16 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 21 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 18 +
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 22 +
.../subtask/processor/PipeProcessorSubtask.java | 4 +-
.../agent/task/subtask/sink/PipeSinkSubtask.java | 198 ++++-
.../common/tablet/PipeRawTabletInsertionEvent.java | 13 +
.../common/tsfile/PipeTsFileInsertionEvent.java | 29 +-
.../legacy/IoTDBLegacyPipeReceiverAgent.java | 82 +-
.../protocol/legacy/loader/DeletionLoader.java | 25 +-
.../receiver/protocol/legacy/loader/ILoader.java | 3 +-
.../protocol/legacy/loader/TsFileLoader.java | 16 +-
.../protocol/thrift/IoTDBDataNodeReceiver.java | 26 +-
.../statement/PipeConvertedInsertRowStatement.java | 30 +-
.../PipeConvertedInsertTabletStatement.java | 14 +-
...leStatementDataTypeConvertExecutionVisitor.java | 16 +-
...eeStatementDataTypeConvertExecutionVisitor.java | 9 +-
...aNodeHardlinkOrCopiedFileDirStartupCleaner.java | 85 +-
.../pipe/resource/tsfile/PipeTsFileResource.java | 8 +-
.../resource/tsfile/PipeTsFileResourceManager.java | 39 +-
.../client/IoTDBDataNodeAsyncClientManager.java | 11 +-
.../iotconsensusv2/IoTConsensusV2AsyncSink.java | 16 +-
.../iotconsensusv2/IoTConsensusV2SyncSink.java | 32 +-
.../handler/IoTConsensusV2DeleteEventHandler.java | 2 +-
.../IoTConsensusV2TabletBatchEventHandler.java | 2 +-
.../IoTConsensusV2TabletInsertionEventHandler.java | 2 +-
.../IoTConsensusV2TsFileInsertionEventHandler.java | 2 +-
.../db/pipe/sink/protocol/opcua/OpcUaSink.java | 3 +-
.../protocol/opcua/client/IoTDBOpcUaClient.java | 2 +-
.../sink/protocol/opcua/server/OpcUaNameSpace.java | 2 +-
.../thrift/async/IoTDBDataRegionAsyncSink.java | 6 +-
.../PipeTransferTabletBatchEventHandler.java | 2 +-
.../PipeTransferTabletInsertionEventHandler.java | 2 +-
.../handler/PipeTransferTrackableHandler.java | 4 +-
.../async/handler/PipeTransferTsFileHandler.java | 8 +-
.../sink/protocol/writeback/WriteBackSink.java | 2 +-
.../source/dataregion/IoTDBDataRegionSource.java | 12 +
...istoricalDataRegionTsFileAndDeletionSource.java | 410 +++++++--
.../iotdb/db/protocol/client/ConfigNodeClient.java | 12 +-
.../DnToCnInternalServiceAsyncRequestManager.java | 4 +-
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 5 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 11 +-
.../analyze/cache/partition/PartitionCache.java | 12 +
.../plan/analyze/load/LoadTsFileAnalyzer.java | 70 +-
.../load/TreeSchemaAutoCreatorAndVerifier.java | 72 +-
.../config/executor/ClusterConfigTaskExecutor.java | 7 +-
.../config/executor/IConfigTaskExecutor.java | 4 +-
.../relational/metadata/TableMetadataImpl.java | 3 +-
.../metadata/fetcher/SchemaPredicateUtil.java | 3 +-
.../fetcher/cache/TableDeviceSchemaCache.java | 6 +-
.../cache/TreeDeviceSchemaCacheManager.java | 13 +-
.../scheduler/load/LoadTsFileDispatcherImpl.java | 75 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 27 +-
.../schemaengine/lease/MetadataLeaseManager.java | 312 +++++++
.../schemaregion/SchemaRegionUtils.java | 14 +
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 4 +-
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 4 +-
.../db/schemaengine/table/DataNodeTableCache.java | 343 ++++++--
.../iotdb/db/schemaengine/table/ITableCache.java | 20 +-
.../db/service/metrics/DataNodeMetricsHelper.java | 3 +
.../metrics/MetadataLeaseMetrics.java} | 43 +-
.../iotdb/db/storageengine/StorageEngine.java | 9 +-
.../db/storageengine/dataregion/DataRegion.java | 53 +-
.../execute/utils/MultiTsFileDeviceIterator.java | 11 +-
.../dataregion/memtable/TsFileProcessor.java | 32 +-
.../dataregion/wal/allocation/ElasticStrategy.java | 5 +-
.../wal/allocation/FirstCreateStrategy.java | 5 +-
.../load/active/ActiveLoadPathHelper.java | 50 +-
.../load/active/ActiveLoadTsFileLoader.java | 9 +
.../iotdb/db/storageengine/load/util/LoadUtil.java | 28 +-
.../db/auth/ClusterAuthorityFetcherLeaseTest.java | 103 +++
.../org/apache/iotdb/db/conf/PropertiesTest.java | 30 +
.../task/subtask/sink/PipeSinkSubtaskTest.java | 254 ++++++
.../pipe/event/PipeTabletInsertionEventTest.java | 59 ++
...ricalDataRegionTsFileAndDeletionSourceTest.java | 491 ++++++++++-
.../realtime/PipeRealtimeDataRegionSourceTest.java | 113 +++
.../analyze/cache/PartitionCacheLeaseTest.java | 129 +++
.../TreeDeviceSchemaCacheManagerLeaseTest.java | 119 +++
.../lease/MetadataLeaseManagerTest.java | 114 +++
.../schemaengine/lease/MetadataLeaseTestUtils.java | 77 ++
.../table/DataNodeTableCacheLeaseTest.java | 89 ++
.../schemaengine/table/DataNodeTableCacheTest.java | 10 +-
.../TsFileResourceProgressIndexTest.java | 42 +
.../dataregion/memtable/TsFileProcessorTest.java | 74 ++
.../load/active/ActiveLoadPathHelperTest.java | 146 ++++
.../apache/iotdb/db/utils/EnvironmentUtils.java | 2 +-
.../conf/iotdb-system.properties.template | 30 +-
.../client/request/AsyncRequestManager.java | 12 +-
.../iotdb/commons/concurrent/ThreadName.java | 6 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 15 +
.../iotdb/commons/conf/CommonDescriptor.java | 5 +
.../commons/consensus/index/ProgressIndexType.java | 8 +-
.../index/impl/TimePartitionProgressIndex.java | 292 +++++++
.../exception/MetadataLeaseFencedException.java} | 13 +-
.../commons/pipe/agent/task/PipeTaskAgent.java | 21 +-
.../connection/UnboundedBlockingPendingQueue.java | 4 +-
.../task/subtask/PipeAbstractSinkSubtask.java | 103 ++-
.../pipe/agent/task/subtask/PipeSubtask.java | 2 +-
.../pipe/config/constant/PipeSourceConstant.java | 5 +
.../commons/pipe/receiver/IoTDBFileReceiver.java | 10 +-
.../commons/pipe/receiver/IoTDBReceiverAgent.java | 4 +-
.../schema/table/NonCommittableTsTable.java | 5 +-
...mmittableTsTable.java => PreDeleteTsTable.java} | 15 +-
.../commons/schema/table/TableNodeStatus.java | 10 +-
.../apache/iotdb/commons/schema/table/TsTable.java | 27 +-
.../org/apache/iotdb/commons/utils/FileUtils.java | 46 +-
.../RegionMigrationFileRemoveRateLimiter.java} | 40 +-
.../commons/utils/RegionMigrationRateLimiter.java} | 28 +-
.../UnboundedBlockingPendingQueueTest.java | 73 ++
.../iotdb/commons/pipe/task/PipeMetaDeSerTest.java | 11 +
.../apache/iotdb/commons/utils/FileUtilsTest.java | 64 ++
iotdb-doap.rdf | 14 +-
.../src/main/thrift/confignode.thrift | 12 +-
scripts/conf/confignode-env.sh | 91 +-
scripts/conf/datanode-env.sh | 86 +-
scripts/conf/iotdb-common.sh | 117 ++-
scripts/conf/windows/confignode-env.bat | 5 +-
scripts/conf/windows/datanode-env.bat | 3 +-
212 files changed, 8621 insertions(+), 2039 deletions(-)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileAuthIT.java
create mode 100644
integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableDDLHAIT.java
create mode 100644
iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/{CommitCreateTablePlan.java
=> RollbackPreDeleteTablePlan.java} (74%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/lease/ClusterCachePropagator.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/lease/DataNodeContactTracker.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/lease/MetadataBroadcastVerdict.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/resource/memory/PipeConfigNodeMemoryManager.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/lease/ClusterCachePropagatorTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/lease/DataNodeContactTrackerTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/lease/MetadataBroadcastVerdictTest.java
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/view/CreateTableViewStatement.java
=>
consensus/src/main/java/org/apache/iotdb/consensus/ratis/RateLimitedGrpcFactory.java}
(59%)
create mode 100644
iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RateLimitedGrpcLogAppender.java
copy
iotdb-core/{calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/fill/identity/IdentityFill.java
=>
consensus/src/main/java/org/apache/iotdb/consensus/ratis/RateLimitedGrpcRpcType.java}
(67%)
create mode 100644
iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/RateLimitedGrpcLogAppenderTest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{subscription/metric/SubscriptionMetrics.java
=> service/metrics/MetadataLeaseMetrics.java} (51%)
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/ClusterAuthorityFetcherLeaseTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheLeaseTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManagerLeaseTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManagerTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseTestUtils.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPathHelperTest.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/consensus/index/impl/TimePartitionProgressIndex.java
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeException.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/exception/MetadataLeaseFencedException.java}
(70%)
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/{NonCommittableTsTable.java
=> PreDeleteTsTable.java} (71%)
copy
iotdb-core/{consensus/src/main/java/org/apache/iotdb/consensus/iot/snapshot/IoTConsensusRateLimiter.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/utils/RegionMigrationFileRemoveRateLimiter.java}
(53%)
copy
iotdb-core/{consensus/src/main/java/org/apache/iotdb/consensus/iot/snapshot/IoTConsensusRateLimiter.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/utils/RegionMigrationRateLimiter.java}
(62%)
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/connection/UnboundedBlockingPendingQueueTest.java
