This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a change to branch feature/subscription-column-filter
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from b7188f25d15 Relax consensus column filter alter tests
add 9dd1bb8dbe3 Improve metadata OOM diagnostics (#18039)
add 8d739903ede Fix StateMachineProcedure EOF state re-execution (#18099)
add 9eb39bee848 Fix false disk-full read-only during concurrent snapshot
directory deletion (#18114)
add 8940ba0e3f5 fix(jdbc): prevent BigDecimal DoS in
StringUtils.consistentToString (#17508)
add 0b48a35c2c2 Fix show configuration displaying raw value instead of
effective value (#18108)
add bd4945780bb Optimize pipe wide table batch append precheck (#18117)
add eec963793bd Fix aligned memtable flush after deleting measurements
(#18116)
add 7bfd4400c69 Support attribute filters in table deletion (#18079)
add 35cf94a780a Improve pipe runtime memory allocation (#18051)
add 71696f333af Add pipe receiver memory protection (#18090)
add bbd424f0013 Avoid parsing fully matched historical table TsFiles
(#18123)
add 1c20df4ea7c Persist writer meta out of lock (#18124)
add a6c654bde56 Fix RegionGroupCache activation race (#18128)
add 63b70cb9ea1 Move IoTConsensus multi-data-dir region migration ITs to
commit suite (#18081)
add 86624a137bb Improve pipe receiver session handling (#18084)
add e44cbbb0e97 Fix load configuration IT target DataNode (#18125)
add 82690873b64 Pipe: throttle TsFile parser retries under memory pressure
(#18141)
add b35390559a9 Fix config region pipe privilege trimming (#18142)
add 611b66aded0 Change
partition_table_recover_max_read_megabytes_per_second into
partition_table_recover_max_read_mb_per_sec (#18138)
add 012026f0358 Add DEVICE_METADATA_INFO_SWAP_THRESHOLD parameter to
read_tsfile table function (#18146)
add 797319b453c The new value cannot be less than the current maximum
quota. The quota uses the default value if unspecified in the ALTER DATABASE
statement (#18154)
add 7558aa7ef80 Fix metadata cache and pipe edge cases (#18147)
add e231bbe5e2a Fix pipe double-living sugar after isolation (#18119)
add f3e161218d6 Fix downgraded TsFileResource serialization (#18155)
add 4c1f833adf1 Pipe: order historical TsFiles by query priority (#18088)
add e123ada23ec Respect pre-set env vars and add cgroup memory detection
in env scripts (#17765)
add d02e6af593a A lease-based self-fencing framework to enable high
availability for table metadata operations
add 81533de49a2 Merge branch 'master' into
feature/subscription-column-filter
No new revisions were added by this update.
Summary of changes:
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../it/env/remote/config/RemoteCommonConfig.java | 5 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../it/database/IoTDBDatabaseRegionControlIT.java | 111 +++
.../commit/IoTDBMigrateMultiRegionForIoTV1IT.java | 66 ++
...TDBRegionMigrateWithDeletionMultiDataDirIT.java | 14 +-
...gionMigrateWithDeletionMultiDataDirTableIT.java | 14 +-
.../RegionMigrateFileAssertions.java | 2 +-
.../IoTDBRegionMigrateNormalITForIoTV2BatchIT.java | 34 +
.../manual/enhanced/IoTDBPipeDoubleLivingIT.java | 10 +-
.../IoTDBPipeReceiverAutoCreateDisabledIT.java | 37 +
.../relational/it/db/it/IoTDBDeletionTableIT.java | 565 ++++++++++++-
.../it/db/it/IoTDBLoadConfigurationTableIT.java | 110 +++
.../schema/IoTDBDatabaseMaxRegionGroupNumIT.java | 73 ++
.../relational/it/schema/IoTDBTableDDLHAIT.java | 289 +++++++
.../java/org/apache/iotdb/jdbc/StringUtils.java | 21 +-
.../org/apache/iotdb/jdbc/StringUtilsTest.java | 74 ++
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../iotdb/confignode/i18n/ConfigNodeMessages.java | 1 +
.../iotdb/confignode/i18n/ManagerMessages.java | 3 +
.../iotdb/confignode/i18n/ProcedureMessages.java | 339 +++++---
.../iotdb/confignode/i18n/ConfigNodeMessages.java | 416 ++++++----
.../iotdb/confignode/i18n/ManagerMessages.java | 3 +
.../iotdb/confignode/i18n/ProcedureMessages.java | 917 +++++++++++++--------
.../CnToCnInternalServiceAsyncRequestManager.java | 4 +-
.../client/async/CnToDnAsyncRequestType.java | 3 +-
.../CnToDnInternalServiceAsyncRequestManager.java | 13 +-
.../heartbeat/DataNodeHeartbeatHandler.java | 6 +
.../rpc/DataNodeAsyncRequestRPCHandler.java | 3 +-
.../handlers/rpc/DataNodeTSStatusRPCHandler.java | 13 +-
.../consensus/request/ConfigPhysicalPlan.java | 4 +
.../consensus/request/ConfigPhysicalPlanType.java | 1 +
.../request/read/table/FetchTablePlan.java | 10 +-
...lePlan.java => RollbackPreDeleteTablePlan.java} | 10 +-
.../statemachine/ConfigRegionStateMachine.java | 10 +
.../iotdb/confignode/manager/ConfigManager.java | 43 +-
.../apache/iotdb/confignode/manager/IManager.java | 4 +-
.../iotdb/confignode/manager/ProcedureManager.java | 6 +
.../manager/lease/ClusterCachePropagator.java | 159 ++++
.../manager/lease/DataNodeContactTracker.java | 110 +++
.../manager/lease/MetadataBroadcastVerdict.java | 66 ++
.../load/cache/region/RegionGroupCache.java | 8 +-
.../pipe/coordinator/task/PipeTaskCoordinator.java | 30 +-
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 56 +-
.../pipe/source/IoTDBConfigRegionSource.java | 21 +-
.../manager/schema/ClusterSchemaManager.java | 97 ++-
.../persistence/executor/ConfigPlanExecutor.java | 3 +
.../persistence/schema/ClusterSchemaInfo.java | 25 +-
.../confignode/persistence/schema/ConfigMTree.java | 48 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 97 +--
.../procedure/env/RemoveDataNodeHandler.java | 3 +
.../procedure/impl/StateMachineProcedure.java | 13 +-
.../impl/schema/AlterLogicalViewProcedure.java | 33 +-
.../schema/AlterTimeSeriesDataTypeProcedure.java | 32 +-
.../impl/schema/DeactivateTemplateProcedure.java | 6 +-
.../impl/schema/DeleteLogicalViewProcedure.java | 30 +-
.../impl/schema/DeleteTimeSeriesProcedure.java | 33 +-
.../procedure/impl/schema/SchemaUtils.java | 150 +++-
.../procedure/impl/schema/SetTTLProcedure.java | 64 +-
.../impl/schema/SetTemplateProcedure.java | 56 +-
.../impl/schema/UnsetTemplateProcedure.java | 46 +-
.../table/AbstractAlterOrDropTableProcedure.java | 36 +-
.../impl/schema/table/CreateTableProcedure.java | 36 +-
.../impl/schema/table/DeleteDevicesProcedure.java | 51 +-
.../schema/table/DropTableColumnProcedure.java | 66 +-
.../impl/schema/table/DropTableProcedure.java | 90 +-
.../procedure/state/schema/DropTableState.java | 3 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 14 +-
.../manager/lease/ClusterCachePropagatorTest.java | 189 +++++
.../manager/lease/DataNodeContactTrackerTest.java | 73 ++
.../lease/MetadataBroadcastVerdictTest.java | 77 ++
.../PipeConfigTreePrivilegeParseVisitorTest.java | 43 +
.../confignode/procedure/STMProcedureTest.java | 70 ++
.../pipe/AbstractOperatePipeProcedureV2Test.java | 3 +-
.../procedure/impl/schema/SetTTLProcedureTest.java | 29 +-
.../consensus/iot/IoTConsensusServerImpl.java | 13 +-
.../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 4 +-
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 18 +-
.../iotdb/db/i18n/DataNodeQueryMessages.java | 70 +-
.../iotdb/db/i18n/DataNodeSchemaMessages.java | 87 +-
.../iotdb/db/i18n/StorageEngineMessages.java | 2 +
.../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 2 +-
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 18 +-
.../iotdb/db/i18n/DataNodeQueryMessages.java | 58 +-
.../iotdb/db/i18n/DataNodeSchemaMessages.java | 65 +-
.../iotdb/db/i18n/StorageEngineMessages.java | 2 +
.../iotdb/db/auth/ClusterAuthorityFetcher.java | 16 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 44 +-
.../DirectBufferMemoryAllocationException.java} | 17 +-
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 455 ++--------
.../task/builder/PipeDataNodeTaskBuilder.java | 36 +
.../agent/task/connection/PipeEventCollector.java | 15 +-
.../agent/task/stage/PipeTaskProcessorStage.java | 5 +-
.../subtask/processor/PipeProcessorSubtask.java | 2 +
.../common/tablet/PipeRawTabletInsertionEvent.java | 13 +
.../common/tsfile/PipeTsFileInsertionEvent.java | 166 +++-
.../tsfile/parser/TsFileInsertionEventParser.java | 12 +-
.../query/TsFileInsertionEventQueryParser.java | 1 +
.../scan/TsFileInsertionEventScanParser.java | 14 +-
.../table/TsFileInsertionEventTableParser.java | 22 +-
.../processor/aggregate/AggregateProcessor.java | 3 +
.../downsampling/DownSamplingProcessor.java | 3 +
.../receiver/TwoStageAggregateReceiver.java | 10 +
.../exchange/sender/TwoStageAggregateSender.java | 75 +-
.../twostage/plugin/TwoStageCountProcessor.java | 9 +-
.../protocol/thrift/IoTDBDataNodeReceiver.java | 178 +++-
.../db/pipe/resource/memory/PipeMemoryManager.java | 95 ++-
.../evolvable/batch/PipeTabletEventBatch.java | 69 +-
.../evolvable/batch/PipeTabletEventPlainBatch.java | 61 +-
.../batch/PipeTabletEventTsFileBatch.java | 30 +-
.../protocol/airgap/IoTDBDataRegionAirGapSink.java | 18 +-
.../iotconsensusv2/IoTConsensusV2SyncSink.java | 20 +-
.../IoTConsensusV2TsFileInsertionEventHandler.java | 34 +-
.../IoTConsensusV2TransferBatchReqBuilder.java | 109 ++-
.../sink/protocol/legacy/IoTDBLegacyPipeSink.java | 17 +-
.../async/handler/PipeTransferTsFileHandler.java | 42 +-
.../thrift/sync/IoTDBDataRegionSyncSink.java | 136 +++
.../source/dataregion/IoTDBDataRegionSource.java | 12 +
...istoricalDataRegionTsFileAndDeletionSource.java | 441 ++++++++--
.../realtime/PipeRealtimeDataRegionSource.java | 17 +-
.../schemaregion/IoTDBSchemaRegionSource.java | 4 +-
.../PipePlanTablePrivilegeParseVisitor.java | 17 +
.../iotdb/db/protocol/client/ConfigNodeClient.java | 12 +-
.../DnToCnInternalServiceAsyncRequestManager.java | 4 +-
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 42 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 15 +-
.../db/queryengine/common/MPPQueryContext.java | 286 ++++++-
.../common/schematree/ClusterSchemaTree.java | 8 +
.../db/queryengine/plan/analyze/AnalyzeUtils.java | 436 ++++++++--
.../queryengine/plan/analyze/AnalyzeVisitor.java | 8 +
.../queryengine/plan/analyze/ExpressionUtils.java | 2 +
.../analyze/cache/partition/PartitionCache.java | 12 +
.../analyze/schema/ClusterSchemaFetchExecutor.java | 9 +
.../config/executor/ClusterConfigTaskExecutor.java | 32 +-
.../config/executor/IConfigTaskExecutor.java | 4 +-
.../relational/analyzer/StatementAnalyzer.java | 2 +-
.../read_tsfile/ExternalTsFileQueryResource.java | 9 +-
.../tvf/read_tsfile/ReadTsFileTableFunction.java | 60 +-
.../relational/metadata/TableMetadataImpl.java | 3 +-
.../fetcher/cache/TableDeviceSchemaCache.java | 6 +-
.../cache/TreeDeviceSchemaCacheManager.java | 13 +-
.../plan/relational/planner/RelationPlanner.java | 5 +-
.../plan/relational/sql/ast/DeleteDevice.java | 5 +-
.../schemaengine/lease/MetadataLeaseManager.java | 312 +++++++
.../db/schemaengine/table/DataNodeTableCache.java | 348 ++++++--
.../iotdb/db/schemaengine/table/ITableCache.java | 20 +-
.../db/service/metrics/DataNodeMetricsHelper.java | 3 +
.../metrics/MetadataLeaseMetrics.java} | 43 +-
.../iotdb/db/storageengine/StorageEngine.java | 18 +-
.../db/storageengine/dataregion/DataRegion.java | 26 +-
.../compaction/execute/utils/CompactionUtils.java | 2 +-
.../execute/utils/MultiTsFileDeviceIterator.java | 11 +-
.../memtable/AlignedWritableMemChunk.java | 92 ++-
.../dataregion/modification/DeletionPredicate.java | 46 +-
.../modification/TableDeletionEntry.java | 6 +-
.../{IDPredicate.java => TagPredicate.java} | 261 +++++-
.../dataregion/tsfile/TsFileResource.java | 41 +-
.../file/UnsealedTsFileRecoverPerformer.java | 2 +-
.../db/auth/ClusterAuthorityFetcherLeaseTest.java | 103 +++
.../iotdb/db/metadata/path/PatternTreeMapTest.java | 2 +-
.../pipe/agent/task/PipeDataNodeTaskAgentTest.java | 88 --
.../db/pipe/consensus/DeletionRecoverTest.java | 2 +-
.../db/pipe/consensus/DeletionResourceTest.java | 2 +-
.../pipe/event/PipeTabletInsertionEventTest.java | 59 ++
.../pipe/event/TsFileInsertionEventParserTest.java | 154 ++++
.../iotdb/db/pipe/sink/PipeReceiverTest.java | 70 +-
.../PipeTabletEventPlainBatchPerformanceTest.java | 137 +++
.../PipePlanTablePatternParseVisitorTest.java | 57 +-
.../db/pipe/source/PipeRealtimeExtractTest.java | 73 ++
...ricalDataRegionTsFileAndDeletionSourceTest.java | 491 ++++++++++-
.../db/queryengine/common/MPPQueryContextTest.java | 125 +++
.../queryengine/plan/analyze/AnalyzeUtilsTest.java | 6 +-
.../analyze/cache/PartitionCacheLeaseTest.java | 129 +++
.../node/write/RelationalDeleteDataNodeTest.java | 8 +-
.../ExternalTsFileQueryResourceTest.java | 7 +-
.../TreeDeviceSchemaCacheManagerLeaseTest.java | 119 +++
.../lease/MetadataLeaseManagerTest.java | 114 +++
.../schemaengine/lease/MetadataLeaseTestUtils.java | 77 ++
.../table/DataNodeTableCacheLeaseTest.java | 89 ++
.../schemaengine/table/DataNodeTableCacheTest.java | 28 +-
.../iotdb/db/storageengine/StorageEngineTest.java | 45 +
.../storageengine/dataregion/DataRegionTest.java | 2 +-
.../TsFileResourceProgressIndexTest.java | 42 +
.../tablemodel/CompactionWithAllNullRowsTest.java | 12 +-
.../modification/ModificationFileTest.java | 27 +-
.../modification/TableDeletionEntryTest.java | 48 +-
.../dataregion/tsfile/TsFileResourceTest.java | 19 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 2 +-
.../conf/iotdb-system.properties.template | 19 +-
.../apache/iotdb/commons/i18n/PipeMessages.java | 2 +
.../apache/iotdb/commons/i18n/UtilMessages.java | 2 +
.../apache/iotdb/commons/i18n/PipeMessages.java | 2 +
.../apache/iotdb/commons/i18n/UtilMessages.java | 2 +
.../client/request/AsyncRequestManager.java | 12 +-
.../iotdb/commons/concurrent/ThreadName.java | 6 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 15 +
.../iotdb/commons/conf/CommonDescriptor.java | 5 +
.../commons/consensus/index/ProgressIndexType.java | 8 +-
.../index/impl/TimePartitionProgressIndex.java | 292 +++++++
.../apache/iotdb/commons/disk/FolderManager.java | 30 +-
.../MinFolderOccupiedSpaceFirstStrategy.java | 3 +-
.../exception/MetadataLeaseFencedException.java} | 13 +-
.../pipe/config/constant/PipeSourceConstant.java | 46 ++
.../env/PipeTaskProcessorRuntimeEnvironment.java | 23 +
.../datastructure/visibility/VisibilityUtils.java | 9 +-
.../commons/pipe/receiver/IoTDBFileReceiver.java | 103 ++-
.../thrift/common/PipeTransferSliceReqHandler.java | 37 +
.../thrift/request/PipeTransferCompressedReq.java | 21 +
.../iotdb/commons/pipe/source/IoTDBSource.java | 18 +-
.../schema/table/NonCommittableTsTable.java | 5 +-
...mmittableTsTable.java => PreDeleteTsTable.java} | 15 +-
.../commons/schema/table/TableNodeStatus.java | 10 +-
.../apache/iotdb/commons/schema/table/TsTable.java | 27 +-
.../apache/iotdb/commons/utils/JVMCommonUtils.java | 30 +-
...FolderOccupiedSpaceFirstStrategyRealFsTest.java | 61 ++
.../visibility/VisibilityUtilsTest.java | 24 +
.../pipe/receiver/IoTDBFileReceiverTest.java | 184 ++++-
.../common/PipeTransferSliceReqBuilderTest.java | 52 ++
.../request/PipeTransferCompressedReqTest.java | 37 +
.../iotdb/commons/pipe/task/PipeMetaDeSerTest.java | 11 +
.../iotdb/commons/utils/JVMCommonUtilsTest.java | 48 ++
.../src/main/thrift/confignode.thrift | 12 +-
scripts/conf/confignode-env.sh | 91 +-
scripts/conf/datanode-env.sh | 86 +-
scripts/conf/iotdb-common.sh | 117 ++-
scripts/conf/windows/confignode-env.bat | 5 +-
scripts/conf/windows/datanode-env.bat | 3 +-
229 files changed, 10860 insertions(+), 2750 deletions(-)
rename
integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/{daily/iotv1
=> commit}/IoTDBRegionMigrateWithDeletionMultiDataDirIT.java (93%)
rename
integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/{daily/iotv1
=> commit}/IoTDBRegionMigrateWithDeletionMultiDataDirTableIT.java (93%)
rename
integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/{daily/iotv1
=> commit}/RegionMigrateFileAssertions.java (98%)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableDDLHAIT.java
create mode 100644
iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/StringUtilsTest.java
copy
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/{CommitCreateTablePlan.java
=> RollbackPreDeleteTablePlan.java} (74%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/lease/ClusterCachePropagator.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/lease/DataNodeContactTracker.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/lease/MetadataBroadcastVerdict.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/lease/ClusterCachePropagatorTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/lease/DataNodeContactTrackerTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/lease/MetadataBroadcastVerdictTest.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{storageengine/dataregion/wal/exception/BrokenWALFileException.java
=> exception/DirectBufferMemoryAllocationException.java} (63%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{subscription/metric/SubscriptionMetrics.java
=> service/metrics/MetadataLeaseMetrics.java} (51%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/{IDPredicate.java
=> TagPredicate.java} (58%)
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/ClusterAuthorityFetcherLeaseTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatchPerformanceTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/common/MPPQueryContextTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheLeaseTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManagerLeaseTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManagerTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseTestUtils.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/consensus/index/impl/TimePartitionProgressIndex.java
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeException.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/exception/MetadataLeaseFencedException.java}
(70%)
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/{NonCommittableTsTable.java
=> PreDeleteTsTable.java} (71%)
