This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a change to branch
fixConcurrentModificationCausedByParallelDispatch-1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 2df09f68cca Fix concurrent modification of non-thread-safe data
structures caused by parallel dispatching
add 173aa30f7c6 [To dev/1.3] Pipe: Modify Sink Batch Metrics (#16018)
(#16064)
add 851ed27e233 [To dev/1.3] Pipe: Modify Sink batch event length related
metrics (#16066) (#16074)
add bc7492cadce [To dev/1.3] Pipe: Fixed the default param of single entry
of disruptor queue & Banned memory checks from some missing pipe ITs & Do not
check non-user pipes (#16078)
add 6de644ea4fe [To dev/1.3] Pipe: Fixed the issue that frequent logins on
the receiving end caused indicator leaks #16076 (#16085)
add 64b5f6cd429 [To dev/1.3] Pipe/Load: Implement multi-disk awareness of
multiple file systems during file copying and moving (#15356) (#16052)
add 9345d9ed297 [To dev/1.3] Pipe: Fixed the missing bans for pipe memory
management (#16088)
add 21a2e9d13cc [To dev/1.3] Pipe: Fixed TsFile resource leak issue
(#16090) (#16092)
add 6c275186107 Fix path attack when loading snapshot of IoTConsensus
(#16098)
add fc6913b7509 [To dev/1.3] Pipe: Fix the stuck state caused by unfair
lock in Sink start phase (#16100) (#16106)
add 1ca6002c290 [To dev/1.3] Pipe: Do not use the fork join pool in
TerminateEvent #16113 (#16114)
add b8d829cce3d [To dev/1.3] Pipe: Totally banned the receiver conversion
(#16086) (#16105)
add 01dcf32fbf0 [To dev/1.3] Pipe: Delete the heartbeat event count in
Remaining Count #16115 (#16116)
add 73721013ad6 [To dev/1.3] Pipe: CN adds logic to check if Pipe is out
of memory #16119 (#16120)
add b9c212489ba [To dev/1.3] Pipe: Fix and improve async tsfile transfer
error handling and logging (avoid client connection leak) (#16008) (#16125)
add bbb74b1ae9b [to dev/1.3] Transfer schema tree in batches & add memory
control for schema tree
add d37e9aa215e [To dev/1.3] Temporarily banned some PBTree IT to avoid CI
failure & Pipe: Fixed the client release problem in trackable handler (#16130)
add 92368e62a90 Pipe: Avoid unnecessary close-client in async client
(Follow up fix for #16008)
add 46b69a7a5ae [To dev/1.3] Pipe: Avoid unnecessary close-client in async
client (Follow up fix for #16008) (#16136)
add 31a407ac402 Skip TTL check when there is no TTL and no mods file in
the data region (#16110) (#16129)
add f3fa09f36c4 [To dev/1.3] Load: Reduce unnecessary MD5 calculations
(#16141) (#16151)
add b9b9119ebd5 [To dev/1.3] Pipe: Use fixed memory settings to configure
batch (#16134) (#16150)
add 436dd370dbb [To dev/1.3] Load: Fix LoadPieceNode reading aligned Chunk
causing OOM (#16152)
add f5f52a483eb [To dev/1.3] Pipe: Removed the useless ser-de in receiver
raw req & Improved the handling logic for rowCount and null value bitmaps in
insertTabletNode (#16133)
add 6f8efa92029 [To dev/1.3] Pipe: Changed the default periodical login
behavior to disabled (#16162)
add e102e339944 [To dev/1.3] Pipe: Optimized the commit queue to avoid OOM
problem (#16145) (#16157)
add 1a3f960da6b Replace the key in FileReaderManager to TsFileID
add e9039a7f44b [To dev/1.3] Pipe: Delete unnecessary metrics (#16165)
(#16168)
add 3c5741cb85f Merge branch 'dev/1.3' into
fixConcurrentModificationCausedByParallelDispatch-1.3
No new revisions were added by this update.
Summary of changes:
.../it/env/cluster/config/MppCommonConfig.java | 12 ++
.../env/cluster/config/MppSharedCommonConfig.java | 13 ++
.../it/env/remote/config/RemoteCommonConfig.java | 10 ++
.../org/apache/iotdb/itbase/env/CommonConfig.java | 4 +
.../iotdb/db/it/orderBy/IoTDBOrderBy2IT.java | 4 +
.../db/it/orderBy/IoTDBOrderByForDebugIT.java | 4 +
.../apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java | 4 +
.../db/it/schema/IoTDBCreateTimeseriesIT.java | 19 +-
.../pipe/it/autocreate/AbstractPipeDualAutoIT.java | 2 +
.../it/autocreate/IoTDBPipeAutoConflictIT.java | 2 +
.../pipe/it/autocreate/IoTDBPipeClusterIT.java | 2 +
.../pipe/it/autocreate/IoTDBPipeIdempotentIT.java | 2 +
...eSinkParallelIT.java => IoTDBPipeMemoryIT.java} | 65 +++----
.../pipe/it/autocreate/IoTDBPipeProcessorIT.java | 2 +
.../pipe/it/autocreate/IoTDBPipeProtocolIT.java | 2 +
.../it/autocreate/IoTDBPipeSinkCompressionIT.java | 2 +
.../pipe/it/autocreate/IoTDBPipeSourceIT.java | 2 +
.../pipe/it/autocreate/IoTDBPipeWithLoadIT.java | 2 +
.../pipe/it/manual/AbstractPipeDualManualIT.java | 2 +
.../manual/IoTDBPipeTypeConversionISessionIT.java | 5 +-
.../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 5 +-
.../it/cluster/IoTDBSubscriptionRestartIT.java | 1 +
.../it/dual/AbstractSubscriptionDualIT.java | 12 +-
.../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 12 +-
.../it/dual/IoTDBSubscriptionTimePrecisionIT.java | 12 +-
.../it/dual/IoTDBSubscriptionTopicIT.java | 12 +-
.../it/local/AbstractSubscriptionLocalIT.java | 1 +
.../it/triple/AbstractSubscriptionTripleIT.java | 13 +-
.../apache/iotdb/tools/it/ExportTsFileTestIT.java | 1 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../iotdb/confignode/persistence/TTLInfo.java | 2 +
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 28 +++
.../consensus/iot/IoTConsensusServerImpl.java | 22 ++-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 6 +-
.../db/pipe/agent/runtime/PipeAgentLauncher.java | 2 -
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 146 +++++++++-------
.../agent/task/subtask/sink/PipeSinkSubtask.java | 40 ++++-
.../subtask/sink/PipeSinkSubtaskLifeCycle.java | 7 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 11 +-
.../event/common/terminate/PipeTerminateEvent.java | 42 ++++-
.../common/tsfile/PipeTsFileInsertionEvent.java | 13 +-
.../PipeDataNodeRemainingEventAndTimeOperator.java | 17 --
.../overview/PipeDataNodeSinglePipeMetrics.java | 44 +----
.../metric/sink/PipeDataRegionSinkMetrics.java | 145 ++++++++++------
.../protocol/thrift/IoTDBDataNodeReceiver.java | 47 ++---
...peStatementDataTypeConvertExecutionVisitor.java | 4 +-
.../evolvable/batch/PipeTabletEventBatch.java | 82 ++-------
.../evolvable/batch/PipeTabletEventPlainBatch.java | 16 +-
.../batch/PipeTabletEventTsFileBatch.java | 16 +-
.../batch/PipeTransferBatchReqBuilder.java | 61 ++++++-
.../request/PipeTransferTabletRawReq.java | 22 +--
.../thrift/async/IoTDBDataRegionAsyncSink.java | 68 ++++++--
.../handler/PipeTransferTrackableHandler.java | 45 ++++-
.../async/handler/PipeTransferTsFileHandler.java | 26 +--
.../thrift/sync/IoTDBDataRegionSyncSink.java | 36 ++++
.../PipeHistoricalDataRegionTsFileSource.java | 20 ++-
.../PipeRealtimeDataRegionHybridSource.java | 1 +
.../listener/PipeInsertionDataNodeListener.java | 4 +
.../impl/DataNodeInternalRPCServiceImpl.java | 21 ++-
.../db/queryengine/common/MPPQueryContext.java | 33 ++++
.../common/schematree/ClusterSchemaTree.java | 192 ++++++++++++++++-----
.../queryengine/common/schematree/ISchemaTree.java | 3 +-
.../common/schematree/node/SchemaEntityNode.java | 23 +++
.../common/schematree/node/SchemaInternalNode.java | 17 ++
.../schematree/node/SchemaMeasurementNode.java | 24 +++
.../common/schematree/node/SchemaNode.java | 6 +-
.../operator/schema/SchemaFetchScanOperator.java | 104 ++++++++---
.../queryengine/plan/analyze/AnalyzeVisitor.java | 2 +-
.../db/queryengine/plan/analyze/Analyzer.java | 14 +-
.../analyze/cache/schema/DataNodeTTLCache.java | 9 +
.../plan/analyze/load/LoadTsFileAnalyzer.java | 72 +-------
.../analyze/schema/ClusterSchemaFetchExecutor.java | 29 +++-
.../planner/plan/node/write/InsertTabletNode.java | 37 ++--
.../plan/statement/crud/InsertTabletStatement.java | 32 ++++
.../iotdb/db/schemaengine/template/Template.java | 15 +-
.../iotdb/db/storageengine/StorageEngine.java | 3 +-
.../db/storageengine/buffer/BloomFilterCache.java | 7 +-
.../iotdb/db/storageengine/buffer/ChunkCache.java | 3 +-
.../buffer/TimeSeriesMetadataCache.java | 5 +-
.../db/storageengine/dataregion/DataRegion.java | 93 +++++-----
.../execute/utils/MultiTsFileDeviceIterator.java | 3 +-
.../writer/ReadPointCrossCompactionWriter.java | 3 +-
.../dataregion/memtable/TsFileProcessor.java | 2 +
.../dataregion/read/control/FileReaderManager.java | 85 ++++-----
.../filescan/impl/ClosedFileScanHandleImpl.java | 8 +-
.../filescan/impl/DiskAlignedChunkHandleImpl.java | 4 +-
.../read/filescan/impl/DiskChunkHandleImpl.java | 7 +-
.../read/filescan/model/AbstractChunkOffset.java | 3 +-
.../read/filescan/model/AlignedChunkOffset.java | 4 +-
.../read/filescan/model/ChunkOffset.java | 5 +-
.../db/storageengine/load/LoadTsFileManager.java | 6 +-
.../load/active/ActiveLoadDirScanner.java | 1 +
.../storageengine/load/active/ActiveLoadUtil.java | 183 ++++++++++++++++++++
.../storageengine/load/disk/ILoadDiskSelector.java | 39 +++--
.../InheritSystemMultiDisksStrategySelector.java | 36 ++--
.../db/storageengine/load/disk/MinIOSelector.java | 65 ++++---
.../load/splitter/AlignedChunkData.java | 59 ++++---
.../splitter/BatchedAlignedValueChunkData.java | 24 +--
.../db/storageengine/load/splitter/ChunkData.java | 2 +-
.../broker/SubscriptionPrefetchingQueue.java | 12 +-
.../schema/SchemaFetchScanOperatorTest.java | 52 ++++--
.../storageengine/buffer/BloomFilterCacheTest.java | 6 +-
.../iotdb/db/storageengine/dataregion/TTLTest.java | 52 ++++++
.../compaction/CompactionWithMinTimestampTest.java | 4 +-
.../compaction/FastAlignedCrossCompactionTest.java | 4 +-
.../FastCrossCompactionPerformerTest.java | 4 +-
.../FastInnerCompactionPerformerTest.java | 4 +-
.../FastNonAlignedCrossCompactionTest.java | 4 +-
.../compaction/ReadChunkInnerCompactionTest.java | 4 +-
.../ReadPointAlignedCrossCompactionTest.java | 4 +-
.../ReadPointCompactionPerformerTest.java | 4 +-
.../ReadPointNonAlignedCrossCompactionTest.java | 4 +-
.../cross/CrossSpaceCompactionSelectorTest.java | 4 +-
.../inner/InnerSpaceCompactionSelectorTest.java | 4 +-
.../SizeTieredCompactionRecoverTest.java | 2 +-
.../utils/MultiTsFileDeviceIteratorTest.java | 4 +-
.../read/control/FileReaderManagerTest.java | 4 +-
.../iotdb/commons/concurrent/ThreadName.java | 1 +
.../apache/iotdb/commons/conf/CommonConfig.java | 19 +-
.../commons/pipe/agent/task/PipeTaskAgent.java | 3 +
.../agent/task/progress/PipeEventCommitter.java | 66 +------
.../task/progress/interval/PipeCommitInterval.java | 79 +++++++++
.../task/progress/interval/PipeCommitQueue.java | 60 +++++++
.../task/subtask/PipeAbstractSinkSubtask.java | 23 +--
.../agent/task/subtask/PipeReportableSubtask.java | 54 ++++--
.../iotdb/commons/pipe/config/PipeConfig.java | 5 +
.../iotdb/commons/pipe/config/PipeDescriptor.java | 5 +
.../datastructure/interval/Interval.java} | 30 ++--
.../datastructure/interval/IntervalManager.java | 74 ++++++++
.../iotdb/commons/pipe/event/EnrichedEvent.java | 39 ++---
.../commons/pipe/sink/protocol/IoTDBSink.java | 21 +++
.../apache/iotdb/commons/schema/ttl/TTLCache.java | 56 +++++-
.../org/apache/iotdb/commons/utils/FileUtils.java | 173 +++++++++----------
.../pipe/datastructure/PipeCommitQueueTest.java | 113 ++++++++++++
pom.xml | 2 +-
135 files changed, 2418 insertions(+), 1067 deletions(-)
copy
integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/{IoTDBPipeSinkParallelIT.java
=> IoTDBPipeMemoryIT.java} (65%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadUtil.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/interval/PipeCommitInterval.java
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/interval/PipeCommitQueue.java
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/{path/fa/dfa/transition/DFAPreciseTransition.java
=> pipe/datastructure/interval/Interval.java} (62%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/interval/IntervalManager.java
create mode 100644
iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/PipeCommitQueueTest.java
