This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a change to branch wal_compression
in repository https://gitbox.apache.org/repos/asf/iotdb.git
omit 747b7dc99d9 Merge remote-tracking branch 'origin/master' into
wal_compression_back
omit a460acb28f0 Hot parameters recover (#12814)
omit a7e0aa4f462 Merge remote-tracking branch 'origin/master' into
wal_compression_back
omit 94f074f61e2 fix bug
omit 8008fff7ab6 enhance test
omit 16bbc224679 remove direct buffer
omit 1b992f29004 Pipe: Fixed some problems cause by reusing connectors
after exceptions occurred in connector stage (#12685)
omit 85dfa900e38 Pipe: Fix PipeTerminateEvent cannot report and mark the
pipe as completed & events with no generated events in pipe processor subtask
may not be reported (#12817)
omit c4828465214 Pipe: Fixed the bug that non-forwarding flag does not take
effect on insertRowsNode / insertRowsOfOneDevicesNode (#12824)
omit 2e1ebf46e93 Pipe: Intoduce TsFileInsertionScanDataContainer to read
data from tsfile sequentially to improve pattern parse performance when filter
rate is high (#12781)
omit 9b03eb673e8 PipeConsensus: enhance receiver when the leader reboots
(#12815)
omit 4b805661eab bump ratis version to 3.1.0 (#12807)
omit 3d1d28141b8 Pipe: Enable 'sink.format'='tablet' to force tsfiles to be
parsed into tablets (#12809)
omit 32bfeaf60cf Pipe/Load: Fix loaded files may be compacted before
listening (#12816)
omit 282147773bd Pipe: Reduced the logs in ConfigNode caused by pipeMeta
sync and heartbeat report (#12813)
omit 28c970b1107 [IOTDB-6344] Display null in cli
omit 389ae8002e2 Fix region migration IT & Add daily IT (#12804)
omit c1df2bf8a65 Pipe: Avoid writing out-of-order data in tsfile when
`'sink.format' = 'file'` (#12810)
omit 691e1159fa1 Allow using python 3.6 and enable CI (#12792)
omit 7106bca5020 Modify ttl (#12811)
omit 3fa0320d81d Pipe/PipeConsensus: support progressIndex for
InsertRowsNode (#12806)
omit ada45de7d37 use template file start 1c1d (#12808)
omit 5c23a355579 Add flush cause metric (#12803)
omit 7f34bc03129 Specify error msg when DriverTask is aborted by
MemoryNotEnoughException
omit 86aa71177f1 Pipe: Added collect invocation count in the caculation of
data region extractor events for data node remaining time (#12799)
omit 0afc1bea507 Pipe: Fix remaining time metrics is not dropped after
dropPipe(pipeName) & alter pipe does not take effect in non-data regions
(#12795)
omit 1020258bdf8 PipeConsensus: fix concurrent problems regarding to
initiate receiver and progressIndex & optimize receiver logs & fix npe (#12790)
omit 4912d902a8b [IOTDB-6343] Fix the device path construction bug in
visitSingleDeviceViewNode of AggragationPushDown
omit 79ea1ef23d2 Pipe: Enable `'sink.format'='tsfile'` to use tsFile as
tablet event batch (#12737)
omit 07792b11254 Fix WAL resource leak when using IoT series consensus
#12796
omit bc4cf473b77 Try to fix set cluster name integration test (#12794)
omit a3f996cbbc9 Fix iotdb-system.properties cluster_name comments #12791
omit cbdade41d5a Use milliseconds to express TTL (#12757)
omit a96349e298c Enhance leader quantity metric for load scenario (#12785)
omit 0b8e31456a4 Load: change the metric type of LoadDiskIO metric from
RATE to COUNTER (#12787)
omit b1e9d96f49e bind cluster name to metrics (#12786)
omit 7ee30fb631a Classification properties (#12764)
omit bce9631e108 Add memory control for MergeReader
omit 3dd6c8c3213 update tsfile version
omit 4900407d8b1 load-tsfile: Add retry logic for load memory controls
(#12782)
omit 500d4a911dd remove wal files when drop database (#12778)
omit 9e0b62181d0 Support editing cluster name (#12776)
omit e58d427f2f6 Load: Avoid potential NPE in warn log when
LoadRuntimeOutOfMemoryException occurs & load-tsfile: Polish logs (#12777)
omit 9c3d3ea8036 Update tsfile version (#12774)
omit fe52d5de3e6 Load: Add some load metrics of time cost, write point and
disk throughput (#12735)
omit df68083d95a Pipe: Avoid event being continuously rate-limited even its
pipe is dropped (#12753)
omit 31cd61f771d Pipe Consensus: Fix events in connector are not in order
because of parameter `'realtime-first' = 'true'` (#12773)
omit 60b12d87351 SystemPropertiesHandler close reader and writer #12772
omit 881816265f6 Synchronize takeSnapshotAsync on RaftGroupID (#12767)
omit 48028a15f3f Pipe: Adjusted some loggers of metrics to avoid
unnecessary warns & Include "lastEvent" into pipe's event count metrics &
Removed the "userConflict" judgment to data sync failure caused by
METADATA_ERROR (#12758)
omit f196cf20900 Pipe: Filter devices by pattern before reading device
metadata from TsFile (#12765)
omit 4e50e847378 SystemPropertiesHandler use Files::move instead of
File::renameTo (#12770)
omit 36dd4890577 Add clean logic for FragmentInstance in case that callback
is not added. (#12768)
omit 6cbdf31bf6f Subscription: support topic loose range for path and time
(#12760)
omit f22e3791694 Fix overlap between tsfile is not correctly marked (#12748)
omit 612ecd4f75e Refactor node registration check (#12766)
omit 3d436b93802 [IOTDB-6342] Clear
FragmentInstanceManager.dataNodeQueryContextMap while encountering
MemoryNotEnough Exception (#12762)
omit b18412d62d5 [IOTDB-6340] Clear Coordinator.queryExecutionMap while
encountering MemoryNotEnough Exception
omit 5d552729ae8 Optimize node startup process & Normalize
system.properties management (#12752)
add b1309ce6ca4 Optimize node startup process & Normalize
system.properties management (#12752)
add 5088a4c6796 [IOTDB-6340] Clear Coordinator.queryExecutionMap while
encountering MemoryNotEnough Exception
add b82800b4464 [IOTDB-6342] Clear
FragmentInstanceManager.dataNodeQueryContextMap while encountering
MemoryNotEnough Exception (#12762)
add f1d8e8a2777 Refactor node registration check (#12766)
add 32f98c146dd Fix overlap between tsfile is not correctly marked (#12748)
add b96dd0ee438 Subscription: support topic loose range for path and time
(#12760)
add cf22dd7e3b9 Add clean logic for FragmentInstance in case that callback
is not added. (#12768)
add 0e03d697f51 SystemPropertiesHandler use Files::move instead of
File::renameTo (#12770)
add d81cb80b210 Pipe: Filter devices by pattern before reading device
metadata from TsFile (#12765)
add 6834c40cb98 Pipe: Adjusted some loggers of metrics to avoid
unnecessary warns & Include "lastEvent" into pipe's event count metrics &
Removed the "userConflict" judgment to data sync failure caused by
METADATA_ERROR (#12758)
add a4a457ce5e1 Synchronize takeSnapshotAsync on RaftGroupID (#12767)
add b6f9733f6d6 SystemPropertiesHandler close reader and writer #12772
add bd76aeaccac Pipe Consensus: Fix events in connector are not in order
because of parameter `'realtime-first' = 'true'` (#12773)
add beb4a9cf80b Pipe: Avoid event being continuously rate-limited even its
pipe is dropped (#12753)
add e06cb7180da Load: Add some load metrics of time cost, write point and
disk throughput (#12735)
add 68b3fd0ff35 Update tsfile version (#12774)
add e87c4084504 Load: Avoid potential NPE in warn log when
LoadRuntimeOutOfMemoryException occurs & load-tsfile: Polish logs (#12777)
add cef8d4e7f80 Support editing cluster name (#12776)
add 6879ffab28d remove wal files when drop database (#12778)
add de5ee16c0ff load-tsfile: Add retry logic for load memory controls
(#12782)
add b816f7a81d6 update tsfile version
add 2fbc789c9c8 Add memory control for MergeReader
add 88794504579 Classification properties (#12764)
add 095c881c46d bind cluster name to metrics (#12786)
add f6969721d65 remove direct buffer
add 5bfebdae13d Load: change the metric type of LoadDiskIO metric from
RATE to COUNTER (#12787)
add a7df9aac243 Enhance leader quantity metric for load scenario (#12785)
add a62b5c16bdd Use milliseconds to express TTL (#12757)
add b6341e0e335 Fix iotdb-system.properties cluster_name comments #12791
add 8e7a8442562 Try to fix set cluster name integration test (#12794)
add 08109f16a80 Fix WAL resource leak when using IoT series consensus
#12796
add 336553a9168 Pipe: Enable `'sink.format'='tsfile'` to use tsFile as
tablet event batch (#12737)
add 02087197a38 [IOTDB-6343] Fix the device path construction bug in
visitSingleDeviceViewNode of AggragationPushDown
add a9b946448c1 enhance test
add dcd5780f75c PipeConsensus: fix concurrent problems regarding to
initiate receiver and progressIndex & optimize receiver logs & fix npe (#12790)
add bbfcbb437f9 Pipe: Fix remaining time metrics is not dropped after
dropPipe(pipeName) & alter pipe does not take effect in non-data regions
(#12795)
add 4420e4505c0 Pipe: Added collect invocation count in the caculation of
data region extractor events for data node remaining time (#12799)
add b91bc23a752 Specify error msg when DriverTask is aborted by
MemoryNotEnoughException
add bd032f781eb Add flush cause metric (#12803)
add 08a237e431a use template file start 1c1d (#12808)
add 96c6bcb2a0c Pipe/PipeConsensus: support progressIndex for
InsertRowsNode (#12806)
add 89b10fe1ace Modify ttl (#12811)
add 47070d0046b Allow using python 3.6 and enable CI (#12792)
add 4cf96171810 fix bug
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (747b7dc99d9)
\
N -- N -- N refs/heads/wal_compression (4cf96171810)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/daily-it.yml | 50 -
integration-test/pom.xml | 14 -
.../org/apache/iotdb/itbase/category/DailyIT.java | 22 -
.../apache/iotdb/cli/it/StartClientScriptIT.java | 65 -
.../IoTDBRegionMigrateReliabilityITFramework.java | 55 +-
.../pass/IoTDBRegionMigrateClusterCrashIT.java | 3 -
.../pass/IoTDBRegionMigrateConfigNodeCrashIT.java | 3 -
.../pass/IoTDBRegionMigrateNormalIT.java | 3 -
...rateCoordinatorCrashWhenRemoveRemotePeerIT.java | 3 -
.../IoTDBRegionMigrateDataNodeCrashIT.java | 6 -
...nMigrateOriginalCrashWhenDeleteLocalPeerIT.java | 3 -
...MigrateOriginalCrashWhenRemoveRemotePeerIT.java | 3 -
.../pipe/it/autocreate/IoTDBPipeAutoDropIT.java | 6 -
.../pipe/it/autocreate/IoTDBPipeDataSinkIT.java | 11 +-
.../java/org/apache/iotdb/cli/AbstractCli.java | 27 +-
.../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 23 +-
.../confignode/conf/ConfigNodeDescriptor.java | 3 +
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 37 +-
.../runtime/heartbeat/PipeHeartbeat.java | 4 -
.../runtime/heartbeat/PipeHeartbeatParser.java | 39 +-
.../pipe/event/PipeConfigRegionSnapshotEvent.java | 9 +-
.../pipe/execution/PipeConfigNodeSubtask.java | 2 -
.../resource/PipeConfigNodeResourceManager.java | 51 -
.../PipeConfigNodeSnapshotResourceManager.java | 11 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 16 +-
.../procedure/env/RegionMaintainHandler.java | 17 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 288 ++-
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 10 +-
.../IoTDBDataNodeCacheLeaderClientManager.java | 8 +-
.../evolvable/batch/PipeTabletEventPlainBatch.java | 4 +-
.../batch/PipeTabletEventTsFileBatch.java | 159 +-
.../request/PipeTransferTabletRawReq.java | 133 +-
.../airgap/IoTDBDataRegionAirGapConnector.java | 3 +-
.../protocol/legacy/IoTDBLegacyPipeConnector.java | 3 +-
.../PipeConsensusTransferBatchReqBuilder.java | 4 +-
.../async/IoTDBDataRegionAsyncConnector.java | 10 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 3 +-
.../protocol/writeback/WriteBackConnector.java | 3 +-
.../pipe/connector/util/PipeTabletEventSorter.java | 202 ---
.../schema/PipeSchemaRegionSnapshotEvent.java | 4 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 6 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 4 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 24 +-
...iner.java => TsFileInsertionDataContainer.java} | 114 +-
...java => TsFileInsertionDataTabletIterator.java} | 22 +-
.../container/TsFileInsertionDataContainer.java | 78 -
.../TsFileInsertionDataContainerProvider.java | 123 --
.../scan/AlignedSinglePageWholeChunkReader.java | 170 --
.../container/scan/SinglePageWholeChunkReader.java | 116 --
.../scan/TsFileInsertionScanDataContainer.java | 388 ----
.../PipeHistoricalDataRegionTsFileExtractor.java | 14 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 6 +-
.../realtime/assigner/DisruptorQueue.java | 12 +-
.../iotdb/db/pipe/metric/PipeResourceMetrics.java | 14 +-
.../downsampling/PartialPathLastObjectCache.java | 4 +-
.../pipeconsensus/PipeConsensusReceiver.java | 11 +-
...sourceManager.java => PipeResourceManager.java} | 8 +-
.../db}/pipe/resource/log/PipeLogManager.java | 7 +-
.../iotdb/db}/pipe/resource/log/PipeLogStatus.java | 6 +-
.../db/pipe/resource/memory/PipeMemoryBlock.java | 18 +-
.../pipe/resource/tsfile/PipeTsFileResource.java | 97 +-
.../resource/tsfile/PipeTsFileResourceManager.java | 10 +-
.../pipe/resource/wal/PipeWALResourceManager.java | 34 +-
.../hardlink/PipeWALHardlinkResourceManager.java | 23 +-
.../selfhost/PipeWALSelfHostResourceManager.java | 8 +-
.../pipe/task/builder/PipeDataNodeTaskBuilder.java | 23 +-
.../pipe/task/connection/PipeEventCollector.java | 25 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 20 +-
.../subtask/connector/PipeConnectorSubtask.java | 60 +-
.../PipeRealtimePriorityBlockingQueue.java | 11 +-
.../subtask/processor/PipeProcessorSubtask.java | 23 +-
.../db/storageengine/dataregion/DataRegion.java | 60 +-
.../dataregion/wal/utils/WALEntryPosition.java | 1 +
.../dataregion/wal/utils/WALInsertNodeCache.java | 4 +-
.../event/SubscriptionEventBinaryCache.java | 8 +-
.../pipe/connector/PipeTabletEventSorterTest.java | 217 ---
.../event/TsFileInsertionDataContainerTest.java | 232 +--
.../PipeDataNodeSnapshotResourceManagerTest.java | 20 +-
.../resources/conf/iotdb-system.properties | 1859 +-------------------
.../conf/iotdb-system.properties.template | 18 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 9 -
.../iotdb/commons/conf/CommonDescriptor.java | 16 +-
.../iotdb/commons/conf/ConfigurationFileUtils.java | 53 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 5 -
.../commons/pipe/pattern/IoTDBPipePattern.java | 4 -
.../pipe/task/connection/BlockingPendingQueue.java | 22 +-
.../task/subtask/PipeAbstractConnectorSubtask.java | 43 +-
pom.xml | 2 +-
88 files changed, 780 insertions(+), 4592 deletions(-)
delete mode 100644 .github/workflows/daily-it.yml
delete mode 100644
integration-test/src/main/java/org/apache/iotdb/itbase/category/DailyIT.java
delete mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/resource/PipeConfigNodeResourceManager.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/PipeTabletEventSorter.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/{container/query/TsFileInsertionQueryDataContainer.java
=> TsFileInsertionDataContainer.java} (79%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/{container/query/TsFileInsertionQueryDataTabletIterator.java
=> TsFileInsertionDataTabletIterator.java} (89%)
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainer.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainerProvider.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/AlignedSinglePageWholeChunkReader.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/SinglePageWholeChunkReader.java
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/{PipeDataNodeResourceManager.java
=> PipeResourceManager.java} (93%)
rename iotdb-core/{node-commons/src/main/java/org/apache/iotdb/commons =>
datanode/src/main/java/org/apache/iotdb/db}/pipe/resource/log/PipeLogManager.java
(88%)
rename iotdb-core/{node-commons/src/main/java/org/apache/iotdb/commons =>
datanode/src/main/java/org/apache/iotdb/db}/pipe/resource/log/PipeLogStatus.java
(87%)
delete mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeTabletEventSorterTest.java
