[
https://issues.apache.org/jira/browse/IGNITE-25384?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Roman Puchkovskiy updated IGNITE-25384:
---------------------------------------
Description:
There seems to be a race between sending a partition Raft snapshot (in
OutgoingSnapshot) and node stop:
[2025-05-15T08:56:37,617][ERROR][%izdrt_tdr_20001%outgoing-snapshots-3][FailureManager]
Critical system error detected. Will be handled accordingly to configured
handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
[ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED,
SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=CRITICAL_ERROR]
org.apache.ignite.internal.failure.StackTraceCapturingException: Something went
wrong while handling a request
at
org.apache.ignite.internal.failure.FailureManager.process(FailureManager.java:161)
~[ignite-failure-handler-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.failure.FailureManager.process(FailureManager.java:138)
~[ignite-failure-handler-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshotsManager.respond(OutgoingSnapshotsManager.java:206)
~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshotsManager.lambda$handleMessage$2(OutgoingSnapshotsManager.java:185)
~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
at
java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
[?:?]
at
java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
[?:?]
at
java.base/java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:482)
[?:?]
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
[?:?]
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
[?:?]
at java.base/java.lang.Thread.run(Thread.java:833) [?:?]
Caused by: java.util.concurrent.CompletionException:
org.apache.ignite.internal.storage.StorageClosedException: IGN-CMN-65535
TraceId:a2515b4b-c1a3-47c5-a48c-4bbd2ebc3bd0 Storage is already closed:
[tableId=6, partitionId=0]
at
java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:315)
~[?:?]
at
java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:320)
~[?:?]
at
java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1770)
~[?:?]
... 3 more
Caused by: org.apache.ignite.internal.storage.StorageClosedException: Storage
is already closed: [tableId=6, partitionId=0]
at
org.apache.ignite.internal.storage.util.StorageUtils.throwExceptionDependingOnStorageState(StorageUtils.java:141)
~[ignite-storage-api-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.storage.util.StorageUtils.throwExceptionIfStorageNotInRunnableState(StorageUtils.java:63)
~[ignite-storage-api-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.storage.pagememory.mv.AbstractPageMemoryMvPartitionStorage.throwExceptionIfStorageNotInRunnableState(AbstractPageMemoryMvPartitionStorage.java:801)
~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.storage.pagememory.mv.ScanVersionsCursor.lambda$hasNext$0(ScanVersionsCursor.java:67)
~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.storage.pagememory.mv.AbstractPageMemoryMvPartitionStorage.busy(AbstractPageMemoryMvPartitionStorage.java:656)
~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.storage.pagememory.mv.ScanVersionsCursor.hasNext(ScanVersionsCursor.java:66)
~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
at java.base/java.util.Iterator.forEachRemaining(Iterator.java:132) ~[?:?]
at
java.base/java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1845)
~[?:?]
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
~[?:?]
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
~[?:?]
at
java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:921)
~[?:?]
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
~[?:?]
at
java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:682)
~[?:?]
at
org.apache.ignite.internal.table.distributed.raft.snapshot.PartitionMvStorageAccessImpl.lambda$getAllRowVersions$0(PartitionMvStorageAccessImpl.java:126)
~[ignite-table-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.storage.pagememory.mv.PersistentPageMemoryMvPartitionStorage.lambda$runConsistently$0(PersistentPageMemoryMvPartitionStorage.java:180)
~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.storage.pagememory.mv.AbstractPageMemoryMvPartitionStorage.busy(AbstractPageMemoryMvPartitionStorage.java:656)
~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.storage.pagememory.mv.PersistentPageMemoryMvPartitionStorage.runConsistently(PersistentPageMemoryMvPartitionStorage.java:170)
~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.table.distributed.raft.snapshot.PartitionMvStorageAccessImpl.getAllRowVersions(PartitionMvStorageAccessImpl.java:122)
~[ignite-table-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshot.rowEntry(OutgoingSnapshot.java:404)
~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshot.tryProcessRowFromPartition(OutgoingSnapshot.java:385)
~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshot.handleSnapshotMvDataRequest(OutgoingSnapshot.java:314)
~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshotsManager.handleSnapshotRequestMessage(OutgoingSnapshotsManager.java:194)
~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
at
org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshotsManager.lambda$handleMessage$1(OutgoingSnapshotsManager.java:184)
~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
at
java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1768)
~[?:?]
... 3 more
> StorageClosedException in Raft snapshot streaming
> -------------------------------------------------
>
> Key: IGNITE-25384
> URL: https://issues.apache.org/jira/browse/IGNITE-25384
> Project: Ignite
> Issue Type: Bug
> Reporter: Roman Puchkovskiy
> Priority: Major
> Labels: ignite-3
> Attachments: _Integration_Tests_Run_All_Other_36172.log.zip
>
>
> There seems to be a race between sending a partition Raft snapshot (in
> OutgoingSnapshot) and node stop:
>
> [2025-05-15T08:56:37,617][ERROR][%izdrt_tdr_20001%outgoing-snapshots-3][FailureManager]
> Critical system error detected. Will be handled accordingly to configured
> handler [hnd=NoOpFailureHandler [super=AbstractFailureHandler
> [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED,
> SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=CRITICAL_ERROR]
> org.apache.ignite.internal.failure.StackTraceCapturingException: Something
> went wrong while handling a request
> at
> org.apache.ignite.internal.failure.FailureManager.process(FailureManager.java:161)
> ~[ignite-failure-handler-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.failure.FailureManager.process(FailureManager.java:138)
> ~[ignite-failure-handler-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshotsManager.respond(OutgoingSnapshotsManager.java:206)
> ~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshotsManager.lambda$handleMessage$2(OutgoingSnapshotsManager.java:185)
> ~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
> at
> java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
> [?:?]
> at
> java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
> [?:?]
> at
> java.base/java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:482)
> [?:?]
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
> [?:?]
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
> [?:?]
> at java.base/java.lang.Thread.run(Thread.java:833) [?:?]
> Caused by: java.util.concurrent.CompletionException:
> org.apache.ignite.internal.storage.StorageClosedException: IGN-CMN-65535
> TraceId:a2515b4b-c1a3-47c5-a48c-4bbd2ebc3bd0 Storage is already closed:
> [tableId=6, partitionId=0]
> at
> java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:315)
> ~[?:?]
> at
> java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:320)
> ~[?:?]
> at
> java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1770)
> ~[?:?]
> ... 3 more
> Caused by: org.apache.ignite.internal.storage.StorageClosedException: Storage
> is already closed: [tableId=6, partitionId=0]
> at
> org.apache.ignite.internal.storage.util.StorageUtils.throwExceptionDependingOnStorageState(StorageUtils.java:141)
> ~[ignite-storage-api-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.storage.util.StorageUtils.throwExceptionIfStorageNotInRunnableState(StorageUtils.java:63)
> ~[ignite-storage-api-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.storage.pagememory.mv.AbstractPageMemoryMvPartitionStorage.throwExceptionIfStorageNotInRunnableState(AbstractPageMemoryMvPartitionStorage.java:801)
> ~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.storage.pagememory.mv.ScanVersionsCursor.lambda$hasNext$0(ScanVersionsCursor.java:67)
> ~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.storage.pagememory.mv.AbstractPageMemoryMvPartitionStorage.busy(AbstractPageMemoryMvPartitionStorage.java:656)
> ~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.storage.pagememory.mv.ScanVersionsCursor.hasNext(ScanVersionsCursor.java:66)
> ~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
> at java.base/java.util.Iterator.forEachRemaining(Iterator.java:132) ~[?:?]
> at
> java.base/java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1845)
> ~[?:?]
> at
> java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
> ~[?:?]
> at
> java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
> ~[?:?]
> at
> java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:921)
> ~[?:?]
> at
> java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> ~[?:?]
> at
> java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:682)
> ~[?:?]
> at
> org.apache.ignite.internal.table.distributed.raft.snapshot.PartitionMvStorageAccessImpl.lambda$getAllRowVersions$0(PartitionMvStorageAccessImpl.java:126)
> ~[ignite-table-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.storage.pagememory.mv.PersistentPageMemoryMvPartitionStorage.lambda$runConsistently$0(PersistentPageMemoryMvPartitionStorage.java:180)
> ~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.storage.pagememory.mv.AbstractPageMemoryMvPartitionStorage.busy(AbstractPageMemoryMvPartitionStorage.java:656)
> ~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.storage.pagememory.mv.PersistentPageMemoryMvPartitionStorage.runConsistently(PersistentPageMemoryMvPartitionStorage.java:170)
> ~[ignite-storage-page-memory-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.table.distributed.raft.snapshot.PartitionMvStorageAccessImpl.getAllRowVersions(PartitionMvStorageAccessImpl.java:122)
> ~[ignite-table-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshot.rowEntry(OutgoingSnapshot.java:404)
> ~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshot.tryProcessRowFromPartition(OutgoingSnapshot.java:385)
> ~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshot.handleSnapshotMvDataRequest(OutgoingSnapshot.java:314)
> ~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshotsManager.handleSnapshotRequestMessage(OutgoingSnapshotsManager.java:194)
> ~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
> at
> org.apache.ignite.internal.partition.replicator.raft.snapshot.outgoing.OutgoingSnapshotsManager.lambda$handleMessage$1(OutgoingSnapshotsManager.java:184)
> ~[ignite-partition-replicator-3.1.0-SNAPSHOT.jar:?]
> at
> java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1768)
> ~[?:?]
> ... 3 more
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
