[ https://issues.apache.org/jira/browse/IGNITE-16805?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Pavel Pereslegin updated IGNITE-16805: -------------------------------------- Description: h5. Cache Restarts 1 suite hangs on TeamCity due to GridCachePartitionedOptimisticTxNodeRestartTest test. https://ci.ignite.apache.org/viewType.html?buildTypeId=IgniteTests24Java8_CacheRestarts1&tab=buildTypeHistoryList&branch_IgniteTests24Java8=%3Cdefault%3E {noformat} Thread [name="test-runner-#376077%near.GridCachePartitionedOptimisticTxNodeRestartTest%", id=383240, state=WAITING, blockCnt=20, waitCnt=42] Lock [object=java.lang.Thread@686cf8ad, ownerName=null, ownerId=-1] at java.base@11.0.8/java.lang.Object.wait(Native Method) at java.base@11.0.8/java.lang.Thread.join(Thread.java:1305) at java.base@11.0.8/java.lang.Thread.join(Thread.java:1380) at o.a.i.i.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.checkRestartWithTx(GridCacheAbstractNodeRestartSelfTest.java:850) at o.a.i.i.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.testRestartWithTxTenNodesTwoBackups(GridCacheAbstractNodeRestartSelfTest.java:543) at o.a.i.i.processors.cache.distributed.near.GridCachePartitionedOptimisticTxNodeRestartTest.testRestartWithTxTenNodesTwoBackups(GridCachePartitionedOptimisticTxNodeRestartTest.java:141) at java.base@11.0.8/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base@11.0.8/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base@11.0.8/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base@11.0.8/java.lang.reflect.Method.invoke(Method.java:566) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at o.a.i.testframework.junits.GridAbstractTest$6.run(GridAbstractTest.java:2431) at java.base@11.0.8/java.lang.Thread.run(Thread.java:834) .... "test-runner-#376077%near.GridCachePartitionedOptimisticTxNodeRestartTest%" #383240 prio=5 os_prio=0 cpu=649.37ms elapsed=6627.99s tid=0x00007f69575ff000 nid=0x6474 waiting on condition [0x00007f68edcc2000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x0000000087987d88> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.8/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.8/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(java.base@11.0.8/AbstractQueuedSynchronizer.java:1039) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(java.base@11.0.8/AbstractQueuedSynchronizer.java:1345) at java.util.concurrent.CountDownLatch.await(java.base@11.0.8/CountDownLatch.java:232) at org.apache.ignite.internal.util.IgniteUtils.awaitQuiet(IgniteUtils.java:8106) at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.grid(IgnitionEx.java:1657) at org.apache.ignite.internal.IgnitionEx.allGrids(IgnitionEx.java:1292) at org.apache.ignite.internal.IgnitionEx.allGrids(IgnitionEx.java:1270) at org.apache.ignite.Ignition.allGrids(Ignition.java:503) at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1563) at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1550) at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1542) at org.apache.ignite.internal.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.checkRestartWithTx(GridCacheAbstractNodeRestartSelfTest.java:856) at org.apache.ignite.internal.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.testRestartWithTxTenNodesTwoBackups(GridCacheAbstractNodeRestartSelfTest.java:543) at org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedOptimisticTxNodeRestartTest.testRestartWithTxTenNodesTwoBackups(GridCachePartitionedOptimisticTxNodeRestartTest.java:141) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@11.0.8/Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@11.0.8/NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.8/DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(java.base@11.0.8/Method.java:566) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.apache.ignite.testframework.junits.GridAbstractTest$6.run(GridAbstractTest.java:2431) {noformat} h5. One of the reasons why the suite hangs is that the context of the exchange future has become null. h6. Example 1 {noformat} java.lang.NullPointerException at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.events(GridDhtPartitionsExchangeFuture.java:818) at org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.update(GridDhtPartitionTopologyImpl.java:1550) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.processFullPartitionUpdate(GridCachePartitionExchangeManager.java:1965) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.lambda$addOrMergeDelayedFullMessage$d3c0dac7$1(GridDhtPartitionsExchangeFuture.java:5532) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2597) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:158) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.onKernalStop0(GridCachePartitionExchangeManager.java:999) at org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter.onKernalStop(GridCacheSharedManagerAdapter.java:135) at org.apache.ignite.internal.processors.cache.GridCacheProcessor.onKernalStop(GridCacheProcessor.java:846) at org.apache.ignite.internal.IgniteKernal.stop0(IgniteKernal.java:2527) at org.apache.ignite.internal.IgniteKernal.stop(IgniteKernal.java:2475) at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2383) at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2206) at org.apache.ignite.internal.IgnitionEx.stop(IgnitionEx.java:351) {noformat} h6. Example 2 {noformat} [00:10:55] (err) Failed to notify listener: o.a.i.i.processors.timeout.GridTimeoutProcessor$2...@5051f46fjava.lang.AssertionError at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.serverNodeDiscoveryEvent(GridDhtPartitionsExchangeFuture.java:2368) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2461) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.finishExchangeOnCoordinator(GridDhtPartitionsExchangeFuture.java:4125) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onAllReceived(GridDhtPartitionsExchangeFuture.java:3833) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onBecomeCoordinator(GridDhtPartitionsExchangeFuture.java:5441) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.access$3600(GridDhtPartitionsExchangeFuture.java:159) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8$1$1.apply(GridDhtPartitionsExchangeFuture.java:5221) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8$1$1.apply(GridDhtPartitionsExchangeFuture.java:5210) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:467) at org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:351) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:152) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:47) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:456) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.InitNewCoordinatorFuture.onNodeLeft(InitNewCoordinatorFuture.java:372) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8.run(GridDhtPartitionsExchangeFuture.java:5112) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDiscoveryEvent(GridDhtPartitionsExchangeFuture.java:5030) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onNodeLeft(GridDhtPartitionsExchangeFuture.java:5091) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.notifyNodeFail(GridCachePartitionExchangeManager.java:389) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.onDiscoveryEvent(GridCachePartitionExchangeManager.java:725) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.access$300(GridCachePartitionExchangeManager.java:197) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$1.onEvent(GridCachePartitionExchangeManager.java:367) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager$DiscoveryListenerWrapper.onEvent(GridEventStorageManager.java:1458) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:903) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:888) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record0(GridEventStorageManager.java:359) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record(GridEventStorageManager.java:322) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.recordEvent(GridDiscoveryManager.java:3055) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscoveryManager.java:3272) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryManager.java:3075) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125) at java.lang.Thread.run(Thread.java:748) {noformat} h5. Another reason for hanging (deadlock on topology lock) {noformat} Stop-thread (tries to cancel current exchange - and gets stuck on the topology lock): "main" #1 prio=5 os_prio=0 tid=0x00007f88d4010000 nid=0x45f2 waiting on condition [0x00007f88da861000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000009ae38570> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943) at org.apache.ignite.internal.util.StripedCompositeReadWriteLock$WriteLock.lock0(StripedCompositeReadWriteLock.java:194) at org.apache.ignite.internal.util.StripedCompositeReadWriteLock$WriteLock.lock(StripedCompositeReadWriteLock.java:163) at org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.update(GridDhtPartitionTopologyImpl.java:1442) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.processFullPartitionUpdate(GridCachePartitionExchangeManager.java:1965) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.lambda$addOrMergeDelayedFullMessage$d3c0dac7$1(GridDhtPartitionsExchangeFuture.java:5532) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$$Lambda$885/815472450.apply(Unknown Source) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2597) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:158) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.onKernalStop0(GridCachePartitionExchangeManager.java:999) at org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter.onKernalStop(GridCacheSharedManagerAdapter.java:135) at org.apache.ignite.internal.processors.cache.GridCacheProcessor.onKernalStop(GridCacheProcessor.java:846) at org.apache.ignite.internal.IgniteKernal.stop0(IgniteKernal.java:2527) at org.apache.ignite.internal.IgniteKernal.stop(IgniteKernal.java:2475) at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2383) - locked <0x000000009a40c248> (a org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance) at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2206) at org.apache.ignite.internal.IgnitionEx.stop(IgnitionEx.java:351) at org.apache.ignite.testframework.junits.GridAbstractTest.stopGrid0(GridAbstractTest.java:1516) at org.apache.ignite.testframework.junits.GridAbstractTest.stopGrid(GridAbstractTest.java:1484) at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1578) at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1550) at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1542) at org.apache.ignite.internal.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.afterTest(GridCacheAbstractNodeRestartSelfTest.java:157) at org.apache.ignite.testframework.junits.GridAbstractTest.cleanUpTestEnviroment(GridAbstractTest.java:739) Tx prepare thread (got top lock and waits topology) "sys-stripe-7-#620097%dht.GridCachePartitionedNearDisabledOptimisticTxNodeRestartTest4% (waiting AffinityTopologyVersion [topVer=58, minorTopVer=0])" #631918 prio=5 os_prio=0 tid=0x00007f88d6c7b800 nid=0x70b2 waiting on condition [0x00007f8872642000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at org.apache.ignite.internal.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:178) at org.apache.ignite.internal.util.future.GridFutureAdapter.get(GridFutureAdapter.java:141) at org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache.awaitTopologyVersion(GridAffinityAssignmentCache.java:944) at org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache.cachedAffinity(GridAffinityAssignmentCache.java:824) at org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache.cachedAffinity(GridAffinityAssignmentCache.java:801) at org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache.nodes(GridAffinityAssignmentCache.java:704) at org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.partitionLocalNode(GridDhtPartitionTopologyImpl.java:758) at org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.localPartition0(GridDhtPartitionTopologyImpl.java:999) at org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.localPartition(GridDhtPartitionTopologyImpl.java:885) at org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedConcurrentMap.localPartition(GridCachePartitionedConcurrentMap.java:70) at org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedConcurrentMap.putEntryIfObsoleteOrAbsent(GridCachePartitionedConcurrentMap.java:91) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.entryEx(GridCacheAdapter.java:1013) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter.entryEx(GridDhtCacheAdapter.java:436) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter.entryExx(GridDhtCacheAdapter.java:456) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.addEntry(GridDhtTxLocalAdapter.java:502) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.prepareAsync(GridDhtTxLocal.java:389) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.prepareNearTx(IgniteTxHandler.java:580) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.prepareNearTx(IgniteTxHandler.java:377) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxPrepareRequest0(IgniteTxHandler.java:200) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxPrepareRequest(IgniteTxHandler.java:174) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$000(IgniteTxHandler.java:134) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$1.apply(IgniteTxHandler.java:222) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$1.apply(IgniteTxHandler.java:220) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1151) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:592) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:393) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:319) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:110) at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:309) at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1909) at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1530) at org.apache.ignite.internal.managers.communication.GridIoManager.access$5300(GridIoManager.java:243) at org.apache.ignite.internal.managers.communication.GridIoManager$9.execute(GridIoManager.java:1423) at org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:55) at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:637) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125) at java.lang.Thread.run(Thread.java:748) {noformat} was: h5. Cache Restarts 1 suite hangs on TeamCity due to GridCachePartitionedOptimisticTxNodeRestartTest test. https://ci.ignite.apache.org/viewType.html?buildTypeId=IgniteTests24Java8_CacheRestarts1&tab=buildTypeHistoryList&branch_IgniteTests24Java8=%3Cdefault%3E {noformat} Thread [name="test-runner-#376077%near.GridCachePartitionedOptimisticTxNodeRestartTest%", id=383240, state=WAITING, blockCnt=20, waitCnt=42] Lock [object=java.lang.Thread@686cf8ad, ownerName=null, ownerId=-1] at java.base@11.0.8/java.lang.Object.wait(Native Method) at java.base@11.0.8/java.lang.Thread.join(Thread.java:1305) at java.base@11.0.8/java.lang.Thread.join(Thread.java:1380) at o.a.i.i.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.checkRestartWithTx(GridCacheAbstractNodeRestartSelfTest.java:850) at o.a.i.i.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.testRestartWithTxTenNodesTwoBackups(GridCacheAbstractNodeRestartSelfTest.java:543) at o.a.i.i.processors.cache.distributed.near.GridCachePartitionedOptimisticTxNodeRestartTest.testRestartWithTxTenNodesTwoBackups(GridCachePartitionedOptimisticTxNodeRestartTest.java:141) at java.base@11.0.8/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base@11.0.8/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base@11.0.8/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base@11.0.8/java.lang.reflect.Method.invoke(Method.java:566) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at o.a.i.testframework.junits.GridAbstractTest$6.run(GridAbstractTest.java:2431) at java.base@11.0.8/java.lang.Thread.run(Thread.java:834) .... "test-runner-#376077%near.GridCachePartitionedOptimisticTxNodeRestartTest%" #383240 prio=5 os_prio=0 cpu=649.37ms elapsed=6627.99s tid=0x00007f69575ff000 nid=0x6474 waiting on condition [0x00007f68edcc2000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) - parking to wait for <0x0000000087987d88> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.8/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.8/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(java.base@11.0.8/AbstractQueuedSynchronizer.java:1039) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(java.base@11.0.8/AbstractQueuedSynchronizer.java:1345) at java.util.concurrent.CountDownLatch.await(java.base@11.0.8/CountDownLatch.java:232) at org.apache.ignite.internal.util.IgniteUtils.awaitQuiet(IgniteUtils.java:8106) at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.grid(IgnitionEx.java:1657) at org.apache.ignite.internal.IgnitionEx.allGrids(IgnitionEx.java:1292) at org.apache.ignite.internal.IgnitionEx.allGrids(IgnitionEx.java:1270) at org.apache.ignite.Ignition.allGrids(Ignition.java:503) at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1563) at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1550) at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1542) at org.apache.ignite.internal.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.checkRestartWithTx(GridCacheAbstractNodeRestartSelfTest.java:856) at org.apache.ignite.internal.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.testRestartWithTxTenNodesTwoBackups(GridCacheAbstractNodeRestartSelfTest.java:543) at org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedOptimisticTxNodeRestartTest.testRestartWithTxTenNodesTwoBackups(GridCachePartitionedOptimisticTxNodeRestartTest.java:141) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@11.0.8/Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@11.0.8/NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.8/DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(java.base@11.0.8/Method.java:566) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.apache.ignite.testframework.junits.GridAbstractTest$6.run(GridAbstractTest.java:2431) {noformat} h5. One of the reasons why the suite hangs is that the context of the exchange future has become null. h6. Example 1 {noformat} java.lang.NullPointerException at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.events(GridDhtPartitionsExchangeFuture.java:818) at org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.update(GridDhtPartitionTopologyImpl.java:1550) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.processFullPartitionUpdate(GridCachePartitionExchangeManager.java:1965) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.lambda$addOrMergeDelayedFullMessage$d3c0dac7$1(GridDhtPartitionsExchangeFuture.java:5532) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2597) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:158) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.onKernalStop0(GridCachePartitionExchangeManager.java:999) at org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter.onKernalStop(GridCacheSharedManagerAdapter.java:135) at org.apache.ignite.internal.processors.cache.GridCacheProcessor.onKernalStop(GridCacheProcessor.java:846) at org.apache.ignite.internal.IgniteKernal.stop0(IgniteKernal.java:2527) at org.apache.ignite.internal.IgniteKernal.stop(IgniteKernal.java:2475) at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2383) at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2206) at org.apache.ignite.internal.IgnitionEx.stop(IgnitionEx.java:351) {noformat} h6. Example 2 {noformat} [00:10:55] (err) Failed to notify listener: o.a.i.i.processors.timeout.GridTimeoutProcessor$2...@5051f46fjava.lang.AssertionError at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.serverNodeDiscoveryEvent(GridDhtPartitionsExchangeFuture.java:2368) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2461) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.finishExchangeOnCoordinator(GridDhtPartitionsExchangeFuture.java:4125) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onAllReceived(GridDhtPartitionsExchangeFuture.java:3833) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onBecomeCoordinator(GridDhtPartitionsExchangeFuture.java:5441) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.access$3600(GridDhtPartitionsExchangeFuture.java:159) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8$1$1.apply(GridDhtPartitionsExchangeFuture.java:5221) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8$1$1.apply(GridDhtPartitionsExchangeFuture.java:5210) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:467) at org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:351) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:152) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:47) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:456) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.InitNewCoordinatorFuture.onNodeLeft(InitNewCoordinatorFuture.java:372) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8.run(GridDhtPartitionsExchangeFuture.java:5112) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDiscoveryEvent(GridDhtPartitionsExchangeFuture.java:5030) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onNodeLeft(GridDhtPartitionsExchangeFuture.java:5091) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.notifyNodeFail(GridCachePartitionExchangeManager.java:389) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.onDiscoveryEvent(GridCachePartitionExchangeManager.java:725) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.access$300(GridCachePartitionExchangeManager.java:197) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$1.onEvent(GridCachePartitionExchangeManager.java:367) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager$DiscoveryListenerWrapper.onEvent(GridEventStorageManager.java:1458) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:903) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:888) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record0(GridEventStorageManager.java:359) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record(GridEventStorageManager.java:322) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.recordEvent(GridDiscoveryManager.java:3055) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscoveryManager.java:3272) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryManager.java:3075) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125) at java.lang.Thread.run(Thread.java:748) {noformat} h5. Another reason for hanging (deadlock on topology lock) {noformat} Stop-thread (wait for topology lock): "main" #1 prio=5 os_prio=0 tid=0x00007f88d4010000 nid=0x45f2 waiting on condition [0x00007f88da861000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000009ae38570> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943) at org.apache.ignite.internal.util.StripedCompositeReadWriteLock$WriteLock.lock0(StripedCompositeReadWriteLock.java:194) at org.apache.ignite.internal.util.StripedCompositeReadWriteLock$WriteLock.lock(StripedCompositeReadWriteLock.java:163) at org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.update(GridDhtPartitionTopologyImpl.java:1442) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.processFullPartitionUpdate(GridCachePartitionExchangeManager.java:1965) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.lambda$addOrMergeDelayedFullMessage$d3c0dac7$1(GridDhtPartitionsExchangeFuture.java:5532) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$$Lambda$885/815472450.apply(Unknown Source) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2597) at org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:158) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) at org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.onKernalStop0(GridCachePartitionExchangeManager.java:999) at org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter.onKernalStop(GridCacheSharedManagerAdapter.java:135) at org.apache.ignite.internal.processors.cache.GridCacheProcessor.onKernalStop(GridCacheProcessor.java:846) at org.apache.ignite.internal.IgniteKernal.stop0(IgniteKernal.java:2527) at org.apache.ignite.internal.IgniteKernal.stop(IgniteKernal.java:2475) at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2383) - locked <0x000000009a40c248> (a org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance) at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2206) at org.apache.ignite.internal.IgnitionEx.stop(IgnitionEx.java:351) at org.apache.ignite.testframework.junits.GridAbstractTest.stopGrid0(GridAbstractTest.java:1516) at org.apache.ignite.testframework.junits.GridAbstractTest.stopGrid(GridAbstractTest.java:1484) at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1578) at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1550) at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1542) at org.apache.ignite.internal.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.afterTest(GridCacheAbstractNodeRestartSelfTest.java:157) at org.apache.ignite.testframework.junits.GridAbstractTest.cleanUpTestEnviroment(GridAbstractTest.java:739) Tx prepare thread (got top lock and waits topology) "sys-stripe-7-#620097%dht.GridCachePartitionedNearDisabledOptimisticTxNodeRestartTest4% (waiting AffinityTopologyVersion [topVer=58, minorTopVer=0])" #631918 prio=5 os_prio=0 tid=0x00007f88d6c7b800 nid=0x70b2 waiting on condition [0x00007f8872642000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) at org.apache.ignite.internal.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:178) at org.apache.ignite.internal.util.future.GridFutureAdapter.get(GridFutureAdapter.java:141) at org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache.awaitTopologyVersion(GridAffinityAssignmentCache.java:944) at org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache.cachedAffinity(GridAffinityAssignmentCache.java:824) at org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache.cachedAffinity(GridAffinityAssignmentCache.java:801) at org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache.nodes(GridAffinityAssignmentCache.java:704) at org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.partitionLocalNode(GridDhtPartitionTopologyImpl.java:758) at org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.localPartition0(GridDhtPartitionTopologyImpl.java:999) at org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.localPartition(GridDhtPartitionTopologyImpl.java:885) at org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedConcurrentMap.localPartition(GridCachePartitionedConcurrentMap.java:70) at org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedConcurrentMap.putEntryIfObsoleteOrAbsent(GridCachePartitionedConcurrentMap.java:91) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.entryEx(GridCacheAdapter.java:1013) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter.entryEx(GridDhtCacheAdapter.java:436) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter.entryExx(GridDhtCacheAdapter.java:456) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.addEntry(GridDhtTxLocalAdapter.java:502) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.prepareAsync(GridDhtTxLocal.java:389) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.prepareNearTx(IgniteTxHandler.java:580) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.prepareNearTx(IgniteTxHandler.java:377) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxPrepareRequest0(IgniteTxHandler.java:200) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxPrepareRequest(IgniteTxHandler.java:174) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$000(IgniteTxHandler.java:134) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$1.apply(IgniteTxHandler.java:222) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$1.apply(IgniteTxHandler.java:220) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1151) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:592) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:393) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:319) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:110) at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:309) at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1909) at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1530) at org.apache.ignite.internal.managers.communication.GridIoManager.access$5300(GridIoManager.java:243) at org.apache.ignite.internal.managers.communication.GridIoManager$9.execute(GridIoManager.java:1423) at org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:55) at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:637) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125) at java.lang.Thread.run(Thread.java:748) {noformat} > Cache Restarts 1 suite hangs > ---------------------------- > > Key: IGNITE-16805 > URL: https://issues.apache.org/jira/browse/IGNITE-16805 > Project: Ignite > Issue Type: Bug > Reporter: Pavel Pereslegin > Assignee: Pavel Pereslegin > Priority: Minor > Fix For: 2.14 > > Time Spent: 0.5h > Remaining Estimate: 0h > > h5. Cache Restarts 1 suite hangs on TeamCity due to > GridCachePartitionedOptimisticTxNodeRestartTest test. > https://ci.ignite.apache.org/viewType.html?buildTypeId=IgniteTests24Java8_CacheRestarts1&tab=buildTypeHistoryList&branch_IgniteTests24Java8=%3Cdefault%3E > {noformat} > Thread > [name="test-runner-#376077%near.GridCachePartitionedOptimisticTxNodeRestartTest%", > id=383240, state=WAITING, blockCnt=20, waitCnt=42] > Lock [object=java.lang.Thread@686cf8ad, ownerName=null, ownerId=-1] > at java.base@11.0.8/java.lang.Object.wait(Native Method) > at java.base@11.0.8/java.lang.Thread.join(Thread.java:1305) > at java.base@11.0.8/java.lang.Thread.join(Thread.java:1380) > at > o.a.i.i.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.checkRestartWithTx(GridCacheAbstractNodeRestartSelfTest.java:850) > at > o.a.i.i.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.testRestartWithTxTenNodesTwoBackups(GridCacheAbstractNodeRestartSelfTest.java:543) > at > o.a.i.i.processors.cache.distributed.near.GridCachePartitionedOptimisticTxNodeRestartTest.testRestartWithTxTenNodesTwoBackups(GridCachePartitionedOptimisticTxNodeRestartTest.java:141) > at > java.base@11.0.8/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native > Method) > at > java.base@11.0.8/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > java.base@11.0.8/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.base@11.0.8/java.lang.reflect.Method.invoke(Method.java:566) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > o.a.i.testframework.junits.GridAbstractTest$6.run(GridAbstractTest.java:2431) > at java.base@11.0.8/java.lang.Thread.run(Thread.java:834) > .... > "test-runner-#376077%near.GridCachePartitionedOptimisticTxNodeRestartTest%" > #383240 prio=5 os_prio=0 cpu=649.37ms elapsed=6627.99s tid=0x00007f69575ff000 > nid=0x6474 waiting on condition [0x00007f68edcc2000] > java.lang.Thread.State: WAITING (parking) > at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method) > - parking to wait for <0x0000000087987d88> (a > java.util.concurrent.CountDownLatch$Sync) > at > java.util.concurrent.locks.LockSupport.park(java.base@11.0.8/LockSupport.java:194) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.8/AbstractQueuedSynchronizer.java:885) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(java.base@11.0.8/AbstractQueuedSynchronizer.java:1039) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(java.base@11.0.8/AbstractQueuedSynchronizer.java:1345) > at > java.util.concurrent.CountDownLatch.await(java.base@11.0.8/CountDownLatch.java:232) > at > org.apache.ignite.internal.util.IgniteUtils.awaitQuiet(IgniteUtils.java:8106) > at > org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.grid(IgnitionEx.java:1657) > at org.apache.ignite.internal.IgnitionEx.allGrids(IgnitionEx.java:1292) > at org.apache.ignite.internal.IgnitionEx.allGrids(IgnitionEx.java:1270) > at org.apache.ignite.Ignition.allGrids(Ignition.java:503) > at > org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1563) > at > org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1550) > at > org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1542) > at > org.apache.ignite.internal.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.checkRestartWithTx(GridCacheAbstractNodeRestartSelfTest.java:856) > at > org.apache.ignite.internal.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.testRestartWithTxTenNodesTwoBackups(GridCacheAbstractNodeRestartSelfTest.java:543) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedOptimisticTxNodeRestartTest.testRestartWithTxTenNodesTwoBackups(GridCachePartitionedOptimisticTxNodeRestartTest.java:141) > at > jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@11.0.8/Native > Method) > at > jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@11.0.8/NativeMethodAccessorImpl.java:62) > at > jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.8/DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(java.base@11.0.8/Method.java:566) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > org.apache.ignite.testframework.junits.GridAbstractTest$6.run(GridAbstractTest.java:2431) > {noformat} > h5. One of the reasons why the suite hangs is that the context of the > exchange future has become null. > h6. Example 1 > {noformat} > java.lang.NullPointerException > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.events(GridDhtPartitionsExchangeFuture.java:818) > at > org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.update(GridDhtPartitionTopologyImpl.java:1550) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.processFullPartitionUpdate(GridCachePartitionExchangeManager.java:1965) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.lambda$addOrMergeDelayedFullMessage$d3c0dac7$1(GridDhtPartitionsExchangeFuture.java:5532) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2597) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:158) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.onKernalStop0(GridCachePartitionExchangeManager.java:999) > at > org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter.onKernalStop(GridCacheSharedManagerAdapter.java:135) > at > org.apache.ignite.internal.processors.cache.GridCacheProcessor.onKernalStop(GridCacheProcessor.java:846) > at > org.apache.ignite.internal.IgniteKernal.stop0(IgniteKernal.java:2527) > at > org.apache.ignite.internal.IgniteKernal.stop(IgniteKernal.java:2475) > at > org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2383) > at > org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2206) > at org.apache.ignite.internal.IgnitionEx.stop(IgnitionEx.java:351) > {noformat} > h6. Example 2 > {noformat} > [00:10:55] (err) Failed to notify listener: > o.a.i.i.processors.timeout.GridTimeoutProcessor$2...@5051f46fjava.lang.AssertionError > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.serverNodeDiscoveryEvent(GridDhtPartitionsExchangeFuture.java:2368) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2461) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.finishExchangeOnCoordinator(GridDhtPartitionsExchangeFuture.java:4125) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onAllReceived(GridDhtPartitionsExchangeFuture.java:3833) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onBecomeCoordinator(GridDhtPartitionsExchangeFuture.java:5441) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.access$3600(GridDhtPartitionsExchangeFuture.java:159) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8$1$1.apply(GridDhtPartitionsExchangeFuture.java:5221) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8$1$1.apply(GridDhtPartitionsExchangeFuture.java:5210) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:467) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:351) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:152) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:47) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:456) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.InitNewCoordinatorFuture.onNodeLeft(InitNewCoordinatorFuture.java:372) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8.run(GridDhtPartitionsExchangeFuture.java:5112) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDiscoveryEvent(GridDhtPartitionsExchangeFuture.java:5030) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onNodeLeft(GridDhtPartitionsExchangeFuture.java:5091) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.notifyNodeFail(GridCachePartitionExchangeManager.java:389) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.onDiscoveryEvent(GridCachePartitionExchangeManager.java:725) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.access$300(GridCachePartitionExchangeManager.java:197) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$1.onEvent(GridCachePartitionExchangeManager.java:367) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager$DiscoveryListenerWrapper.onEvent(GridEventStorageManager.java:1458) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:903) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:888) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record0(GridEventStorageManager.java:359) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record(GridEventStorageManager.java:322) > at > org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.recordEvent(GridDiscoveryManager.java:3055) > at > org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscoveryManager.java:3272) > at > org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryManager.java:3075) > at > org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125) > at java.lang.Thread.run(Thread.java:748) > {noformat} > h5. Another reason for hanging (deadlock on topology lock) > {noformat} > Stop-thread (tries to cancel current exchange - and gets stuck on the > topology lock): > "main" #1 prio=5 os_prio=0 tid=0x00007f88d4010000 nid=0x45f2 waiting on > condition [0x00007f88da861000] > java.lang.Thread.State: WAITING (parking) > at sun.misc.Unsafe.park(Native Method) > - parking to wait for <0x000000009ae38570> (a > java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) > at > java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943) > at > org.apache.ignite.internal.util.StripedCompositeReadWriteLock$WriteLock.lock0(StripedCompositeReadWriteLock.java:194) > at > org.apache.ignite.internal.util.StripedCompositeReadWriteLock$WriteLock.lock(StripedCompositeReadWriteLock.java:163) > at > org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.update(GridDhtPartitionTopologyImpl.java:1442) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.processFullPartitionUpdate(GridCachePartitionExchangeManager.java:1965) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.lambda$addOrMergeDelayedFullMessage$d3c0dac7$1(GridDhtPartitionsExchangeFuture.java:5532) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$$Lambda$885/815472450.apply(Unknown > Source) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2597) > at > org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:158) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) > at > org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager.onKernalStop0(GridCachePartitionExchangeManager.java:999) > at > org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter.onKernalStop(GridCacheSharedManagerAdapter.java:135) > at > org.apache.ignite.internal.processors.cache.GridCacheProcessor.onKernalStop(GridCacheProcessor.java:846) > at org.apache.ignite.internal.IgniteKernal.stop0(IgniteKernal.java:2527) > at org.apache.ignite.internal.IgniteKernal.stop(IgniteKernal.java:2475) > at > org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2383) > - locked <0x000000009a40c248> (a > org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance) > at > org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2206) > at org.apache.ignite.internal.IgnitionEx.stop(IgnitionEx.java:351) > at > org.apache.ignite.testframework.junits.GridAbstractTest.stopGrid0(GridAbstractTest.java:1516) > at > org.apache.ignite.testframework.junits.GridAbstractTest.stopGrid(GridAbstractTest.java:1484) > at > org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1578) > at > org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1550) > at > org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1542) > at > org.apache.ignite.internal.processors.cache.distributed.GridCacheAbstractNodeRestartSelfTest.afterTest(GridCacheAbstractNodeRestartSelfTest.java:157) > at > org.apache.ignite.testframework.junits.GridAbstractTest.cleanUpTestEnviroment(GridAbstractTest.java:739) > Tx prepare thread (got top lock and waits topology) > "sys-stripe-7-#620097%dht.GridCachePartitionedNearDisabledOptimisticTxNodeRestartTest4% > (waiting AffinityTopologyVersion [topVer=58, minorTopVer=0])" #631918 prio=5 > os_prio=0 tid=0x00007f88d6c7b800 nid=0x70b2 waiting on condition > [0x00007f8872642000] > java.lang.Thread.State: WAITING (parking) > at sun.misc.Unsafe.park(Native Method) > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:178) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.get(GridFutureAdapter.java:141) > at > org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache.awaitTopologyVersion(GridAffinityAssignmentCache.java:944) > at > org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache.cachedAffinity(GridAffinityAssignmentCache.java:824) > at > org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache.cachedAffinity(GridAffinityAssignmentCache.java:801) > at > org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache.nodes(GridAffinityAssignmentCache.java:704) > at > org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.partitionLocalNode(GridDhtPartitionTopologyImpl.java:758) > at > org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.localPartition0(GridDhtPartitionTopologyImpl.java:999) > at > org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl.localPartition(GridDhtPartitionTopologyImpl.java:885) > at > org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedConcurrentMap.localPartition(GridCachePartitionedConcurrentMap.java:70) > at > org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedConcurrentMap.putEntryIfObsoleteOrAbsent(GridCachePartitionedConcurrentMap.java:91) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter.entryEx(GridCacheAdapter.java:1013) > at > org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter.entryEx(GridDhtCacheAdapter.java:436) > at > org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter.entryExx(GridDhtCacheAdapter.java:456) > at > org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.addEntry(GridDhtTxLocalAdapter.java:502) > at > org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal.prepareAsync(GridDhtTxLocal.java:389) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.prepareNearTx(IgniteTxHandler.java:580) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.prepareNearTx(IgniteTxHandler.java:377) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxPrepareRequest0(IgniteTxHandler.java:200) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processNearTxPrepareRequest(IgniteTxHandler.java:174) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$000(IgniteTxHandler.java:134) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$1.apply(IgniteTxHandler.java:222) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$1.apply(IgniteTxHandler.java:220) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1151) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:592) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:393) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:319) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:110) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:309) > at > org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1909) > at > org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1530) > at > org.apache.ignite.internal.managers.communication.GridIoManager.access$5300(GridIoManager.java:243) > at > org.apache.ignite.internal.managers.communication.GridIoManager$9.execute(GridIoManager.java:1423) > at > org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:55) > at > org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:637) > at > org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125) > at java.lang.Thread.run(Thread.java:748) > {noformat} -- This message was sent by Atlassian Jira (v8.20.1#820001)