[ https://issues.apache.org/jira/browse/IGNITE-6425?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Eduard Shangareev updated IGNITE-6425: -------------------------------------- Description: I have found during stress-test (start-stop nodes during transactions running): {code} [2017-09-20 12:37:04,224][ERROR][updater-1][GridDhtColocatedCache] <cache1> Failed to rollback transaction (cache may contain stale locks): GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=null], nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=null, hasRemoteLocks=false, thread=updater-1, mappings=IgniteTxMappingsSingleImpl [mapping=null], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true, recovery=false], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=117380231, order=1505900296519, nodeOrder=1], writeVer=null, implicit=true, loc=true, threadId=15914, startTime=1505900224068, nodeId=2699eb85-b97a-431f-a038-a6970ee00000, startVer=GridCacheVersion [topVer=117380231, order=1505900296519, nodeOrder=1], endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=GridCacheVersion [topVer=117380231, order=1505900296519, nodeOrder=1], finalizing=NONE, invalidParts=null, state=ROLLED_BACK, timedOut=false, topVer=AffinityTopologyVersion [topVer=12, minorTopVer=0], duration=153ms, onePhaseCommit=false], size=1]]] class org.apache.ignite.IgniteCheckedException: Failed to commit transaction: GridNearTxLocal[id=749f6ae9e51-00000000-06ff-1487-0000-000000000001, concurrency=OPTIMISTIC, isolation=READ_COMMITTED, state=ROLLED_BACK, invalidate=false, rollbackOnly=true, nodeId=2699eb85-b97a-431f-a038-a6970ee00000, duration=153] at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:423) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackNearTxLocalAsync(GridNearTxLocal.java:3310) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2411) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2393) at org.apache.ignite.internal.util.lang.IgniteClosureX.apply(IgniteClosureX.java:38) at org.apache.ignite.internal.util.future.GridFutureChainListener.applyCallback(GridFutureChainListener.java:78) at org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:70) at org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:30) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) at org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture.onDone(GridCacheCompoundIdentityFuture.java:56) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:340) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:69) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) at org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onResponse(GridNearTxFinishFuture.java:1124) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onDhtFinishResponse(GridNearTxFinishFuture.java:1105) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onResult(GridNearTxFinishFuture.java:260) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processDhtTxFinishResponse(IgniteTxHandler.java:741) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$800(IgniteTxHandler.java:97) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:225) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:223) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1060) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99) at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293) at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1555) at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1183) at org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:126) at org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1090) at org.apache.ignite.internal.util.StripedExecutor$Stripe.run(StripedExecutor.java:505) at java.lang.Thread.run(Thread.java:745) {code} And as I have invistigated we call onError and rollback two times. First will change state to ROLLED_BACK (finish state) and rollback would fail if it was called for such transaction. Stack trace of calling onDone with commit = false: {code} java.lang.Throwable: disco-event-worker-#37%persistence.IgnitePdsContinuousRestartTestWithSharedGroup0% at org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter.state(IgniteTxAdapter.java:977) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter.state(IgniteTxAdapter.java:935) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.localFinish(GridNearTxLocal.java:3107) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:408) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$16.apply(GridNearTxLocal.java:3238) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$16.apply(GridNearTxLocal.java:3217) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) at org.apache.ignite.internal.processors.cache.GridCacheCompoundFuture.onDone(GridCacheCompoundFuture.java:56) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture.onComplete(GridNearOptimisticTxPrepareFuture.java:305) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture.onError(GridNearOptimisticTxPrepareFuture.java:181) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture$MiniFuture.onNodeLeft(GridNearOptimisticTxPrepareFuture.java:903) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture.onNodeLeft(GridNearOptimisticTxPrepareFuture.java:128) at org.apache.ignite.internal.processors.cache.GridCacheMvccManager$4.onEvent(GridCacheMvccManager.java:263) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager$LocalListenerWrapper.onEvent(GridEventStorageManager.java:1384) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:873) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:858) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record0(GridEventStorageManager.java:341) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record(GridEventStorageManager.java:307) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.recordEvent(GridDiscoveryManager.java:2483) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscoveryManager.java:2689) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryManager.java:2512) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110) at java.lang.Thread.run(Thread.java:745) {code} Stack trace of calling rollback: {code} "sys-stripe-7-#9%persistence.IgnitePdsContinuousRestartTestWithSharedGroup0%@1958" prio=5 tid=0x17 nid=NA runnable java.lang.Thread.State: RUNNABLE blocks exchange-worker-#38%persistence.IgnitePdsContinuousRestartTestWithSharedGroup0%@2755 blocks main@1 at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:426) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackNearTxLocalAsync(GridNearTxLocal.java:3300) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2401) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2383) at org.apache.ignite.internal.util.lang.IgniteClosureX.apply(IgniteClosureX.java:38) at org.apache.ignite.internal.util.future.GridFutureChainListener.applyCallback(GridFutureChainListener.java:78) at org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:70) at org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:30) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) at org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture.onDone(GridCacheCompoundIdentityFuture.java:56) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:341) - locked <0x1266> (a org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:70) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) at org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onResponse(GridNearTxFinishFuture.java:1127) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onDhtFinishResponse(GridNearTxFinishFuture.java:1108) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onResult(GridNearTxFinishFuture.java:261) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processDhtTxFinishResponse(IgniteTxHandler.java:741) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$800(IgniteTxHandler.java:97) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:225) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:223) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1060) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99) at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293) at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1555) at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1183) at org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:126) at org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1090) at org.apache.ignite.internal.util.StripedExecutor$Stripe.run(StripedExecutor.java:505) at java.lang.Thread.run(Thread.java:745) {code} I don't have good reproducer, but this fails some times with error: org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest#testRebalncingDuringLoad_10_10_1_1 was: I have found during stress-test (start-stop nodes during transactions running): {code} [2017-09-20 12:37:04,224][ERROR][updater-1][GridDhtColocatedCache] <cache1> Failed to rollback transaction (cache may contain stale locks): GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=null], nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=null, hasRemoteLocks=false, thread=updater-1, mappings=IgniteTxMappingsSingleImpl [mapping=null], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true, recovery=false], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=117380231, order=1505900296519, nodeOrder=1], writeVer=null, implicit=true, loc=true, threadId=15914, startTime=1505900224068, nodeId=2699eb85-b97a-431f-a038-a6970ee00000, startVer=GridCacheVersion [topVer=117380231, order=1505900296519, nodeOrder=1], endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=GridCacheVersion [topVer=117380231, order=1505900296519, nodeOrder=1], finalizing=NONE, invalidParts=null, state=ROLLED_BACK, timedOut=false, topVer=AffinityTopologyVersion [topVer=12, minorTopVer=0], duration=153ms, onePhaseCommit=false], size=1]]] class org.apache.ignite.IgniteCheckedException: Failed to commit transaction: GridNearTxLocal[id=749f6ae9e51-00000000-06ff-1487-0000-000000000001, concurrency=OPTIMISTIC, isolation=READ_COMMITTED, state=ROLLED_BACK, invalidate=false, rollbackOnly=true, nodeId=2699eb85-b97a-431f-a038-a6970ee00000, duration=153] at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:423) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackNearTxLocalAsync(GridNearTxLocal.java:3310) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2411) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2393) at org.apache.ignite.internal.util.lang.IgniteClosureX.apply(IgniteClosureX.java:38) at org.apache.ignite.internal.util.future.GridFutureChainListener.applyCallback(GridFutureChainListener.java:78) at org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:70) at org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:30) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) at org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture.onDone(GridCacheCompoundIdentityFuture.java:56) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:340) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:69) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) at org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onResponse(GridNearTxFinishFuture.java:1124) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onDhtFinishResponse(GridNearTxFinishFuture.java:1105) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onResult(GridNearTxFinishFuture.java:260) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processDhtTxFinishResponse(IgniteTxHandler.java:741) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$800(IgniteTxHandler.java:97) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:225) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:223) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1060) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99) at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293) at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1555) at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1183) at org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:126) at org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1090) at org.apache.ignite.internal.util.StripedExecutor$Stripe.run(StripedExecutor.java:505) at java.lang.Thread.run(Thread.java:745) {code} And as I have invistigated we call onError and rollback two times. First will change state to ROLLED_BACK (finish state) and rollback would fail if it was called for such transaction. Stack trace of calling onDone with commit = false: {code} java.lang.Throwable: disco-event-worker-#37%persistence.IgnitePdsContinuousRestartTestWithSharedGroup0% at org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter.state(IgniteTxAdapter.java:977) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter.state(IgniteTxAdapter.java:935) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.localFinish(GridNearTxLocal.java:3107) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:408) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$16.apply(GridNearTxLocal.java:3238) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$16.apply(GridNearTxLocal.java:3217) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) at org.apache.ignite.internal.processors.cache.GridCacheCompoundFuture.onDone(GridCacheCompoundFuture.java:56) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture.onComplete(GridNearOptimisticTxPrepareFuture.java:305) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture.onError(GridNearOptimisticTxPrepareFuture.java:181) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture$MiniFuture.onNodeLeft(GridNearOptimisticTxPrepareFuture.java:903) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture.onNodeLeft(GridNearOptimisticTxPrepareFuture.java:128) at org.apache.ignite.internal.processors.cache.GridCacheMvccManager$4.onEvent(GridCacheMvccManager.java:263) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager$LocalListenerWrapper.onEvent(GridEventStorageManager.java:1384) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:873) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:858) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record0(GridEventStorageManager.java:341) at org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record(GridEventStorageManager.java:307) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.recordEvent(GridDiscoveryManager.java:2483) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscoveryManager.java:2689) at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryManager.java:2512) at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110) at java.lang.Thread.run(Thread.java:745) {code} Stack trace of calling rollback: {code} "sys-stripe-7-#9%persistence.IgnitePdsContinuousRestartTestWithSharedGroup0%@1958" prio=5 tid=0x17 nid=NA runnable java.lang.Thread.State: RUNNABLE blocks exchange-worker-#38%persistence.IgnitePdsContinuousRestartTestWithSharedGroup0%@2755 blocks main@1 at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:426) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackNearTxLocalAsync(GridNearTxLocal.java:3300) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2401) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2383) at org.apache.ignite.internal.util.lang.IgniteClosureX.apply(IgniteClosureX.java:38) at org.apache.ignite.internal.util.future.GridFutureChainListener.applyCallback(GridFutureChainListener.java:78) at org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:70) at org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:30) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) at org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture.onDone(GridCacheCompoundIdentityFuture.java:56) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:341) - locked <0x1266> (a org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:70) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) at org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) at org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onResponse(GridNearTxFinishFuture.java:1127) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onDhtFinishResponse(GridNearTxFinishFuture.java:1108) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onResult(GridNearTxFinishFuture.java:261) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processDhtTxFinishResponse(IgniteTxHandler.java:741) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$800(IgniteTxHandler.java:97) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:225) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:223) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1060) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304) at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99) at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293) at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1555) at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1183) at org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:126) at org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1090) at org.apache.ignite.internal.util.StripedExecutor$Stripe.run(StripedExecutor.java:505) at java.lang.Thread.run(Thread.java:745) {code} > Races during transaction finalization > ------------------------------------- > > Key: IGNITE-6425 > URL: https://issues.apache.org/jira/browse/IGNITE-6425 > Project: Ignite > Issue Type: Bug > Reporter: Eduard Shangareev > Assignee: Eduard Shangareev > Priority: Critical > Fix For: 2.3 > > > I have found during stress-test (start-stop nodes during transactions > running): > {code} > [2017-09-20 12:37:04,224][ERROR][updater-1][GridDhtColocatedCache] <cache1> > Failed to rollback transaction (cache may contain stale locks): > GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=null], > nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=null, > hasRemoteLocks=false, thread=updater-1, mappings=IgniteTxMappingsSingleImpl > [mapping=null], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, > nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter > [completedBase=null, sndTransformedVals=false, depEnabled=false, > txState=IgniteTxImplicitSingleStateImpl [init=true, recovery=false], > super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=117380231, > order=1505900296519, nodeOrder=1], writeVer=null, implicit=true, loc=true, > threadId=15914, startTime=1505900224068, > nodeId=2699eb85-b97a-431f-a038-a6970ee00000, startVer=GridCacheVersion > [topVer=117380231, order=1505900296519, nodeOrder=1], endVer=null, > isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, > sysInvalidate=false, sys=false, plc=2, commitVer=GridCacheVersion > [topVer=117380231, order=1505900296519, nodeOrder=1], finalizing=NONE, > invalidParts=null, state=ROLLED_BACK, timedOut=false, > topVer=AffinityTopologyVersion [topVer=12, minorTopVer=0], duration=153ms, > onePhaseCommit=false], size=1]]] > class org.apache.ignite.IgniteCheckedException: Failed to commit transaction: > GridNearTxLocal[id=749f6ae9e51-00000000-06ff-1487-0000-000000000001, > concurrency=OPTIMISTIC, isolation=READ_COMMITTED, state=ROLLED_BACK, > invalidate=false, rollbackOnly=true, > nodeId=2699eb85-b97a-431f-a038-a6970ee00000, duration=153] > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:423) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackNearTxLocalAsync(GridNearTxLocal.java:3310) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2411) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2393) > at > org.apache.ignite.internal.util.lang.IgniteClosureX.apply(IgniteClosureX.java:38) > at > org.apache.ignite.internal.util.future.GridFutureChainListener.applyCallback(GridFutureChainListener.java:78) > at > org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:70) > at > org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:30) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) > at > org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture.onDone(GridCacheCompoundIdentityFuture.java:56) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:340) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:69) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onResponse(GridNearTxFinishFuture.java:1124) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onDhtFinishResponse(GridNearTxFinishFuture.java:1105) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onResult(GridNearTxFinishFuture.java:260) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processDhtTxFinishResponse(IgniteTxHandler.java:741) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$800(IgniteTxHandler.java:97) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:225) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:223) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1060) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293) > at > org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1555) > at > org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1183) > at > org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:126) > at > org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1090) > at > org.apache.ignite.internal.util.StripedExecutor$Stripe.run(StripedExecutor.java:505) > at java.lang.Thread.run(Thread.java:745) > {code} > And as I have invistigated we call onError and rollback two times. First will > change state to ROLLED_BACK (finish state) and rollback would fail if it was > called for such transaction. > Stack trace of calling onDone with commit = false: > {code} > java.lang.Throwable: > disco-event-worker-#37%persistence.IgnitePdsContinuousRestartTestWithSharedGroup0% > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter.state(IgniteTxAdapter.java:977) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter.state(IgniteTxAdapter.java:935) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.localFinish(GridNearTxLocal.java:3107) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:408) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$16.apply(GridNearTxLocal.java:3238) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$16.apply(GridNearTxLocal.java:3217) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) > at > org.apache.ignite.internal.processors.cache.GridCacheCompoundFuture.onDone(GridCacheCompoundFuture.java:56) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture.onComplete(GridNearOptimisticTxPrepareFuture.java:305) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture.onError(GridNearOptimisticTxPrepareFuture.java:181) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture$MiniFuture.onNodeLeft(GridNearOptimisticTxPrepareFuture.java:903) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture.onNodeLeft(GridNearOptimisticTxPrepareFuture.java:128) > at > org.apache.ignite.internal.processors.cache.GridCacheMvccManager$4.onEvent(GridCacheMvccManager.java:263) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager$LocalListenerWrapper.onEvent(GridEventStorageManager.java:1384) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:873) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:858) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record0(GridEventStorageManager.java:341) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record(GridEventStorageManager.java:307) > at > org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.recordEvent(GridDiscoveryManager.java:2483) > at > org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscoveryManager.java:2689) > at > org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryManager.java:2512) > at > org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110) > at java.lang.Thread.run(Thread.java:745) > {code} > Stack trace of calling rollback: > {code} > "sys-stripe-7-#9%persistence.IgnitePdsContinuousRestartTestWithSharedGroup0%@1958" > prio=5 tid=0x17 nid=NA runnable > java.lang.Thread.State: RUNNABLE > blocks > exchange-worker-#38%persistence.IgnitePdsContinuousRestartTestWithSharedGroup0%@2755 > blocks main@1 > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:426) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackNearTxLocalAsync(GridNearTxLocal.java:3300) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2401) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2383) > at > org.apache.ignite.internal.util.lang.IgniteClosureX.apply(IgniteClosureX.java:38) > at > org.apache.ignite.internal.util.future.GridFutureChainListener.applyCallback(GridFutureChainListener.java:78) > at > org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:70) > at > org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:30) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) > at > org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture.onDone(GridCacheCompoundIdentityFuture.java:56) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:341) > - locked <0x1266> (a > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:70) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onResponse(GridNearTxFinishFuture.java:1127) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onDhtFinishResponse(GridNearTxFinishFuture.java:1108) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onResult(GridNearTxFinishFuture.java:261) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processDhtTxFinishResponse(IgniteTxHandler.java:741) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$800(IgniteTxHandler.java:97) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:225) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:223) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1060) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293) > at > org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1555) > at > org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1183) > at > org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:126) > at > org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1090) > at > org.apache.ignite.internal.util.StripedExecutor$Stripe.run(StripedExecutor.java:505) > at java.lang.Thread.run(Thread.java:745) > {code} > I don't have good reproducer, but this fails some times with error: > org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest#testRebalncingDuringLoad_10_10_1_1 -- This message was sent by Atlassian JIRA (v6.4.14#64029)