[ https://issues.apache.org/jira/browse/IGNITE-6425?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Eduard Shangareev updated IGNITE-6425: -------------------------------------- Fix Version/s: 2.3 > Races during transaction finalization > ------------------------------------- > > Key: IGNITE-6425 > URL: https://issues.apache.org/jira/browse/IGNITE-6425 > Project: Ignite > Issue Type: Bug > Reporter: Eduard Shangareev > Assignee: Eduard Shangareev > Fix For: 2.3 > > > I have found during stress-test (start-stop nodes during transactions > running): > {code} > [2017-09-20 12:37:04,224][ERROR][updater-1][GridDhtColocatedCache] <cache1> > Failed to rollback transaction (cache may contain stale locks): > GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=null], > nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=null, > hasRemoteLocks=false, thread=updater-1, mappings=IgniteTxMappingsSingleImpl > [mapping=null], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, > nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter > [completedBase=null, sndTransformedVals=false, depEnabled=false, > txState=IgniteTxImplicitSingleStateImpl [init=true, recovery=false], > super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=117380231, > order=1505900296519, nodeOrder=1], writeVer=null, implicit=true, loc=true, > threadId=15914, startTime=1505900224068, > nodeId=2699eb85-b97a-431f-a038-a6970ee00000, startVer=GridCacheVersion > [topVer=117380231, order=1505900296519, nodeOrder=1], endVer=null, > isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, > sysInvalidate=false, sys=false, plc=2, commitVer=GridCacheVersion > [topVer=117380231, order=1505900296519, nodeOrder=1], finalizing=NONE, > invalidParts=null, state=ROLLED_BACK, timedOut=false, > topVer=AffinityTopologyVersion [topVer=12, minorTopVer=0], duration=153ms, > onePhaseCommit=false], size=1]]] > class org.apache.ignite.IgniteCheckedException: Failed to commit transaction: > GridNearTxLocal[id=749f6ae9e51-00000000-06ff-1487-0000-000000000001, > concurrency=OPTIMISTIC, isolation=READ_COMMITTED, state=ROLLED_BACK, > invalidate=false, rollbackOnly=true, > nodeId=2699eb85-b97a-431f-a038-a6970ee00000, duration=153] > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:423) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackNearTxLocalAsync(GridNearTxLocal.java:3310) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2411) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2393) > at > org.apache.ignite.internal.util.lang.IgniteClosureX.apply(IgniteClosureX.java:38) > at > org.apache.ignite.internal.util.future.GridFutureChainListener.applyCallback(GridFutureChainListener.java:78) > at > org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:70) > at > org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:30) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) > at > org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture.onDone(GridCacheCompoundIdentityFuture.java:56) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:340) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:69) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onResponse(GridNearTxFinishFuture.java:1124) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onDhtFinishResponse(GridNearTxFinishFuture.java:1105) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onResult(GridNearTxFinishFuture.java:260) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processDhtTxFinishResponse(IgniteTxHandler.java:741) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$800(IgniteTxHandler.java:97) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:225) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:223) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1060) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293) > at > org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1555) > at > org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1183) > at > org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:126) > at > org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1090) > at > org.apache.ignite.internal.util.StripedExecutor$Stripe.run(StripedExecutor.java:505) > at java.lang.Thread.run(Thread.java:745) > {code} > And as I have invistigated we call onError and rollback two times. First will > change state to ROLLED_BACK (finish state) and rollback would fail if it was > called for such transaction. > Stack trace of calling onDone with commit = false: > {code} > java.lang.Throwable: > disco-event-worker-#37%persistence.IgnitePdsContinuousRestartTestWithSharedGroup0% > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter.state(IgniteTxAdapter.java:977) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter.state(IgniteTxAdapter.java:935) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.localFinish(GridNearTxLocal.java:3107) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:408) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$16.apply(GridNearTxLocal.java:3238) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$16.apply(GridNearTxLocal.java:3217) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) > at > org.apache.ignite.internal.processors.cache.GridCacheCompoundFuture.onDone(GridCacheCompoundFuture.java:56) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture.onComplete(GridNearOptimisticTxPrepareFuture.java:305) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture.onError(GridNearOptimisticTxPrepareFuture.java:181) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture$MiniFuture.onNodeLeft(GridNearOptimisticTxPrepareFuture.java:903) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture.onNodeLeft(GridNearOptimisticTxPrepareFuture.java:128) > at > org.apache.ignite.internal.processors.cache.GridCacheMvccManager$4.onEvent(GridCacheMvccManager.java:263) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager$LocalListenerWrapper.onEvent(GridEventStorageManager.java:1384) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:873) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:858) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record0(GridEventStorageManager.java:341) > at > org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record(GridEventStorageManager.java:307) > at > org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.recordEvent(GridDiscoveryManager.java:2483) > at > org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body0(GridDiscoveryManager.java:2689) > at > org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryWorker.body(GridDiscoveryManager.java:2512) > at > org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110) > at java.lang.Thread.run(Thread.java:745) > {code} > Stack trace of calling rollback: > {code} > "sys-stripe-7-#9%persistence.IgnitePdsContinuousRestartTestWithSharedGroup0%@1958" > prio=5 tid=0x17 nid=NA runnable > java.lang.Thread.State: RUNNABLE > blocks > exchange-worker-#38%persistence.IgnitePdsContinuousRestartTestWithSharedGroup0%@2755 > blocks main@1 > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:426) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackNearTxLocalAsync(GridNearTxLocal.java:3300) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2401) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10.applyx(GridNearTxLocal.java:2383) > at > org.apache.ignite.internal.util.lang.IgniteClosureX.apply(IgniteClosureX.java:38) > at > org.apache.ignite.internal.util.future.GridFutureChainListener.applyCallback(GridFutureChainListener.java:78) > at > org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:70) > at > org.apache.ignite.internal.util.future.GridFutureChainListener.apply(GridFutureChainListener.java:30) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) > at > org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture.onDone(GridCacheCompoundIdentityFuture.java:56) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:341) > - locked <0x1266> (a > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:70) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:285) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:144) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:382) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:346) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:334) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:494) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:473) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:450) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onResponse(GridNearTxFinishFuture.java:1127) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$CheckRemoteTxMiniFuture.onDhtFinishResponse(GridNearTxFinishFuture.java:1108) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onResult(GridNearTxFinishFuture.java:261) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.processDhtTxFinishResponse(IgniteTxHandler.java:741) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler.access$800(IgniteTxHandler.java:97) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:225) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$9.apply(IgniteTxHandler.java:223) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1060) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:579) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:378) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:304) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:99) > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:293) > at > org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1555) > at > org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1183) > at > org.apache.ignite.internal.managers.communication.GridIoManager.access$4200(GridIoManager.java:126) > at > org.apache.ignite.internal.managers.communication.GridIoManager$9.run(GridIoManager.java:1090) > at > org.apache.ignite.internal.util.StripedExecutor$Stripe.run(StripedExecutor.java:505) > at java.lang.Thread.run(Thread.java:745) > {code} -- This message was sent by Atlassian JIRA (v6.4.14#64029)