[ https://issues.apache.org/jira/browse/IGNITE-3142?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Vladislav Pyatkov updated IGNITE-3142: -------------------------------------- Description: If you try to start one server node and another then. After update of topology, you can stop the first node and get the exception: {noformat} [12:25:25] (err) Failed to execute compound future reducer: GridNearTxFinishFuture [futId=fbe41e8b451-b79beebc-da21-4cfc-a83e-e1cb4b34151b, tx=GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=false, hasRemoteLocks=false, thread=sys-#20%null%, mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], writeVer=null, implicit=true, loc=true, threadId=32, startTime=1463390725843, nodeId=df815072-615a-49d4-92e1-2842369ea15a, startVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, sysInvalidate=false, sys=true, plc=5, commitVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], finalizing=NONE, preparing=false, invalidParts=null, state=PREPARED, timedOut=false, topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], duration=50ms, onePhaseCommit=true], size=1]]], commit=true, mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], trackable=true, finishOnePhaseCalled=false, innerFuts=[CheckBackupFuture[node=df815072-615a-49d4-92e1-2842369ea15a, loc=true, done=true]], super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=o.a.i.i.util.lang.GridFunc$27@1b01b273, initFlag=0, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[true]]]]class org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException: Failed to commit transaction (transaction has been rolled back on backup node): GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3] at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:450) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:347) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:860) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:852) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) at org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:226) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.commitAsync(GridNearTxLocal.java:852) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.optimisticPutFuture(IgniteTxLocalAdapter.java:3423) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.putAsync0(IgniteTxLocalAdapter.java:3211) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.invokeAsync(IgniteTxLocalAdapter.java:2097) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2299) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4210) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke0(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke(GridCacheAdapter.java:2262) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1575) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:566) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$2.addMeta(CacheObjectBinaryProcessorImpl.java:204) at org.apache.ignite.internal.binary.BinaryContext.registerUserClassDescriptor(BinaryContext.java:711) at org.apache.ignite.internal.binary.BinaryContext.registerClassDescriptor(BinaryContext.java:663) at org.apache.ignite.internal.binary.BinaryContext.descriptorForClass(BinaryContext.java:563) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:145) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:132) at org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:239) at org.apache.ignite.internal.binary.BinaryMarshaller.marshal(BinaryMarshaller.java:92) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1216) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1197) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1179) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:840) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:799) at org.apache.ignite.internal.GridEventConsumeHandler$2$1.run(GridEventConsumeHandler.java:231) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: class org.apache.ignite.internal.cluster.ClusterTopologyCheckedException: Primary node left grid: 5f3be800-b89a-419b-8d7e-ba5adbb36fab at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:444) ... 35 more 2016-05-16 12:25:25 ERROR GridNearTxFinishFuture:495 - Failed to automatically rollback transaction: GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=false, hasRemoteLocks=false, thread=sys-#20%null%, mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], writeVer=null, implicit=true, loc=true, threadId=32, startTime=1463390725843, nodeId=df815072-615a-49d4-92e1-2842369ea15a, startVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, sysInvalidate=false, sys=true, plc=5, commitVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], finalizing=NONE, preparing=false, invalidParts=null, state=ROLLED_BACK, timedOut=false, topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], duration=70ms, onePhaseCommit=true], size=1]]] class org.apache.ignite.IgniteCheckedException: Invalid transaction state for commit [state=ROLLED_BACK, tx=GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=false, hasRemoteLocks=false, thread=sys-#20%null%, mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], writeVer=null, implicit=true, loc=true, threadId=32, startTime=1463390725843, nodeId=df815072-615a-49d4-92e1-2842369ea15a, startVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, sysInvalidate=false, sys=true, plc=5, commitVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], finalizing=NONE, preparing=false, invalidParts=null, state=ROLLED_BACK, timedOut=false, topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], duration=60ms, onePhaseCommit=true], size=1]]]] at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.finish(GridNearTxLocal.java:722) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:276) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:67) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:324) at org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:241) at org.apache.ignite.internal.util.future.GridCompoundFuture.markInitialized(GridCompoundFuture.java:232) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:369) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackAsync(GridNearTxLocal.java:920) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.rollback(GridDhtTxLocalAdapter.java:859) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:251) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:67) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:335) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:118) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:41) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListeners(GridFutureAdapter.java:251) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:381) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:347) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:335) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:449) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:347) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:860) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:852) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) at org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:226) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.commitAsync(GridNearTxLocal.java:852) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.optimisticPutFuture(IgniteTxLocalAdapter.java:3423) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.putAsync0(IgniteTxLocalAdapter.java:3211) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.invokeAsync(IgniteTxLocalAdapter.java:2097) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2299) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4210) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke0(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke(GridCacheAdapter.java:2262) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1575) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:566) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$2.addMeta(CacheObjectBinaryProcessorImpl.java:204) at org.apache.ignite.internal.binary.BinaryContext.registerUserClassDescriptor(BinaryContext.java:711) at org.apache.ignite.internal.binary.BinaryContext.registerClassDescriptor(BinaryContext.java:663) at org.apache.ignite.internal.binary.BinaryContext.descriptorForClass(BinaryContext.java:563) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:145) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:132) at org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:239) at org.apache.ignite.internal.binary.BinaryMarshaller.marshal(BinaryMarshaller.java:92) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1216) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1197) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1179) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:840) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:799) at org.apache.ignite.internal.GridEventConsumeHandler$2$1.run(GridEventConsumeHandler.java:231) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2016-05-16 12:25:25 ERROR GridEventConsumeHandler:495 - Failed to send event notification to node: 28b7b448-d0bb-404b-9643-abf87ce27fad class org.apache.ignite.binary.BinaryObjectException: Failed to update meta data for type: org.jsr166.ConcurrentLinkedDeque8 at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:572) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$2.addMeta(CacheObjectBinaryProcessorImpl.java:204) at org.apache.ignite.internal.binary.BinaryContext.registerUserClassDescriptor(BinaryContext.java:711) at org.apache.ignite.internal.binary.BinaryContext.registerClassDescriptor(BinaryContext.java:663) at org.apache.ignite.internal.binary.BinaryContext.descriptorForClass(BinaryContext.java:563) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:145) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:132) at org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:239) at org.apache.ignite.internal.binary.BinaryMarshaller.marshal(BinaryMarshaller.java:92) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1216) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1197) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1179) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:840) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:799) at org.apache.ignite.internal.GridEventConsumeHandler$2$1.run(GridEventConsumeHandler.java:231) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: javax.cache.CacheException: class org.apache.ignite.transactions.TransactionRollbackException: Failed to commit transaction (transaction has been rolled back on backup node): GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3] at org.apache.ignite.internal.processors.cache.GridCacheUtils.convertToCacheException(GridCacheUtils.java:1478) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.cacheException(IgniteCacheProxy.java:1944) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1585) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:566) ... 19 more Caused by: class org.apache.ignite.transactions.TransactionRollbackException: Failed to commit transaction (transaction has been rolled back on backup node): GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3] at org.apache.ignite.internal.util.IgniteUtils$10.apply(IgniteUtils.java:800) at org.apache.ignite.internal.util.IgniteUtils$10.apply(IgniteUtils.java:798) ... 23 more Caused by: class org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException: Failed to commit transaction (transaction has been rolled back on backup node): GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3] at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:450) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:347) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:860) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:852) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) at org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:226) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.commitAsync(GridNearTxLocal.java:852) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.optimisticPutFuture(IgniteTxLocalAdapter.java:3423) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.putAsync0(IgniteTxLocalAdapter.java:3211) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.invokeAsync(IgniteTxLocalAdapter.java:2097) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2299) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4210) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke0(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke(GridCacheAdapter.java:2262) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1575) ... 20 more Caused by: class org.apache.ignite.internal.cluster.ClusterTopologyCheckedException: Primary node left grid: 5f3be800-b89a-419b-8d7e-ba5adbb36fab at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:444) ... 35 more {noformat} Other node does not get event NODE_FAILED or EVT_NODE_LEFT. *Reproduction steps:* # Start first server node. # Start client. # Start second server node. # Stop first server node. was: If you try to start one server node and another then. After update of topology, you can stop the first node and get the exception: {noformat} [12:25:25] (err) Failed to execute compound future reducer: GridNearTxFinishFuture [futId=fbe41e8b451-b79beebc-da21-4cfc-a83e-e1cb4b34151b, tx=GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=false, hasRemoteLocks=false, thread=sys-#20%null%, mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], writeVer=null, implicit=true, loc=true, threadId=32, startTime=1463390725843, nodeId=df815072-615a-49d4-92e1-2842369ea15a, startVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, sysInvalidate=false, sys=true, plc=5, commitVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], finalizing=NONE, preparing=false, invalidParts=null, state=PREPARED, timedOut=false, topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], duration=50ms, onePhaseCommit=true], size=1]]], commit=true, mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], trackable=true, finishOnePhaseCalled=false, innerFuts=[CheckBackupFuture[node=df815072-615a-49d4-92e1-2842369ea15a, loc=true, done=true]], super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=o.a.i.i.util.lang.GridFunc$27@1b01b273, initFlag=0, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[true]]]]class org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException: Failed to commit transaction (transaction has been rolled back on backup node): GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3] at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:450) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:347) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:860) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:852) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) at org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:226) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.commitAsync(GridNearTxLocal.java:852) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.optimisticPutFuture(IgniteTxLocalAdapter.java:3423) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.putAsync0(IgniteTxLocalAdapter.java:3211) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.invokeAsync(IgniteTxLocalAdapter.java:2097) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2299) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4210) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke0(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke(GridCacheAdapter.java:2262) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1575) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:566) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$2.addMeta(CacheObjectBinaryProcessorImpl.java:204) at org.apache.ignite.internal.binary.BinaryContext.registerUserClassDescriptor(BinaryContext.java:711) at org.apache.ignite.internal.binary.BinaryContext.registerClassDescriptor(BinaryContext.java:663) at org.apache.ignite.internal.binary.BinaryContext.descriptorForClass(BinaryContext.java:563) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:145) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:132) at org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:239) at org.apache.ignite.internal.binary.BinaryMarshaller.marshal(BinaryMarshaller.java:92) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1216) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1197) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1179) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:840) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:799) at org.apache.ignite.internal.GridEventConsumeHandler$2$1.run(GridEventConsumeHandler.java:231) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: class org.apache.ignite.internal.cluster.ClusterTopologyCheckedException: Primary node left grid: 5f3be800-b89a-419b-8d7e-ba5adbb36fab at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:444) ... 35 more 2016-05-16 12:25:25 ERROR GridNearTxFinishFuture:495 - Failed to automatically rollback transaction: GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=false, hasRemoteLocks=false, thread=sys-#20%null%, mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], writeVer=null, implicit=true, loc=true, threadId=32, startTime=1463390725843, nodeId=df815072-615a-49d4-92e1-2842369ea15a, startVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, sysInvalidate=false, sys=true, plc=5, commitVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], finalizing=NONE, preparing=false, invalidParts=null, state=ROLLED_BACK, timedOut=false, topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], duration=70ms, onePhaseCommit=true], size=1]]] class org.apache.ignite.IgniteCheckedException: Invalid transaction state for commit [state=ROLLED_BACK, tx=GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=false, hasRemoteLocks=false, thread=sys-#20%null%, mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], writeVer=null, implicit=true, loc=true, threadId=32, startTime=1463390725843, nodeId=df815072-615a-49d4-92e1-2842369ea15a, startVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, sysInvalidate=false, sys=true, plc=5, commitVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], finalizing=NONE, preparing=false, invalidParts=null, state=ROLLED_BACK, timedOut=false, topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], duration=60ms, onePhaseCommit=true], size=1]]]] at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.finish(GridNearTxLocal.java:722) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:276) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:67) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:324) at org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:241) at org.apache.ignite.internal.util.future.GridCompoundFuture.markInitialized(GridCompoundFuture.java:232) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:369) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackAsync(GridNearTxLocal.java:920) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.rollback(GridDhtTxLocalAdapter.java:859) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:251) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:67) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:335) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:118) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:41) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListeners(GridFutureAdapter.java:251) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:381) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:347) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:335) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:449) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:347) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:860) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:852) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) at org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:226) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.commitAsync(GridNearTxLocal.java:852) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.optimisticPutFuture(IgniteTxLocalAdapter.java:3423) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.putAsync0(IgniteTxLocalAdapter.java:3211) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.invokeAsync(IgniteTxLocalAdapter.java:2097) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2299) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4210) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke0(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke(GridCacheAdapter.java:2262) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1575) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:566) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$2.addMeta(CacheObjectBinaryProcessorImpl.java:204) at org.apache.ignite.internal.binary.BinaryContext.registerUserClassDescriptor(BinaryContext.java:711) at org.apache.ignite.internal.binary.BinaryContext.registerClassDescriptor(BinaryContext.java:663) at org.apache.ignite.internal.binary.BinaryContext.descriptorForClass(BinaryContext.java:563) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:145) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:132) at org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:239) at org.apache.ignite.internal.binary.BinaryMarshaller.marshal(BinaryMarshaller.java:92) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1216) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1197) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1179) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:840) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:799) at org.apache.ignite.internal.GridEventConsumeHandler$2$1.run(GridEventConsumeHandler.java:231) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2016-05-16 12:25:25 ERROR GridEventConsumeHandler:495 - Failed to send event notification to node: 28b7b448-d0bb-404b-9643-abf87ce27fad class org.apache.ignite.binary.BinaryObjectException: Failed to update meta data for type: org.jsr166.ConcurrentLinkedDeque8 at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:572) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$2.addMeta(CacheObjectBinaryProcessorImpl.java:204) at org.apache.ignite.internal.binary.BinaryContext.registerUserClassDescriptor(BinaryContext.java:711) at org.apache.ignite.internal.binary.BinaryContext.registerClassDescriptor(BinaryContext.java:663) at org.apache.ignite.internal.binary.BinaryContext.descriptorForClass(BinaryContext.java:563) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:145) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:132) at org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:239) at org.apache.ignite.internal.binary.BinaryMarshaller.marshal(BinaryMarshaller.java:92) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1216) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1197) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1179) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:840) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:799) at org.apache.ignite.internal.GridEventConsumeHandler$2$1.run(GridEventConsumeHandler.java:231) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: javax.cache.CacheException: class org.apache.ignite.transactions.TransactionRollbackException: Failed to commit transaction (transaction has been rolled back on backup node): GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3] at org.apache.ignite.internal.processors.cache.GridCacheUtils.convertToCacheException(GridCacheUtils.java:1478) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.cacheException(IgniteCacheProxy.java:1944) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1585) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:566) ... 19 more Caused by: class org.apache.ignite.transactions.TransactionRollbackException: Failed to commit transaction (transaction has been rolled back on backup node): GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3] at org.apache.ignite.internal.util.IgniteUtils$10.apply(IgniteUtils.java:800) at org.apache.ignite.internal.util.IgniteUtils$10.apply(IgniteUtils.java:798) ... 23 more Caused by: class org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException: Failed to commit transaction (transaction has been rolled back on backup node): GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3] at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:450) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:347) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:860) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:852) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) at org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:226) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.commitAsync(GridNearTxLocal.java:852) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.optimisticPutFuture(IgniteTxLocalAdapter.java:3423) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.putAsync0(IgniteTxLocalAdapter.java:3211) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.invokeAsync(IgniteTxLocalAdapter.java:2097) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2299) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4210) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke0(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke(GridCacheAdapter.java:2262) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1575) ... 20 more Caused by: class org.apache.ignite.internal.cluster.ClusterTopologyCheckedException: Primary node left grid: 5f3be800-b89a-419b-8d7e-ba5adbb36fab at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:444) ... 35 more {noformat} Other node does not get event NODE_FAILED or EVT_NODE_LEFT. > When remote filter is set, event is not send, if primary node has left grid. > ---------------------------------------------------------------------------- > > Key: IGNITE-3142 > URL: https://issues.apache.org/jira/browse/IGNITE-3142 > Project: Ignite > Issue Type: Bug > Reporter: Vladislav Pyatkov > Attachments: Ignite-Events - Copy.zip > > > If you try to start one server node and another then. After update of > topology, you can stop the first node and get the exception: > {noformat} > [12:25:25] (err) Failed to execute compound future reducer: > GridNearTxFinishFuture > [futId=fbe41e8b451-b79beebc-da21-4cfc-a83e-e1cb4b34151b, tx=GridNearTxLocal > [mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping > [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, > txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], cacheId=-2100569601], > val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], > entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor > [newMeta=BinaryMetadata [typeId=887264644, > typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, > affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], > ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, > dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, > entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry > [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, > ver=GridCacheVersion [topVer=74870619, time=1463390725860, > order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], > prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, > locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, > partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, > time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, > dhtVer=null, last=false, near=false, clientFirst=false, > node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], nearLocallyMapped=false, > colocatedLocallyMapped=false, needCheckBackup=false, hasRemoteLocks=false, > thread=sys-#20%null%, mappings=IgniteTxMappingsSingleImpl > [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry > [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], > hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey > [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], > hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], > prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple > [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, > typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, > affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], > ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, > dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, > entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry > [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, > ver=GridCacheVersion [topVer=74870619, time=1463390725860, > order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], > prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, > locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, > partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, > time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, > dhtVer=null, last=false, near=false, clientFirst=false, > node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], super=GridDhtTxLocalAdapter > [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, > super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, > depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true], > super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=74870619, > time=1463390725845, order=1463390656380, nodeOrder=3], writeVer=null, > implicit=true, loc=true, threadId=32, startTime=1463390725843, > nodeId=df815072-615a-49d4-92e1-2842369ea15a, startVer=GridCacheVersion > [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], > endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, > sysInvalidate=false, sys=true, plc=5, commitVer=GridCacheVersion > [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], > finalizing=NONE, preparing=false, invalidParts=null, state=PREPARED, > timedOut=false, topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], > duration=50ms, onePhaseCommit=true], size=1]]], commit=true, > mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping > [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, > txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], cacheId=-2100569601], > val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], > entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor > [newMeta=BinaryMetadata [typeId=887264644, > typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, > affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], > ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, > dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, > entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry > [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, > ver=GridCacheVersion [topVer=74870619, time=1463390725860, > order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], > prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, > locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, > partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, > time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, > dhtVer=null, last=false, near=false, clientFirst=false, > node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], trackable=true, > finishOnePhaseCalled=false, > innerFuts=[CheckBackupFuture[node=df815072-615a-49d4-92e1-2842369ea15a, > loc=true, done=true]], super=GridCompoundIdentityFuture > [super=GridCompoundFuture [rdc=o.a.i.i.util.lang.GridFunc$27@1b01b273, > initFlag=0, lsnrCalls=0, done=false, cancelled=false, err=null, > futs=[true]]]]class > org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException: > Failed to commit transaction (transaction has been rolled back on backup > node): GridCacheVersion [topVer=74870619, time=1463390725845, > order=1463390656380, nodeOrder=3] > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:450) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:347) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:860) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:852) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:226) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.commitAsync(GridNearTxLocal.java:852) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.optimisticPutFuture(IgniteTxLocalAdapter.java:3423) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.putAsync0(IgniteTxLocalAdapter.java:3211) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.invokeAsync(IgniteTxLocalAdapter.java:2097) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2299) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2291) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4210) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke0(GridCacheAdapter.java:2291) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke(GridCacheAdapter.java:2262) > at > org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1575) > at > org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:566) > at > org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$2.addMeta(CacheObjectBinaryProcessorImpl.java:204) > at > org.apache.ignite.internal.binary.BinaryContext.registerUserClassDescriptor(BinaryContext.java:711) > at > org.apache.ignite.internal.binary.BinaryContext.registerClassDescriptor(BinaryContext.java:663) > at > org.apache.ignite.internal.binary.BinaryContext.descriptorForClass(BinaryContext.java:563) > at > org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:145) > at > org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:132) > at > org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:239) > at > org.apache.ignite.internal.binary.BinaryMarshaller.marshal(BinaryMarshaller.java:92) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1216) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1197) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1179) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:840) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:799) > at > org.apache.ignite.internal.GridEventConsumeHandler$2$1.run(GridEventConsumeHandler.java:231) > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Caused by: class > org.apache.ignite.internal.cluster.ClusterTopologyCheckedException: Primary > node left grid: 5f3be800-b89a-419b-8d7e-ba5adbb36fab > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:444) > ... 35 more > 2016-05-16 12:25:25 ERROR GridNearTxFinishFuture:495 - Failed to > automatically rollback transaction: GridNearTxLocal > [mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping > [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, > txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], cacheId=-2100569601], > val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], > entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor > [newMeta=BinaryMetadata [typeId=887264644, > typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, > affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], > ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, > dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, > entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry > [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, > ver=GridCacheVersion [topVer=74870619, time=1463390725860, > order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], > prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, > locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, > partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, > time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, > dhtVer=null, last=false, near=false, clientFirst=false, > node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], nearLocallyMapped=false, > colocatedLocallyMapped=false, needCheckBackup=false, hasRemoteLocks=false, > thread=sys-#20%null%, mappings=IgniteTxMappingsSingleImpl > [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry > [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], > hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey > [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], > hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], > prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple > [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, > typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, > affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], > ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, > dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, > entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry > [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, > ver=GridCacheVersion [topVer=74870619, time=1463390725860, > order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], > prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, > locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, > partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, > time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, > dhtVer=null, last=false, near=false, clientFirst=false, > node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], super=GridDhtTxLocalAdapter > [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, > super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, > depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true], > super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=74870619, > time=1463390725845, order=1463390656380, nodeOrder=3], writeVer=null, > implicit=true, loc=true, threadId=32, startTime=1463390725843, > nodeId=df815072-615a-49d4-92e1-2842369ea15a, startVer=GridCacheVersion > [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], > endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, > sysInvalidate=false, sys=true, plc=5, commitVer=GridCacheVersion > [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], > finalizing=NONE, preparing=false, invalidParts=null, state=ROLLED_BACK, > timedOut=false, topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], > duration=70ms, onePhaseCommit=true], size=1]]] > class org.apache.ignite.IgniteCheckedException: Invalid transaction state for > commit [state=ROLLED_BACK, tx=GridNearTxLocal > [mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping > [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, > txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], cacheId=-2100569601], > val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], > entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor > [newMeta=BinaryMetadata [typeId=887264644, > typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, > affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], > ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, > dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, > entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry > [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, > ver=GridCacheVersion [topVer=74870619, time=1463390725860, > order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], > prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, > locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, > partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, > time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, > dhtVer=null, last=false, near=false, clientFirst=false, > node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], nearLocallyMapped=false, > colocatedLocallyMapped=false, needCheckBackup=false, hasRemoteLocks=false, > thread=sys-#20%null%, mappings=IgniteTxMappingsSingleImpl > [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry > [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], > hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey > [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], > hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], > prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple > [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, > typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, > affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], > ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, > dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, > entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry > [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey > [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, > ver=GridCacheVersion [topVer=74870619, time=1463390725860, > order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], > prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, > locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, > partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, > time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, > dhtVer=null, last=false, near=false, clientFirst=false, > node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], super=GridDhtTxLocalAdapter > [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, > super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, > depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true], > super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=74870619, > time=1463390725845, order=1463390656380, nodeOrder=3], writeVer=null, > implicit=true, loc=true, threadId=32, startTime=1463390725843, > nodeId=df815072-615a-49d4-92e1-2842369ea15a, startVer=GridCacheVersion > [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], > endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, > sysInvalidate=false, sys=true, plc=5, commitVer=GridCacheVersion > [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], > finalizing=NONE, preparing=false, invalidParts=null, state=ROLLED_BACK, > timedOut=false, topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], > duration=60ms, onePhaseCommit=true], size=1]]]] > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.finish(GridNearTxLocal.java:722) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:276) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:67) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:324) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:241) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.markInitialized(GridCompoundFuture.java:232) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:369) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackAsync(GridNearTxLocal.java:920) > at > org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.rollback(GridDhtTxLocalAdapter.java:859) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:251) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:67) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:335) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:118) > at > org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:41) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListeners(GridFutureAdapter.java:251) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:381) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:347) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:335) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:449) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:347) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:860) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:852) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:226) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.commitAsync(GridNearTxLocal.java:852) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.optimisticPutFuture(IgniteTxLocalAdapter.java:3423) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.putAsync0(IgniteTxLocalAdapter.java:3211) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.invokeAsync(IgniteTxLocalAdapter.java:2097) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2299) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2291) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4210) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke0(GridCacheAdapter.java:2291) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke(GridCacheAdapter.java:2262) > at > org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1575) > at > org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:566) > at > org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$2.addMeta(CacheObjectBinaryProcessorImpl.java:204) > at > org.apache.ignite.internal.binary.BinaryContext.registerUserClassDescriptor(BinaryContext.java:711) > at > org.apache.ignite.internal.binary.BinaryContext.registerClassDescriptor(BinaryContext.java:663) > at > org.apache.ignite.internal.binary.BinaryContext.descriptorForClass(BinaryContext.java:563) > at > org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:145) > at > org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:132) > at > org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:239) > at > org.apache.ignite.internal.binary.BinaryMarshaller.marshal(BinaryMarshaller.java:92) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1216) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1197) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1179) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:840) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:799) > at > org.apache.ignite.internal.GridEventConsumeHandler$2$1.run(GridEventConsumeHandler.java:231) > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > 2016-05-16 12:25:25 ERROR GridEventConsumeHandler:495 - Failed to send event > notification to node: 28b7b448-d0bb-404b-9643-abf87ce27fad > class org.apache.ignite.binary.BinaryObjectException: Failed to update meta > data for type: org.jsr166.ConcurrentLinkedDeque8 > at > org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:572) > at > org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$2.addMeta(CacheObjectBinaryProcessorImpl.java:204) > at > org.apache.ignite.internal.binary.BinaryContext.registerUserClassDescriptor(BinaryContext.java:711) > at > org.apache.ignite.internal.binary.BinaryContext.registerClassDescriptor(BinaryContext.java:663) > at > org.apache.ignite.internal.binary.BinaryContext.descriptorForClass(BinaryContext.java:563) > at > org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:145) > at > org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:132) > at > org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:239) > at > org.apache.ignite.internal.binary.BinaryMarshaller.marshal(BinaryMarshaller.java:92) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1216) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1197) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1179) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:840) > at > org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:799) > at > org.apache.ignite.internal.GridEventConsumeHandler$2$1.run(GridEventConsumeHandler.java:231) > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Caused by: javax.cache.CacheException: class > org.apache.ignite.transactions.TransactionRollbackException: Failed to commit > transaction (transaction has been rolled back on backup node): > GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, > nodeOrder=3] > at > org.apache.ignite.internal.processors.cache.GridCacheUtils.convertToCacheException(GridCacheUtils.java:1478) > at > org.apache.ignite.internal.processors.cache.IgniteCacheProxy.cacheException(IgniteCacheProxy.java:1944) > at > org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1585) > at > org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:566) > ... 19 more > Caused by: class org.apache.ignite.transactions.TransactionRollbackException: > Failed to commit transaction (transaction has been rolled back on backup > node): GridCacheVersion [topVer=74870619, time=1463390725845, > order=1463390656380, nodeOrder=3] > at > org.apache.ignite.internal.util.IgniteUtils$10.apply(IgniteUtils.java:800) > at > org.apache.ignite.internal.util.IgniteUtils$10.apply(IgniteUtils.java:798) > ... 23 more > Caused by: class > org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException: > Failed to commit transaction (transaction has been rolled back on backup > node): GridCacheVersion [topVer=74870619, time=1463390725845, > order=1463390656380, nodeOrder=3] > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:450) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:347) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:860) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:852) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) > at > org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:226) > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.commitAsync(GridNearTxLocal.java:852) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.optimisticPutFuture(IgniteTxLocalAdapter.java:3423) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.putAsync0(IgniteTxLocalAdapter.java:3211) > at > org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.invokeAsync(IgniteTxLocalAdapter.java:2097) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2299) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2291) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4210) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke0(GridCacheAdapter.java:2291) > at > org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke(GridCacheAdapter.java:2262) > at > org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1575) > ... 20 more > Caused by: class > org.apache.ignite.internal.cluster.ClusterTopologyCheckedException: Primary > node left grid: 5f3be800-b89a-419b-8d7e-ba5adbb36fab > at > org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:444) > ... 35 more > {noformat} > Other node does not get event NODE_FAILED or EVT_NODE_LEFT. > *Reproduction steps:* > # Start first server node. > # Start client. > # Start second server node. > # Stop first server node. -- This message was sent by Atlassian JIRA (v6.3.4#6332)