Vladislav Pyatkov created IGNITE-3142: -----------------------------------------
Summary: Do not send event when primary node has left grid. Key: IGNITE-3142 URL: https://issues.apache.org/jira/browse/IGNITE-3142 Project: Ignite Issue Type: Bug Reporter: Vladislav Pyatkov If you try to start one server node and another then. After update of topology, you can stop the first node and get the exception: {noformat} [12:25:25] (err) Failed to execute compound future reducer: GridNearTxFinishFuture [futId=fbe41e8b451-b79beebc-da21-4cfc-a83e-e1cb4b34151b, tx=GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=false, hasRemoteLocks=false, thread=sys-#20%null%, mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], writeVer=null, implicit=true, loc=true, threadId=32, startTime=1463390725843, nodeId=df815072-615a-49d4-92e1-2842369ea15a, startVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, sysInvalidate=false, sys=true, plc=5, commitVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], finalizing=NONE, preparing=false, invalidParts=null, state=PREPARED, timedOut=false, topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], duration=50ms, onePhaseCommit=true], size=1]]], commit=true, mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], trackable=true, finishOnePhaseCalled=false, innerFuts=[CheckBackupFuture[node=df815072-615a-49d4-92e1-2842369ea15a, loc=true, done=true]], super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=o.a.i.i.util.lang.GridFunc$27@1b01b273, initFlag=0, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[true]]]]class org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException: Failed to commit transaction (transaction has been rolled back on backup node): GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3] at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:450) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:347) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:860) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:852) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) at org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:226) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.commitAsync(GridNearTxLocal.java:852) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.optimisticPutFuture(IgniteTxLocalAdapter.java:3423) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.putAsync0(IgniteTxLocalAdapter.java:3211) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.invokeAsync(IgniteTxLocalAdapter.java:2097) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2299) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4210) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke0(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke(GridCacheAdapter.java:2262) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1575) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:566) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$2.addMeta(CacheObjectBinaryProcessorImpl.java:204) at org.apache.ignite.internal.binary.BinaryContext.registerUserClassDescriptor(BinaryContext.java:711) at org.apache.ignite.internal.binary.BinaryContext.registerClassDescriptor(BinaryContext.java:663) at org.apache.ignite.internal.binary.BinaryContext.descriptorForClass(BinaryContext.java:563) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:145) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:132) at org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:239) at org.apache.ignite.internal.binary.BinaryMarshaller.marshal(BinaryMarshaller.java:92) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1216) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1197) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1179) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:840) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:799) at org.apache.ignite.internal.GridEventConsumeHandler$2$1.run(GridEventConsumeHandler.java:231) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: class org.apache.ignite.internal.cluster.ClusterTopologyCheckedException: Primary node left grid: 5f3be800-b89a-419b-8d7e-ba5adbb36fab at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:444) ... 35 more 2016-05-16 12:25:25 ERROR GridNearTxFinishFuture:495 - Failed to automatically rollback transaction: GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=false, hasRemoteLocks=false, thread=sys-#20%null%, mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], writeVer=null, implicit=true, loc=true, threadId=32, startTime=1463390725843, nodeId=df815072-615a-49d4-92e1-2842369ea15a, startVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, sysInvalidate=false, sys=true, plc=5, commitVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], finalizing=NONE, preparing=false, invalidParts=null, state=ROLLED_BACK, timedOut=false, topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], duration=70ms, onePhaseCommit=true], size=1]]] class org.apache.ignite.IgniteCheckedException: Invalid transaction state for commit [state=ROLLED_BACK, tx=GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=false, hasRemoteLocks=false, thread=sys-#20%null%, mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601, partId=-1, txKey=IgniteTxKey [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], cacheId=-2100569601], val=[op=TRANSFORM, val=null], prevVal=[op=NOOP, val=null], entryProcessorsCol=[IgniteBiTuple [val1=MetadataProcessor [newMeta=BinaryMetadata [typeId=887264644, typeName=org.jsr166.ConcurrentLinkedDeque8, fields=null, affKeyFieldName=null, isEnum=false]], val2=[Ljava.lang.Object;@4807a494]], ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtDetachedCacheEntry [super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [val=BinaryMetadataKey [typeId=887264644], hasValBytes=true], val=null, startVer=1463390656382, ver=GridCacheVersion [topVer=74870619, time=1463390725860, order=1463390656382, nodeOrder=3], hash=887264644, extras=null, flags=0]]], prepared=0, locked=false, nodeId=5f3be800-b89a-419b-8d7e-ba5adbb36fab, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3]]], explicitLock=false, dhtVer=null, last=false, near=false, clientFirst=false, node=5f3be800-b89a-419b-8d7e-ba5adbb36fab]], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], writeVer=null, implicit=true, loc=true, threadId=32, startTime=1463390725843, nodeId=df815072-615a-49d4-92e1-2842369ea15a, startVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, sysInvalidate=false, sys=true, plc=5, commitVer=GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3], finalizing=NONE, preparing=false, invalidParts=null, state=ROLLED_BACK, timedOut=false, topVer=AffinityTopologyVersion [topVer=3, minorTopVer=1], duration=60ms, onePhaseCommit=true], size=1]]]] at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.finish(GridNearTxLocal.java:722) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:276) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:67) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:324) at org.apache.ignite.internal.util.future.GridCompoundFuture.checkComplete(GridCompoundFuture.java:241) at org.apache.ignite.internal.util.future.GridCompoundFuture.markInitialized(GridCompoundFuture.java:232) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:369) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.rollbackAsync(GridNearTxLocal.java:920) at org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter.rollback(GridDhtTxLocalAdapter.java:859) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:251) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.onDone(GridNearTxFinishFuture.java:67) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:335) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:118) at org.apache.ignite.internal.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:41) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListeners(GridFutureAdapter.java:251) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:381) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:347) at org.apache.ignite.internal.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:335) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:449) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:347) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:860) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:852) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) at org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:226) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.commitAsync(GridNearTxLocal.java:852) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.optimisticPutFuture(IgniteTxLocalAdapter.java:3423) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.putAsync0(IgniteTxLocalAdapter.java:3211) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.invokeAsync(IgniteTxLocalAdapter.java:2097) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2299) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4210) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke0(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke(GridCacheAdapter.java:2262) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1575) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:566) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$2.addMeta(CacheObjectBinaryProcessorImpl.java:204) at org.apache.ignite.internal.binary.BinaryContext.registerUserClassDescriptor(BinaryContext.java:711) at org.apache.ignite.internal.binary.BinaryContext.registerClassDescriptor(BinaryContext.java:663) at org.apache.ignite.internal.binary.BinaryContext.descriptorForClass(BinaryContext.java:563) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:145) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:132) at org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:239) at org.apache.ignite.internal.binary.BinaryMarshaller.marshal(BinaryMarshaller.java:92) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1216) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1197) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1179) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:840) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:799) at org.apache.ignite.internal.GridEventConsumeHandler$2$1.run(GridEventConsumeHandler.java:231) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2016-05-16 12:25:25 ERROR GridEventConsumeHandler:495 - Failed to send event notification to node: 28b7b448-d0bb-404b-9643-abf87ce27fad class org.apache.ignite.binary.BinaryObjectException: Failed to update meta data for type: org.jsr166.ConcurrentLinkedDeque8 at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:572) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl$2.addMeta(CacheObjectBinaryProcessorImpl.java:204) at org.apache.ignite.internal.binary.BinaryContext.registerUserClassDescriptor(BinaryContext.java:711) at org.apache.ignite.internal.binary.BinaryContext.registerClassDescriptor(BinaryContext.java:663) at org.apache.ignite.internal.binary.BinaryContext.descriptorForClass(BinaryContext.java:563) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:145) at org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:132) at org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:239) at org.apache.ignite.internal.binary.BinaryMarshaller.marshal(BinaryMarshaller.java:92) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1216) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1197) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendWithRetries(GridContinuousProcessor.java:1179) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.sendNotification(GridContinuousProcessor.java:840) at org.apache.ignite.internal.processors.continuous.GridContinuousProcessor.addNotification(GridContinuousProcessor.java:799) at org.apache.ignite.internal.GridEventConsumeHandler$2$1.run(GridEventConsumeHandler.java:231) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: javax.cache.CacheException: class org.apache.ignite.transactions.TransactionRollbackException: Failed to commit transaction (transaction has been rolled back on backup node): GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3] at org.apache.ignite.internal.processors.cache.GridCacheUtils.convertToCacheException(GridCacheUtils.java:1478) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.cacheException(IgniteCacheProxy.java:1944) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1585) at org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.addMeta(CacheObjectBinaryProcessorImpl.java:566) ... 19 more Caused by: class org.apache.ignite.transactions.TransactionRollbackException: Failed to commit transaction (transaction has been rolled back on backup node): GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3] at org.apache.ignite.internal.util.IgniteUtils$10.apply(IgniteUtils.java:800) at org.apache.ignite.internal.util.IgniteUtils$10.apply(IgniteUtils.java:798) ... 23 more Caused by: class org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException: Failed to commit transaction (transaction has been rolled back on backup node): GridCacheVersion [topVer=74870619, time=1463390725845, order=1463390656380, nodeOrder=3] at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:450) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.finish(GridNearTxFinishFuture.java:347) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:860) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4.apply(GridNearTxLocal.java:852) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:263) at org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:226) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal.commitAsync(GridNearTxLocal.java:852) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.optimisticPutFuture(IgniteTxLocalAdapter.java:3423) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.putAsync0(IgniteTxLocalAdapter.java:3211) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter.invokeAsync(IgniteTxLocalAdapter.java:2097) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2299) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$25.op(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4210) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke0(GridCacheAdapter.java:2291) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.invoke(GridCacheAdapter.java:2262) at org.apache.ignite.internal.processors.cache.IgniteCacheProxy.invoke(IgniteCacheProxy.java:1575) ... 20 more Caused by: class org.apache.ignite.internal.cluster.ClusterTopologyCheckedException: Primary node left grid: 5f3be800-b89a-419b-8d7e-ba5adbb36fab at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture.checkBackup(GridNearTxFinishFuture.java:444) ... 35 more {noformat} -- This message was sent by Atlassian JIRA (v6.3.4#6332)