[
https://issues.apache.org/jira/browse/GEODE-9484?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17532066#comment-17532066
]
Ivan Godwin edited comment on GEODE-9484 at 5/5/22 5:14 AM:
------------------------------------------------------------
Noting possible issue found during internal testing. Will analyze further and
update.
{code:java}
[fatal 2022/05/04 18:27:45.084 GMT <server-member-name-3> <ServerConnection on
port 40404 Thread 2> tid=0x6c] While pushing message
<UpdateOperation$UpdateWithContextMessage(region
path='/__PR/_B__<region-name>';
sender=<server-member-name-3>(<server-member-name-3>:1)<v4>:41000;
callbackArg=null; processorId=0; op=CREATE; applied=false; directAck=true;
posdup=false; hasDelta=false; hasOldValue=false; version={v1; rv1;
time=1651688854339}
FilterRoutingInfo(remote={<server-member-name-0>(<server-member-name-0>:1)<v2>:41000=});
lastModified=1651688854339; key=0; newValue=(5 bytes); eventId=EventID[id=58
bytes;threadID=853971;sequenceID=0]; deserializationPolicy=LAZY;
context=identity(<region-name>(SpringBasedClientCacheApplication:1:loner):48828:dd7d5290:SpringBasedClientCacheApplication,connection=1)>
to recipients: <<server-member-name-0>(<server-member-name-0>:1)<v2>:41000>
java.lang.NullPointerException
at
org.apache.geode.internal.tcp.TCPConduit.getFirstScanForConnection(TCPConduit.java:958)
at
org.apache.geode.distributed.internal.direct.DirectChannel.getConnections(DirectChannel.java:477)
at
org.apache.geode.distributed.internal.direct.DirectChannel.sendToMany(DirectChannel.java:277)
at
org.apache.geode.distributed.internal.direct.DirectChannel.sendToOne(DirectChannel.java:186)
at
org.apache.geode.distributed.internal.direct.DirectChannel.send(DirectChannel.java:541)
at
org.apache.geode.distributed.internal.DistributionImpl.directChannelSend(DistributionImpl.java:348)
at
org.apache.geode.distributed.internal.DistributionImpl.send(DistributionImpl.java:293)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.sendViaMembershipManager(ClusterDistributionManager.java:2067)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.sendOutgoing(ClusterDistributionManager.java:1994)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.sendMessage(ClusterDistributionManager.java:2031)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.putOutgoing(ClusterDistributionManager.java:1088)
at
org.apache.geode.internal.cache.DistributedCacheOperation._distribute(DistributedCacheOperation.java:556)
at
org.apache.geode.internal.cache.DistributedCacheOperation.startOperation(DistributedCacheOperation.java:267)
at
org.apache.geode.internal.cache.BucketRegion.basicPutPart2(BucketRegion.java:715)
at
org.apache.geode.internal.cache.map.RegionMapPut.doBeforeCompletionActions(RegionMapPut.java:282)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutAndDeliverEvent(AbstractRegionMapPut.java:301)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.runWithIndexUpdatingInProgress(AbstractRegionMapPut.java:308)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutIfPreconditionsSatisified(AbstractRegionMapPut.java:296)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutOnSynchronizedRegionEntry(AbstractRegionMapPut.java:282)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutOnRegionEntryInMap(AbstractRegionMapPut.java:273)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.addRegionEntryToMapAndDoPut(AbstractRegionMapPut.java:251)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutRetryingIfNeeded(AbstractRegionMapPut.java:216)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doWithIndexInUpdateMode(AbstractRegionMapPut.java:198)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPut(AbstractRegionMapPut.java:180)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.runWhileLockedForCacheModification(AbstractRegionMapPut.java:119)
at
org.apache.geode.internal.cache.map.RegionMapPut.runWhileLockedForCacheModification(RegionMapPut.java:161)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.put(AbstractRegionMapPut.java:169)
at
org.apache.geode.internal.cache.AbstractRegionMap.basicPut(AbstractRegionMap.java:2016)
at
org.apache.geode.internal.cache.BucketRegion.virtualPut(BucketRegion.java:544)
at
org.apache.geode.internal.cache.LocalRegion.virtualPut(LocalRegion.java:5635)
at
org.apache.geode.internal.cache.PartitionedRegionDataStore.putLocally(PartitionedRegionDataStore.java:1193)
at
org.apache.geode.internal.cache.PartitionedRegion.putInBucket(PartitionedRegion.java:3033)
at
org.apache.geode.internal.cache.PartitionedRegion.virtualPut(PartitionedRegion.java:2248)
at
org.apache.geode.internal.cache.LocalRegionDataView.putEntry(LocalRegionDataView.java:171)
at
org.apache.geode.internal.cache.LocalRegion.basicUpdate(LocalRegion.java:5628)
at
org.apache.geode.internal.cache.LocalRegion.basicUpdate(LocalRegion.java:5588)
at
org.apache.geode.internal.cache.LocalRegion.basicBridgePut(LocalRegion.java:5259)
at
org.apache.geode.internal.cache.tier.sockets.command.Put70.cmdExecute(Put70.java:384)
at
org.apache.geode.internal.cache.tier.sockets.BaseCommand.execute(BaseCommand.java:187)
at
org.apache.geode.internal.cache.tier.sockets.ServerConnection.doNormalMessage(ServerConnection.java:880)
at
org.apache.geode.internal.cache.tier.sockets.ServerConnection.doOneMessage(ServerConnection.java:1074)
at
org.apache.geode.internal.cache.tier.sockets.ServerConnection.run(ServerConnection.java:1356)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at
org.apache.geode.internal.cache.tier.sockets.AcceptorImpl.lambda$initializeServerConnectionThreadPool$3(AcceptorImpl.java:690)
at
org.apache.geode.logging.internal.executors.LoggingThreadFactory.lambda$newThread$0(LoggingThreadFactory.java:120)
at java.base/java.lang.Thread.run(Thread.java:829)
{code}
was (Author: igodwin):
Noting possible issue found during internal testing. Will analyze further and
update.
{code:java}
[fatal 2022/05/04 18:27:45.084 GMT gemfire-cluster-server-3 <ServerConnection
on port 40404 Thread 2> tid=0x6c] While pushing message
<UpdateOperation$UpdateWithContextMessage(region
path='/__PR/_B__gemfire-clients-564c765b59-vg4wb-region-8631f0c98b0a0892_0';
sender=gemfire-cluster-server-3(gemfire-cluster-server-3:1)<v4>:41000;
callbackArg=null; processorId=0; op=CREATE; applied=false; directAck=true;
posdup=false; hasDelta=false; hasOldValue=false; version={v1; rv1;
time=1651688854339}
FilterRoutingInfo(remote={gemfire-cluster-server-0(gemfire-cluster-server-0:1)<v2>:41000=});
lastModified=1651688854339; key=0; newValue=(5 bytes); eventId=EventID[id=58
bytes;threadID=853971;sequenceID=0]; deserializationPolicy=LAZY;
context=identity(gemfire-clients-564c765b59-vg4wb(SpringBasedClientCacheApplication:1:loner):48828:dd7d5290:SpringBasedClientCacheApplication,connection=1)>
to recipients: <gemfire-cluster-server-0(gemfire-cluster-server-0:1)<v2>:41000>
java.lang.NullPointerException
at
org.apache.geode.internal.tcp.TCPConduit.getFirstScanForConnection(TCPConduit.java:958)
at
org.apache.geode.distributed.internal.direct.DirectChannel.getConnections(DirectChannel.java:477)
at
org.apache.geode.distributed.internal.direct.DirectChannel.sendToMany(DirectChannel.java:277)
at
org.apache.geode.distributed.internal.direct.DirectChannel.sendToOne(DirectChannel.java:186)
at
org.apache.geode.distributed.internal.direct.DirectChannel.send(DirectChannel.java:541)
at
org.apache.geode.distributed.internal.DistributionImpl.directChannelSend(DistributionImpl.java:348)
at
org.apache.geode.distributed.internal.DistributionImpl.send(DistributionImpl.java:293)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.sendViaMembershipManager(ClusterDistributionManager.java:2067)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.sendOutgoing(ClusterDistributionManager.java:1994)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.sendMessage(ClusterDistributionManager.java:2031)
at
org.apache.geode.distributed.internal.ClusterDistributionManager.putOutgoing(ClusterDistributionManager.java:1088)
at
org.apache.geode.internal.cache.DistributedCacheOperation._distribute(DistributedCacheOperation.java:556)
at
org.apache.geode.internal.cache.DistributedCacheOperation.startOperation(DistributedCacheOperation.java:267)
at
org.apache.geode.internal.cache.BucketRegion.basicPutPart2(BucketRegion.java:715)
at
org.apache.geode.internal.cache.map.RegionMapPut.doBeforeCompletionActions(RegionMapPut.java:282)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutAndDeliverEvent(AbstractRegionMapPut.java:301)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.runWithIndexUpdatingInProgress(AbstractRegionMapPut.java:308)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutIfPreconditionsSatisified(AbstractRegionMapPut.java:296)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutOnSynchronizedRegionEntry(AbstractRegionMapPut.java:282)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutOnRegionEntryInMap(AbstractRegionMapPut.java:273)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.addRegionEntryToMapAndDoPut(AbstractRegionMapPut.java:251)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPutRetryingIfNeeded(AbstractRegionMapPut.java:216)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doWithIndexInUpdateMode(AbstractRegionMapPut.java:198)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.doPut(AbstractRegionMapPut.java:180)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.runWhileLockedForCacheModification(AbstractRegionMapPut.java:119)
at
org.apache.geode.internal.cache.map.RegionMapPut.runWhileLockedForCacheModification(RegionMapPut.java:161)
at
org.apache.geode.internal.cache.map.AbstractRegionMapPut.put(AbstractRegionMapPut.java:169)
at
org.apache.geode.internal.cache.AbstractRegionMap.basicPut(AbstractRegionMap.java:2016)
at
org.apache.geode.internal.cache.BucketRegion.virtualPut(BucketRegion.java:544)
at
org.apache.geode.internal.cache.LocalRegion.virtualPut(LocalRegion.java:5635)
at
org.apache.geode.internal.cache.PartitionedRegionDataStore.putLocally(PartitionedRegionDataStore.java:1193)
at
org.apache.geode.internal.cache.PartitionedRegion.putInBucket(PartitionedRegion.java:3033)
at
org.apache.geode.internal.cache.PartitionedRegion.virtualPut(PartitionedRegion.java:2248)
at
org.apache.geode.internal.cache.LocalRegionDataView.putEntry(LocalRegionDataView.java:171)
at
org.apache.geode.internal.cache.LocalRegion.basicUpdate(LocalRegion.java:5628)
at
org.apache.geode.internal.cache.LocalRegion.basicUpdate(LocalRegion.java:5588)
at
org.apache.geode.internal.cache.LocalRegion.basicBridgePut(LocalRegion.java:5259)
at
org.apache.geode.internal.cache.tier.sockets.command.Put70.cmdExecute(Put70.java:384)
at
org.apache.geode.internal.cache.tier.sockets.BaseCommand.execute(BaseCommand.java:187)
at
org.apache.geode.internal.cache.tier.sockets.ServerConnection.doNormalMessage(ServerConnection.java:880)
at
org.apache.geode.internal.cache.tier.sockets.ServerConnection.doOneMessage(ServerConnection.java:1074)
at
org.apache.geode.internal.cache.tier.sockets.ServerConnection.run(ServerConnection.java:1356)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at
org.apache.geode.internal.cache.tier.sockets.AcceptorImpl.lambda$initializeServerConnectionThreadPool$3(AcceptorImpl.java:690)
at
org.apache.geode.logging.internal.executors.LoggingThreadFactory.lambda$newThread$0(LoggingThreadFactory.java:120)
at java.base/java.lang.Thread.run(Thread.java:829)
{code}
> Data inconsistency in replicated region with 3 or more servers, and one
> server is down
> ---------------------------------------------------------------------------------------
>
> Key: GEODE-9484
> URL: https://issues.apache.org/jira/browse/GEODE-9484
> Project: Geode
> Issue Type: Improvement
> Components: client/server, regions
> Affects Versions: 1.13.0
> Reporter: Mario Ivanac
> Assignee: Mario Ivanac
> Priority: Major
> Labels: pull-request-available
> Fix For: 1.15.0
>
>
> We have configured replicated region with 3 or more servers, and client is
> configured with read timeout set to value same or smaller than member timeout.
> In case while client is putting data in region, one of replicated servers is
> shutdown, it is observed that we have data inconsistency.
>
> We see that data part of data is written in server connected with client, but
> in remaining replicated servers it is missing.
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
