Dan Smith created GEODE-216:
-------------------------------
Summary: Failure to process GetMessage due RegionDestroyedException
Key: GEODE-216
URL: https://issues.apache.org/jira/browse/GEODE-216
Project: Geode
Issue Type: Bug
Reporter: Dan Smith
Assignee: Darrel Schneider
This failure showed up in a nightly build, the test
RedisDistDUnitTest.testConcCreateDestroy timed out on a read.
But to me it looks like the underlying issue is some missing exception handling
code in GetMessage that caused a hang due to a RegionDestroyedException.
https://builds.apache.org/job/Geode-nightly/183/testReport/junit/com.gemstone.gemfire.redis/RedisDistDUnitTest/testConcCreateDestroy/
{noformat}
dunit.RMIException: While invoking
com.gemstone.gemfire.redis.RedisDistDUnitTest$1ConcCreateDestroy.call in VM 3
running on Host jenkins-ubuntu-1404-4gb-f9a with 4 VMs
at dunit.VM.invoke(VM.java:359)
at dunit.VM.invoke(VM.java:303)
at dunit.VM.invoke(VM.java:271)
at
com.gemstone.gemfire.redis.RedisDistDUnitTest.testConcCreateDestroy(RedisDistDUnitTest.java:168)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at junit.framework.TestCase.runTest(TestCase.java:176)
at junit.framework.TestCase.runBare(TestCase.java:141)
at junit.framework.TestResult$1.protect(TestResult.java:122)
at junit.framework.TestResult.runProtected(TestResult.java:142)
at junit.framework.TestResult.run(TestResult.java:125)
at junit.framework.TestCase.run(TestCase.java:129)
at junit.framework.TestSuite.runTest(TestSuite.java:252)
at junit.framework.TestSuite.run(TestSuite.java:247)
at
org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:86)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.runTestClass(JUnitTestClassExecuter.java:86)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.execute(JUnitTestClassExecuter.java:49)
at
org.gradle.api.internal.tasks.testing.junit.JUnitTestClassProcessor.processTestClass(JUnitTestClassProcessor.java:64)
at
org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:50)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
at
org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at
org.gradle.messaging.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32)
at
org.gradle.messaging.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93)
at com.sun.proxy.$Proxy2.processTestClass(Unknown Source)
at
org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:106)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35)
at
org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at
org.gradle.messaging.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:360)
at
org.gradle.internal.concurrent.DefaultExecutorFactory$StoppableExecutorImpl$1.run(DefaultExecutorFactory.java:64)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: redis.clients.jedis.exceptions.JedisConnectionException:
java.net.SocketTimeoutException: Read timed out
at
redis.clients.util.RedisInputStream.ensureFill(RedisInputStream.java:201)
at
redis.clients.util.RedisInputStream.readByte(RedisInputStream.java:40)
at redis.clients.jedis.Protocol.process(Protocol.java:132)
at redis.clients.jedis.Protocol.read(Protocol.java:196)
at
redis.clients.jedis.Connection.readProtocolWithCheckingBroken(Connection.java:288)
at redis.clients.jedis.Connection.getIntegerReply(Connection.java:213)
at redis.clients.jedis.Jedis.lpush(Jedis.java:814)
at
com.gemstone.gemfire.redis.RedisDistDUnitTest$1ConcCreateDestroy.call(RedisDistDUnitTest.java:144)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at hydra.MethExecutor.executeObject(MethExecutor.java:258)
at
dunit.standalone.RemoteDUnitVM.executeMethodOnObject(RemoteDUnitVM.java:67)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:322)
at sun.rmi.transport.Transport$1.run(Transport.java:177)
at sun.rmi.transport.Transport$1.run(Transport.java:174)
at java.security.AccessController.doPrivileged(Native Method)
at sun.rmi.transport.Transport.serviceCall(Transport.java:173)
at
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:556)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:811)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:670)
... 3 more
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.read(SocketInputStream.java:152)
at java.net.SocketInputStream.read(SocketInputStream.java:122)
at java.net.SocketInputStream.read(SocketInputStream.java:108)
at
redis.clients.util.RedisInputStream.ensureFill(RedisInputStream.java:195)
... 28 more
{noformat}
Here's what I see in the logs. This message failed to be processed because it
threw an exception. It may have hung the system by not sending a reply.
{noformat}
vm_0][fatal 2015/08/13 08:11:55.144 UTC <P2P message reader for
jenkins-ubuntu-1404-4gb-f9a(17969)<v1>:23448 shared ordered uid=6 port=34039>
tid=0x350] While pulling a message
[vm_0]com.gemstone.gemfire.cache.RegionDestroyedException: Region for prId= 16
is destroyed
[vm_0] at
com.gemstone.gemfire.internal.cache.PartitionedRegion$PRIdMap.getRegion(PartitionedRegion.java:604)
[vm_0] at
com.gemstone.gemfire.internal.cache.PartitionedRegion.getPRFromId(PartitionedRegion.java:5494)
[vm_0] at
com.gemstone.gemfire.internal.cache.partitioned.GetMessage.getProcessorType(GetMessage.java:118)
[vm_0] at
com.gemstone.gemfire.distributed.internal.DistributionMessage.schedule(DistributionMessage.java:414)
[vm_0] at
com.gemstone.gemfire.distributed.internal.DistributionManager.scheduleIncomingMessage(DistributionManager.java:3862)
[vm_0] at
com.gemstone.gemfire.distributed.internal.DistributionManager.handleIncomingDMsg(DistributionManager.java:3486)
[vm_0] at
com.gemstone.gemfire.distributed.internal.DistributionManager$MyListener.messageReceived(DistributionManager.java:4683)
[vm_0] at
com.gemstone.gemfire.distributed.internal.membership.jgroup.JGroupMembershipManager.processMessage(JGroupMembershipManager.java:2160)
[vm_0] at
com.gemstone.gemfire.distributed.internal.membership.jgroup.JGroupMembershipManager.handleOrDeferMessage(JGroupMembershipManager.java:2069)
[vm_0] at
com.gemstone.gemfire.distributed.internal.membership.jgroup.JGroupMembershipManager$MyDCReceiver.messageReceived(JGroupMembershipManager.java:668)
[vm_0] at
com.gemstone.gemfire.distributed.internal.direct.DirectChannel.receive(DirectChannel.java:804)
[vm_0] at
com.gemstone.gemfire.internal.tcp.TCPConduit.messageReceived(TCPConduit.java:835)
[vm_0] at
com.gemstone.gemfire.internal.tcp.Connection.dispatchMessage(Connection.java:3932)
[vm_0] at
com.gemstone.gemfire.internal.tcp.Connection.processNIOBuffer(Connection.java:3515)
[vm_0] at
com.gemstone.gemfire.internal.tcp.Connection.runNioReader(Connection.java:1827)
[vm_0] at
com.gemstone.gemfire.internal.tcp.Connection.run(Connection.java:1702)
[vm_0] at java.lang.Thread.run(Thread.java:745)
{noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
