[ https://issues.apache.org/jira/browse/GEODE-216?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Darrel Schneider updated GEODE-216: ----------------------------------- Component/s: core > Failure to process GetMessage due RegionDestroyedException > ---------------------------------------------------------- > > Key: GEODE-216 > URL: https://issues.apache.org/jira/browse/GEODE-216 > Project: Geode > Issue Type: Bug > Components: core > Reporter: Dan Smith > Assignee: Darrel Schneider > > This failure showed up in a nightly build, the test > RedisDistDUnitTest.testConcCreateDestroy timed out on a read. > But to me it looks like the underlying issue is some missing exception > handling code in GetMessage that caused a hang due to a > RegionDestroyedException. > https://builds.apache.org/job/Geode-nightly/183/testReport/junit/com.gemstone.gemfire.redis/RedisDistDUnitTest/testConcCreateDestroy/ > {noformat} > dunit.RMIException: While invoking > com.gemstone.gemfire.redis.RedisDistDUnitTest$1ConcCreateDestroy.call in VM 3 > running on Host jenkins-ubuntu-1404-4gb-f9a with 4 VMs > at dunit.VM.invoke(VM.java:359) > at dunit.VM.invoke(VM.java:303) > at dunit.VM.invoke(VM.java:271) > at > com.gemstone.gemfire.redis.RedisDistDUnitTest.testConcCreateDestroy(RedisDistDUnitTest.java:168) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at junit.framework.TestCase.runTest(TestCase.java:176) > at junit.framework.TestCase.runBare(TestCase.java:141) > at junit.framework.TestResult$1.protect(TestResult.java:122) > at junit.framework.TestResult.runProtected(TestResult.java:142) > at junit.framework.TestResult.run(TestResult.java:125) > at junit.framework.TestCase.run(TestCase.java:129) > at junit.framework.TestSuite.runTest(TestSuite.java:252) > at junit.framework.TestSuite.run(TestSuite.java:247) > at > org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:86) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.runTestClass(JUnitTestClassExecuter.java:86) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.execute(JUnitTestClassExecuter.java:49) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassProcessor.processTestClass(JUnitTestClassProcessor.java:64) > at > org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:50) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.messaging.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32) > at > org.gradle.messaging.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93) > at com.sun.proxy.$Proxy2.processTestClass(Unknown Source) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:106) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.messaging.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:360) > at > org.gradle.internal.concurrent.DefaultExecutorFactory$StoppableExecutorImpl$1.run(DefaultExecutorFactory.java:64) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:745) > Caused by: redis.clients.jedis.exceptions.JedisConnectionException: > java.net.SocketTimeoutException: Read timed out > at > redis.clients.util.RedisInputStream.ensureFill(RedisInputStream.java:201) > at > redis.clients.util.RedisInputStream.readByte(RedisInputStream.java:40) > at redis.clients.jedis.Protocol.process(Protocol.java:132) > at redis.clients.jedis.Protocol.read(Protocol.java:196) > at > redis.clients.jedis.Connection.readProtocolWithCheckingBroken(Connection.java:288) > at redis.clients.jedis.Connection.getIntegerReply(Connection.java:213) > at redis.clients.jedis.Jedis.lpush(Jedis.java:814) > at > com.gemstone.gemfire.redis.RedisDistDUnitTest$1ConcCreateDestroy.call(RedisDistDUnitTest.java:144) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at hydra.MethExecutor.executeObject(MethExecutor.java:258) > at > dunit.standalone.RemoteDUnitVM.executeMethodOnObject(RemoteDUnitVM.java:67) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:322) > at sun.rmi.transport.Transport$1.run(Transport.java:177) > at sun.rmi.transport.Transport$1.run(Transport.java:174) > at java.security.AccessController.doPrivileged(Native Method) > at sun.rmi.transport.Transport.serviceCall(Transport.java:173) > at > sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:556) > at > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:811) > at > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:670) > ... 3 more > Caused by: java.net.SocketTimeoutException: Read timed out > at java.net.SocketInputStream.socketRead0(Native Method) > at java.net.SocketInputStream.read(SocketInputStream.java:152) > at java.net.SocketInputStream.read(SocketInputStream.java:122) > at java.net.SocketInputStream.read(SocketInputStream.java:108) > at > redis.clients.util.RedisInputStream.ensureFill(RedisInputStream.java:195) > ... 28 more > {noformat} > Here's what I see in the logs. This message failed to be processed because it > threw an exception. It may have hung the system by not sending a reply. > {noformat} > vm_0][fatal 2015/08/13 08:11:55.144 UTC <P2P message reader for > jenkins-ubuntu-1404-4gb-f9a(17969)<v1>:23448 shared ordered uid=6 port=34039> > tid=0x350] While pulling a message > [vm_0]com.gemstone.gemfire.cache.RegionDestroyedException: Region for prId= > 16 is destroyed > [vm_0] at > com.gemstone.gemfire.internal.cache.PartitionedRegion$PRIdMap.getRegion(PartitionedRegion.java:604) > [vm_0] at > com.gemstone.gemfire.internal.cache.PartitionedRegion.getPRFromId(PartitionedRegion.java:5494) > [vm_0] at > com.gemstone.gemfire.internal.cache.partitioned.GetMessage.getProcessorType(GetMessage.java:118) > [vm_0] at > com.gemstone.gemfire.distributed.internal.DistributionMessage.schedule(DistributionMessage.java:414) > [vm_0] at > com.gemstone.gemfire.distributed.internal.DistributionManager.scheduleIncomingMessage(DistributionManager.java:3862) > [vm_0] at > com.gemstone.gemfire.distributed.internal.DistributionManager.handleIncomingDMsg(DistributionManager.java:3486) > [vm_0] at > com.gemstone.gemfire.distributed.internal.DistributionManager$MyListener.messageReceived(DistributionManager.java:4683) > [vm_0] at > com.gemstone.gemfire.distributed.internal.membership.jgroup.JGroupMembershipManager.processMessage(JGroupMembershipManager.java:2160) > [vm_0] at > com.gemstone.gemfire.distributed.internal.membership.jgroup.JGroupMembershipManager.handleOrDeferMessage(JGroupMembershipManager.java:2069) > [vm_0] at > com.gemstone.gemfire.distributed.internal.membership.jgroup.JGroupMembershipManager$MyDCReceiver.messageReceived(JGroupMembershipManager.java:668) > [vm_0] at > com.gemstone.gemfire.distributed.internal.direct.DirectChannel.receive(DirectChannel.java:804) > [vm_0] at > com.gemstone.gemfire.internal.tcp.TCPConduit.messageReceived(TCPConduit.java:835) > [vm_0] at > com.gemstone.gemfire.internal.tcp.Connection.dispatchMessage(Connection.java:3932) > [vm_0] at > com.gemstone.gemfire.internal.tcp.Connection.processNIOBuffer(Connection.java:3515) > [vm_0] at > com.gemstone.gemfire.internal.tcp.Connection.runNioReader(Connection.java:1827) > [vm_0] at > com.gemstone.gemfire.internal.tcp.Connection.run(Connection.java:1702) > [vm_0] at java.lang.Thread.run(Thread.java:745) > {noformat} -- This message was sent by Atlassian JIRA (v6.3.4#6332)