[
https://issues.apache.org/jira/browse/GEODE-986?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15190080#comment-15190080
]
Bruce Schuchardt commented on GEODE-986:
----------------------------------------
This is caused by old backward-compatibility code in TcpClient. It is assuming
that the locator is running v5.7 if it can't fetch the locator's product
version information, but the client then goes on to tell the locator that it is
running 9.0. The FindCoordinator request is then serialized with v5.7 but the
locator tries to deserialize it with v9.0 and runs into trouble when assembling
an InternalDistributedMember. The on-wire form of this class has changed in
9.0.
> CI Failure: MultiuserAPIDUnitTest.testMultiUserUnsupportedAPIs failed with
> SocketException
> ------------------------------------------------------------------------------------------
>
> Key: GEODE-986
> URL: https://issues.apache.org/jira/browse/GEODE-986
> Project: Geode
> Issue Type: Bug
> Components: membership
> Reporter: Barry Oglesby
> Labels: ci
>
> Geode_develop_DistributedTests
> Private Build #1662
> Revision: e685fd85ac7e2607f70b47bfb448b1d91a56b103
> {noformat}
> [vm_1][error 2016/02/18 16:24:59.642 PST <RMI TCP
> Connection(18)-10.118.32.91> tid=0x12] Unexpected problem starting up
> membership services
> [vm_1]com.gemstone.gemfire.ToDataException: toData failed on DataSerializable
> class
> com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.invokeToData(InternalDataSerializer.java:2453)
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.writeDSFID(InternalDataSerializer.java:1412)
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.basicWriteObject(InternalDataSerializer.java:2150)
> [vm_1] at
> com.gemstone.gemfire.DataSerializer.writeObject(DataSerializer.java:3241)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.locator.FindCoordinatorRequest.toData(FindCoordinatorRequest.java:87)
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.invokeToData(InternalDataSerializer.java:2419)
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.writeDSFID(InternalDataSerializer.java:1412)
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.basicWriteObject(InternalDataSerializer.java:2150)
> [vm_1] at
> com.gemstone.gemfire.DataSerializer.writeObject(DataSerializer.java:3241)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.tcpserver.TcpClient.requestToServer(TcpClient.java:145)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave$TcpClientWrapper.sendCoordinatorFindRequest(GMSJoinLeave.java:988)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave.findCoordinator(GMSJoinLeave.java:910)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave.join(GMSJoinLeave.java:242)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.mgr.GMSMembershipManager.join(GMSMembershipManager.java:676)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.mgr.GMSMembershipManager.joinDistributedSystem(GMSMembershipManager.java:765)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.Services.start(Services.java:174)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.GMSMemberFactory.newMembershipManager(GMSMemberFactory.java:105)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.MemberFactory.newMembershipManager(MemberFactory.java:93)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.DistributionManager.<init>(DistributionManager.java:1159)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.DistributionManager.<init>(DistributionManager.java:1211)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.DistributionManager.create(DistributionManager.java:573)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.InternalDistributedSystem.initialize(InternalDistributedSystem.java:652)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.InternalDistributedSystem.newInstance(InternalDistributedSystem.java:277)
> [vm_1] at
> com.gemstone.gemfire.distributed.DistributedSystem.connect(DistributedSystem.java:1641)
> [vm_1] at
> com.gemstone.gemfire.test.dunit.DistributedTestCase.getSystem(DistributedTestCase.java:145)
> [vm_1] at
> com.gemstone.gemfire.security.SecurityTestUtil.createSystem(SecurityTestUtil.java:184)
> [vm_1] at
> com.gemstone.gemfire.security.SecurityTestUtil.createCacheServer(SecurityTestUtil.java:306)
> [vm_1] at
> com.gemstone.gemfire.security.SecurityTestUtil.createCacheServer(SecurityTestUtil.java:278)
> [vm_1] at
> com.gemstone.gemfire.security.MultiuserAPIDUnitTest.createCacheServer(MultiuserAPIDUnitTest.java:105)
> [vm_1] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> [vm_1] at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> [vm_1] at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> [vm_1] at java.lang.reflect.Method.invoke(Method.java:497)
> [vm_1] at hydra.MethExecutor.execute(MethExecutor.java:199)
> [vm_1] at
> com.gemstone.gemfire.test.dunit.standalone.RemoteDUnitVM.executeMethodOnClass(RemoteDUnitVM.java:118)
> [vm_1] at sun.reflect.GeneratedMethodAccessor452.invoke(Unknown Source)
> [vm_1] at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> [vm_1] at java.lang.reflect.Method.invoke(Method.java:497)
> [vm_1] at
> sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:323)
> [vm_1] at sun.rmi.transport.Transport$1.run(Transport.java:200)
> [vm_1] at sun.rmi.transport.Transport$1.run(Transport.java:197)
> [vm_1] at java.security.AccessController.doPrivileged(Native Method)
> [vm_1] at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
> [vm_1] at
> sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
> [vm_1] at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
> [vm_1] at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
> [vm_1] at java.security.AccessController.doPrivileged(Native Method)
> [vm_1] at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
> [vm_1] at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [vm_1] at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [vm_1] at java.lang.Thread.run(Thread.java:745)
> [vm_1]Caused by: java.lang.reflect.InvocationTargetException
> [vm_1] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> [vm_1] at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> [vm_1] at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> [vm_1] at java.lang.reflect.Method.invoke(Method.java:497)
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.invokeToData(InternalDataSerializer.java:2409)
> [vm_1] ... 50 more
> [vm_1]Caused by: java.net.SocketException: Broken pipe
> [vm_1] at java.net.SocketOutputStream.socketWrite0(Native Method)
> [vm_1] at
> java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:109)
> [vm_1] at
> java.net.SocketOutputStream.write(SocketOutputStream.java:153)
> [vm_1] at java.io.DataOutputStream.write(DataOutputStream.java:107)
> [vm_1] at java.io.DataOutputStream.write(DataOutputStream.java:107)
> [vm_1] at
> com.gemstone.gemfire.DataSerializer.writeByteArray(DataSerializer.java:1461)
> [vm_1] at
> com.gemstone.gemfire.DataSerializer.writeByteArray(DataSerializer.java:1424)
> [vm_1] at
> com.gemstone.gemfire.DataSerializer.writeInetAddress(DataSerializer.java:540)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember.toDataPre_GFE_7_1_0_0(InternalDistributedMember.java:994)
> [vm_1] ... 55 more
> {noformat}
> {noformat}
> [vm_1][info 2016/02/18 16:24:59.650 PST <RMI TCP Connection(18)-10.118.32.91>
> tid=0x12] Got result: EXCEPTION_OCCURRED
> [vm_1]java.lang.AssertionError: Got unexpected exception when starting peer
> [vm_1] at com.gemstone.gemfire.test.dunit.Assert.fail(Assert.java:64)
> [vm_1] at
> com.gemstone.gemfire.security.SecurityTestUtil.createCacheServer(SecurityTestUtil.java:330)
> [vm_1] at
> com.gemstone.gemfire.security.SecurityTestUtil.createCacheServer(SecurityTestUtil.java:278)
> [vm_1] at
> com.gemstone.gemfire.security.MultiuserAPIDUnitTest.createCacheServer(MultiuserAPIDUnitTest.java:105)
> [vm_1] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> [vm_1] at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> [vm_1] at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> [vm_1] at java.lang.reflect.Method.invoke(Method.java:497)
> [vm_1] at hydra.MethExecutor.execute(MethExecutor.java:199)
> [vm_1] at
> com.gemstone.gemfire.test.dunit.standalone.RemoteDUnitVM.executeMethodOnClass(RemoteDUnitVM.java:118)
> [vm_1] at sun.reflect.GeneratedMethodAccessor452.invoke(Unknown Source)
> [vm_1] at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> [vm_1] at java.lang.reflect.Method.invoke(Method.java:497)
> [vm_1] at
> sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:323)
> [vm_1] at sun.rmi.transport.Transport$1.run(Transport.java:200)
> [vm_1] at sun.rmi.transport.Transport$1.run(Transport.java:197)
> [vm_1] at java.security.AccessController.doPrivileged(Native Method)
> [vm_1] at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
> [vm_1] at
> sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
> [vm_1] at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
> [vm_1] at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
> [vm_1] at java.security.AccessController.doPrivileged(Native Method)
> [vm_1] at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
> [vm_1] at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [vm_1] at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [vm_1] at java.lang.Thread.run(Thread.java:745)
> [vm_1]Caused by: com.gemstone.gemfire.SystemConnectException: Problem
> starting up membership services
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.GMSMemberFactory.newMembershipManager(GMSMemberFactory.java:118)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.MemberFactory.newMembershipManager(MemberFactory.java:93)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.DistributionManager.<init>(DistributionManager.java:1159)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.DistributionManager.<init>(DistributionManager.java:1211)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.DistributionManager.create(DistributionManager.java:573)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.InternalDistributedSystem.initialize(InternalDistributedSystem.java:652)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.InternalDistributedSystem.newInstance(InternalDistributedSystem.java:277)
> [vm_1] at
> com.gemstone.gemfire.distributed.DistributedSystem.connect(DistributedSystem.java:1641)
> [vm_1] at
> com.gemstone.gemfire.test.dunit.DistributedTestCase.getSystem(DistributedTestCase.java:145)
> [vm_1] at
> com.gemstone.gemfire.security.SecurityTestUtil.createSystem(SecurityTestUtil.java:184)
> [vm_1] at
> com.gemstone.gemfire.security.SecurityTestUtil.createCacheServer(SecurityTestUtil.java:306)
> [vm_1] ... 24 more
> [vm_1]Caused by: com.gemstone.gemfire.ToDataException: toData failed on
> DataSerializable class
> com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.invokeToData(InternalDataSerializer.java:2453)
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.writeDSFID(InternalDataSerializer.java:1412)
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.basicWriteObject(InternalDataSerializer.java:2150)
> [vm_1] at
> com.gemstone.gemfire.DataSerializer.writeObject(DataSerializer.java:3241)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.locator.FindCoordinatorRequest.toData(FindCoordinatorRequest.java:87)
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.invokeToData(InternalDataSerializer.java:2419)
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.writeDSFID(InternalDataSerializer.java:1412)
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.basicWriteObject(InternalDataSerializer.java:2150)
> [vm_1] at
> com.gemstone.gemfire.DataSerializer.writeObject(DataSerializer.java:3241)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.tcpserver.TcpClient.requestToServer(TcpClient.java:145)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave$TcpClientWrapper.sendCoordinatorFindRequest(GMSJoinLeave.java:988)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave.findCoordinator(GMSJoinLeave.java:910)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave.join(GMSJoinLeave.java:242)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.mgr.GMSMembershipManager.join(GMSMembershipManager.java:676)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.mgr.GMSMembershipManager.joinDistributedSystem(GMSMembershipManager.java:765)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.Services.start(Services.java:174)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.gms.GMSMemberFactory.newMembershipManager(GMSMemberFactory.java:105)
> [vm_1] ... 34 more
> [vm_1]Caused by: java.lang.reflect.InvocationTargetException
> [vm_1] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> [vm_1] at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> [vm_1] at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> [vm_1] at java.lang.reflect.Method.invoke(Method.java:497)
> [vm_1] at
> com.gemstone.gemfire.internal.InternalDataSerializer.invokeToData(InternalDataSerializer.java:2409)
> [vm_1] ... 50 more
> [vm_1]Caused by: java.net.SocketException: Broken pipe
> [vm_1] at java.net.SocketOutputStream.socketWrite0(Native Method)
> [vm_1] at
> java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:109)
> [vm_1] at
> java.net.SocketOutputStream.write(SocketOutputStream.java:153)
> [vm_1] at java.io.DataOutputStream.write(DataOutputStream.java:107)
> [vm_1] at java.io.DataOutputStream.write(DataOutputStream.java:107)
> [vm_1] at
> com.gemstone.gemfire.DataSerializer.writeByteArray(DataSerializer.java:1461)
> [vm_1] at
> com.gemstone.gemfire.DataSerializer.writeByteArray(DataSerializer.java:1424)
> [vm_1] at
> com.gemstone.gemfire.DataSerializer.writeInetAddress(DataSerializer.java:540)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember.toDataPre_GFE_7_1_0_0(InternalDistributedMember.java:994)
> [vm_1] ... 55 more
> [vm_1] from
> com.gemstone.gemfire.security.MultiuserAPIDUnitTest.createCacheServer with 5
> args (took 64 ms)
> {noformat}
> Later on in the test MultiuserAPIDUnitTest.testSingleUserUnsupportedAPIs,
> vm_1 attempts to start a locator, but the one above was never closed so this
> exception occurs:
> {noformat}
> [vm_1][info 2016/02/18 16:24:59.965 PST <RMI TCP Connection(18)-10.118.32.91>
> tid=0x12] Got result: EXCEPTION_OCCURRED
> [vm_1]java.lang.AssertionError: Got unexpected exception when starting peer
> [vm_1] at com.gemstone.gemfire.test.dunit.Assert.fail(Assert.java:64)
> [vm_1] at
> com.gemstone.gemfire.security.SecurityTestUtil.createCacheServer(SecurityTestUtil.java:330)
> [vm_1] at
> com.gemstone.gemfire.security.SecurityTestUtil.createCacheServer(SecurityTestUtil.java:278)
> [vm_1] at
> com.gemstone.gemfire.security.MultiuserAPIDUnitTest.createCacheServer(MultiuserAPIDUnitTest.java:105)
> [vm_1] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> [vm_1] at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> [vm_1] at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> [vm_1] at java.lang.reflect.Method.invoke(Method.java:497)
> [vm_1] at hydra.MethExecutor.execute(MethExecutor.java:199)
> [vm_1] at
> com.gemstone.gemfire.test.dunit.standalone.RemoteDUnitVM.executeMethodOnClass(RemoteDUnitVM.java:118)
> [vm_1] at sun.reflect.GeneratedMethodAccessor452.invoke(Unknown Source)
> [vm_1] at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> [vm_1] at java.lang.reflect.Method.invoke(Method.java:497)
> [vm_1] at
> sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:323)
> [vm_1] at sun.rmi.transport.Transport$1.run(Transport.java:200)
> [vm_1] at sun.rmi.transport.Transport$1.run(Transport.java:197)
> [vm_1] at java.security.AccessController.doPrivileged(Native Method)
> [vm_1] at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
> [vm_1] at
> sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
> [vm_1] at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
> [vm_1] at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
> [vm_1] at java.security.AccessController.doPrivileged(Native Method)
> [vm_1] at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
> [vm_1] at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [vm_1] at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [vm_1] at java.lang.Thread.run(Thread.java:745)
> [vm_1]Caused by: java.lang.IllegalStateException: A locator can not be
> created because one already exists in this JVM.
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.InternalLocator.createLocator(InternalLocator.java:272)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.InternalDistributedSystem.startInitLocator(InternalDistributedSystem.java:750)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.InternalDistributedSystem.initialize(InternalDistributedSystem.java:638)
> [vm_1] at
> com.gemstone.gemfire.distributed.internal.InternalDistributedSystem.newInstance(InternalDistributedSystem.java:277)
> [vm_1] at
> com.gemstone.gemfire.distributed.DistributedSystem.connect(DistributedSystem.java:1641)
> [vm_1] at
> com.gemstone.gemfire.test.dunit.DistributedTestCase.getSystem(DistributedTestCase.java:145)
> [vm_1] at
> com.gemstone.gemfire.security.SecurityTestUtil.createSystem(SecurityTestUtil.java:184)
> [vm_1] at
> com.gemstone.gemfire.security.SecurityTestUtil.createCacheServer(SecurityTestUtil.java:306)
> [vm_1] ... 24 more
> [vm_1] from
> com.gemstone.gemfire.security.MultiuserAPIDUnitTest.createCacheServer with 5
> args (took 12 ms)
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
