[ https://issues.apache.org/jira/browse/GEODE-1328?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Eric Shu updated GEODE-1328: ---------------------------- Component/s: (was: membership) gfsh > CI failure: > GMSJoinLeaveJUnitTest.testRemoveRequestWhileWaitingForFinalResponse > ------------------------------------------------------------------------------- > > Key: GEODE-1328 > URL: https://issues.apache.org/jira/browse/GEODE-1328 > Project: Geode > Issue Type: Bug > Components: gfsh > Reporter: Eric Shu > > {noformat} > Error Message > java.lang.RuntimeException: timeout waiting for view #1 > Stacktrace > java.lang.RuntimeException: timeout waiting for view #1 > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeaveJUnitTest.waitForViewAndFinalCheckInProgress(GMSJoinLeaveJUnitTest.java:1127) > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeaveJUnitTest.testRemoveRequestWhileWaitingForFinalResponse(GMSJoinLeaveJUnitTest.java:1180) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) > at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) > at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) > at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) > at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) > at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) > at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) > at org.junit.runners.ParentRunner.run(ParentRunner.java:363) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.runTestClass(JUnitTestClassExecuter.java:105) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.execute(JUnitTestClassExecuter.java:56) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassProcessor.processTestClass(JUnitTestClassProcessor.java:64) > at > org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:50) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.messaging.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32) > at > org.gradle.messaging.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93) > at com.sun.proxy.$Proxy2.processTestClass(Unknown Source) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:106) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.messaging.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:360) > at > org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:54) > at > org.gradle.internal.concurrent.StoppableExecutorImpl$1.run(StoppableExecutorImpl.java:40) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Standard Output > [info 2016/04/28 22:25:53.056 PDT <Test worker> tid=0xa] received join > request from rooktwo(7630)<ec>:8888 > [info 2016/04/28 22:25:53.083 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887, rooktwo(7630)<ec>:8889, rooktwo(7630)<ec>:8890, > rooktwo(7630)<ec>:8891] > old view is: null > [info 2016/04/28 22:25:53.084 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec>:8888 for rooktwo(7630)<ec>:8888 > [info 2016/04/28 22:25:53.085 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:25:53.088 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v11>:8887 > [info 2016/04/28 22:25:53.089 PDT <Geode Membership View Creator> tid=0x10] > View Creator thread is starting > [info 2016/04/28 22:25:53.094 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:25:53.095 PDT <Test worker> tid=0xa] Membership received > a request to remove rooktwo(7630)<ec>:8887 from rooktwo(7630)<ec>:8889 > reason=testing > [info 2016/04/28 22:25:53.103 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:25:53.105 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:25:53.106 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v12>:8887 > [info 2016/04/28 22:25:53.107 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|2] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec><v12>:8887] > old view is: View[rooktwo(7630)<ec>:8888|1] members: > [rooktwo(7630)<ec>:8888{lead}, rooktwo(7630)<ec><v12>:8887, > rooktwo(7630)<ec><v12>:8887] > [info 2016/04/28 22:25:53.107 PDT <Geode Membership View Creator> tid=0x13] > View Creator thread is starting > [info 2016/04/28 22:25:53.112 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:25:53.113 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|2] members: [rooktwo(7630)<ec>:8887{lead}, > rooktwo(7630)<ec>:8887{lead}] > old view is: View[rooktwo(7630)<ec>:8888|1] members: > [rooktwo(7630)<ec>:8888{lead}, rooktwo(7630)<ec>:8887, rooktwo(7630)<ec>:8887] > [info 2016/04/28 22:25:53.113 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:25:53.114 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v9>:8887 > [info 2016/04/28 22:25:53.115 PDT <Geode Membership View Creator> tid=0x15] > View Creator thread is starting > [info 2016/04/28 22:25:53.211 PDT <Test worker> tid=0xa] Attempting to join > the distributed system through coordinator rooktwo(7630)<ec>:8888 using > address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:25:53.211 PDT <Thread-10> tid=0x17] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec>:8890, rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:25:53.233 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:25:53.234 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|0] members: [rooktwo(7630)<ec><v0>:8887{lead}] > old view is: null > [info 2016/04/28 22:25:53.235 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v0>:8887 > [info 2016/04/28 22:25:53.236 PDT <Geode Membership View Creator> tid=0x1a] > View Creator thread is starting > [info 2016/04/28 22:25:53.241 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:25:53.242 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|2] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec>:8890, rooktwo(7630)<ec>:8887] > old view is: View[rooktwo(7630)<ec>:8888|1] members: > [rooktwo(7630)<ec>:8888{lead}, rooktwo(7630)<ec>:8887] > [info 2016/04/28 22:25:53.243 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|3] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] crashed: [rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec>:8890] > old view is: View[rooktwo(7630)<ec>:8888|2] members: > [rooktwo(7630)<ec>:8888{lead}, rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec>:8890, rooktwo(7630)<ec>:8887] > [warn 2016/04/28 22:25:53.243 PDT <Test worker> tid=0xa] total weight lost in > this view change is 40 of 65. Quorum has been lost! > [info 2016/04/28 22:25:53.249 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:25:53.250 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|2] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec>:8890, rooktwo(7630)<ec>:8887] > old view is: View[rooktwo(7630)<ec>:8888|1] members: > [rooktwo(7630)<ec>:8888{lead}, rooktwo(7630)<ec>:8887] > [info 2016/04/28 22:25:53.251 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|3] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] crashed: [rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec>:8890] > old view is: View[rooktwo(7630)<ec>:8888|2] members: > [rooktwo(7630)<ec>:8888{lead}, rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec>:8890, rooktwo(7630)<ec>:8887] > [warn 2016/04/28 22:25:53.251 PDT <Test worker> tid=0xa] total weight lost in > this view change is 40 of 65. Quorum has been lost! > [info 2016/04/28 22:25:53.257 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:25:53.258 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|0] members: [rooktwo(7630)<ec><v0>:8887{lead}] > old view is: null > [info 2016/04/28 22:25:53.259 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v0>:8887 > [info 2016/04/28 22:25:53.259 PDT <Geode Membership View Creator> tid=0x1e] > View Creator thread is starting > [info 2016/04/28 22:25:53.260 PDT <Geode Membership View Creator> tid=0x1e] > no recipients for new view aside from myself > [info 2016/04/28 22:25:53.762 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|1] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889] > old view is: View[rooktwo(7630)<ec><v0>:8887|0] members: > [rooktwo(7630)<ec><v0>:8887{lead}] > Current viewid 1 > [info 2016/04/28 22:25:53.763 PDT <Test worker> tid=0xa] received join > request from rooktwo(7630)<ec>:8890 > [info 2016/04/28 22:25:54.073 PDT <Geode Membership View Creator> tid=0x1e] > View Creator is processing 1 requests for the next membership view > [info 2016/04/28 22:25:54.074 PDT <Geode Membership View Creator> tid=0x1e] > preparing new view View[rooktwo(7630)<ec><v0>:8887|2] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec><v2>:8890] > failure detection ports: -1 -1 -1 98989 > Empty sleeps 1 stoppping: false > Empty sleeps 2 stoppping: false > Empty sleeps 3 stoppping: false > Empty sleeps 4 stoppping: false > Empty sleeps 5 stoppping: false > [info 2016/04/28 22:25:59.090 PDT <Geode Membership View Creator> tid=0x1e] > finished waiting for responses to view preparation > [warn 2016/04/28 22:25:59.091 PDT <Geode Membership View Creator> tid=0x1e] > these members failed to respond to the view change: [rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec><v2>:8890, rooktwo(7630)<ec>:8888] > [fatal 2016/04/28 22:25:59.123 PDT <Geode Membership View Creator> tid=0x1e] > Uncaught exception in thread Thread[Geode Membership View Creator,5,GemFire > Membership] > java.util.ConcurrentModificationException > at java.util.HashMap$HashIterator.nextNode(HashMap.java:1429) > at java.util.HashMap$KeyIterator.next(HashMap.java:1453) > at > com.gemstone.org.apache.logging.log4j.message.GemFireParameterizedMessage.recursiveDeepToString(GemFireParameterizedMessage.java:487) > at > com.gemstone.org.apache.logging.log4j.message.GemFireParameterizedMessage.deepToString(GemFireParameterizedMessage.java:374) > at > com.gemstone.org.apache.logging.log4j.message.GemFireParameterizedMessage.parseArguments(GemFireParameterizedMessage.java:165) > at > com.gemstone.org.apache.logging.log4j.message.GemFireParameterizedMessage.<init>(GemFireParameterizedMessage.java:122) > at > com.gemstone.org.apache.logging.log4j.message.GemFireParameterizedMessageFactory.newMessage(GemFireParameterizedMessageFactory.java:52) > at > org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:1021) > at > org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:980) > at > org.apache.logging.log4j.spi.AbstractLogger.info(AbstractLogger.java:679) > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave$ViewCreator.prepareAndSendView(GMSJoinLeave.java:2222) > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave$ViewCreator.createAndSendView(GMSJoinLeave.java:2137) > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave$ViewCreator.run(GMSJoinLeave.java:1955) > Empty sleeps 6 stoppping: false > Empty sleeps 7 stoppping: false > Empty sleeps 8 stoppping: false > Empty sleeps 9 stoppping: false > Empty sleeps 10 stoppping: false > Empty sleeps 11 stoppping: false > Empty sleeps 12 stoppping: false > Empty sleeps 13 stoppping: false > Empty sleeps 14 stoppping: false > Empty sleeps 15 stoppping: false > Empty sleeps 16 stoppping: false > Empty sleeps 17 stoppping: false > Empty sleeps 18 stoppping: false > Empty sleeps 19 stoppping: false > Empty sleeps 20 stoppping: false > Empty sleeps 21 stoppping: false > view requests: [] > current view: View[rooktwo(7630)<ec><v0>:8887|1] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889] > [info 2016/04/28 22:26:14.857 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v1>:8887|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec>:8890, rooktwo(7630)<ec>:8891, > rooktwo(7630)<ec><v1>:8887] > old view is: null > [info 2016/04/28 22:26:14.858 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec><v1>:8887 > [info 2016/04/28 22:26:14.859 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v1>:8887 > [info 2016/04/28 22:26:14.860 PDT <Geode Membership View Creator> tid=0x24] > View Creator thread is starting > [info 2016/04/28 22:26:14.879 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:26:14.891 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec>:8890, rooktwo(7630)<ec>:8887, > rooktwo(7630)<ec>:8891] > old view is: null > [info 2016/04/28 22:26:14.898 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec>:8889 for rooktwo(7630)<ec>:8889 > [info 2016/04/28 22:26:14.899 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec>:8890 for rooktwo(7630)<ec>:8890 > [info 2016/04/28 22:26:14.900 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|3] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887, rooktwo(7630)<ec>:8889, rooktwo(7630)<ec>:8891] > old view is: View[rooktwo(7630)<ec>:8888|1] members: > [rooktwo(7630)<ec>:8888{lead}, rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec>:8890, rooktwo(7630)<ec>:8887, rooktwo(7630)<ec>:8891] > [info 2016/04/28 22:26:14.902 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec>:8888 for rooktwo(7630)<ec>:8888 > [info 2016/04/28 22:26:14.903 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:14.905 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v9>:8887 > [info 2016/04/28 22:26:14.911 PDT <Geode Membership View Creator> tid=0x28] > View Creator thread is starting > [info 2016/04/28 22:26:14.921 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec>:8890, rooktwo(7630)<ec>:8887, > rooktwo(7630)<ec>:8891] > old view is: null > [info 2016/04/28 22:26:14.922 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec>:8889 for rooktwo(7630)<ec>:8889 > [info 2016/04/28 22:26:14.923 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec>:8890 for rooktwo(7630)<ec>:8890 > [info 2016/04/28 22:26:14.924 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec>:8888 for rooktwo(7630)<ec>:8888 > [info 2016/04/28 22:26:14.926 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:14.928 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v12>:8887 > [info 2016/04/28 22:26:14.930 PDT <Geode Membership View Creator> tid=0x2a] > View Creator thread is starting > [info 2016/04/28 22:26:14.938 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8887|1] members: [rooktwo(7630)<ec>:8887{lead}, > rooktwo(7630)<ec>:8888] > old view is: null > [info 2016/04/28 22:26:14.939 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec>:8889 for rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:14.940 PDT <Test worker> tid=0xa] I am being told to > leave the distributed system by rooktwo(7630)<ec>:8889 > [info 2016/04/28 22:26:14.948 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:14.948 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|0] members: [rooktwo(7630)<ec><v0>:8887{lead}] > old view is: null > [info 2016/04/28 22:26:14.949 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v0>:8887 > [info 2016/04/28 22:26:14.949 PDT <Test worker> tid=0xa] received join > request from rooktwo(7630)<ec>:8888 > [info 2016/04/28 22:26:14.950 PDT <Geode Membership View Creator> tid=0x2d] > View Creator thread is starting > [info 2016/04/28 22:26:14.954 PDT <Geode Membership View Creator> tid=0x2d] > no recipients for new view aside from myself > [info 2016/04/28 22:26:15.263 PDT <Geode Membership View Creator> tid=0x2d] > View Creator is processing 1 requests for the next membership view > [info 2016/04/28 22:26:15.265 PDT <Geode Membership View Creator> tid=0x2d] > preparing new view View[rooktwo(7630)<ec><v0>:8887|1] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec><v1>:8888] > failure detection ports: -1 -1 > [info 2016/04/28 22:26:15.555 PDT <Geode Membership View Creator> tid=0x2d] > finished waiting for responses to view preparation > [warn 2016/04/28 22:26:15.556 PDT <Geode Membership View Creator> tid=0x2d] > these members failed to respond to the view change: > [rooktwo(7630)<ec><v1>:8888] > [info 2016/04/28 22:26:15.557 PDT <Geode View Creator verification thread 1> > tid=0x2e] checking state of member rooktwo(7630)<ec><v1>:8888 > [info 2016/04/28 22:26:15.561 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|3] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec>:8890, rooktwo(7630)<ec>:8887, > rooktwo(7630)<ec>:8891] > old view is: null > [info 2016/04/28 22:26:15.565 PDT <Geode View Creator verification thread 1> > tid=0x2e] member rooktwo(7630)<ec><v1>:8888 failed availability check > [info 2016/04/28 22:26:15.572 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8887|1] members: [rooktwo(7630)<ec>:8887{lead}, > rooktwo(7630)<ec>:8888] > old view is: null > [info 2016/04/28 22:26:15.573 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:15.574 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v6>:8887 > [info 2016/04/28 22:26:15.574 PDT <Test worker> tid=0xa] Membership received > a request to remove rooktwo(7630)<ec>:8888 from rooktwo(7630)<ec>:8888 > reason=testing > [info 2016/04/28 22:26:15.575 PDT <Test worker> tid=0xa] Membership received > a request to remove rooktwo(7630)<ec>:8888 from rooktwo(7630)<ec>:8888 > reason=testing > [info 2016/04/28 22:26:15.576 PDT <Geode Membership View Creator> tid=0x31] > View Creator thread is starting > [info 2016/04/28 22:26:15.578 PDT <Geode Membership View Creator> tid=0x31] > preparing new view View[rooktwo(7630)<ec><v6>:8887|6] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889] > failure detection ports: -1 -1 -1 > [info 2016/04/28 22:26:20.591 PDT <Geode Membership View Creator> tid=0x31] > finished waiting for responses to view preparation > [warn 2016/04/28 22:26:20.592 PDT <Geode Membership View Creator> tid=0x31] > these members failed to respond to the view change: [rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec>:8888] > [info 2016/04/28 22:26:20.593 PDT <Geode View Creator verification thread 1> > tid=0x32] checking state of member rooktwo(7630)<ec>:8889 > [info 2016/04/28 22:26:20.596 PDT <Geode View Creator verification thread 1> > tid=0x32] member rooktwo(7630)<ec>:8889 failed availability check > [info 2016/04/28 22:26:25.570 PDT <Geode Membership View Creator> tid=0x31] > adding these unresponsive members to the crash-set for the next view: > [rooktwo(7630)<ec>:8889, rooktwo(7630)<ec>:8888] > [info 2016/04/28 22:26:25.572 PDT <Geode Membership View Creator> tid=0x31] > received new view: View[rooktwo(7630)<ec><v6>:8887|7] members: > [rooktwo(7630)<ec><v6>:8887{lead}] crashed: [rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec>:8888] > old view is: View[rooktwo(7630)<ec><v6>:8887|1] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889] > [info 2016/04/28 22:26:25.572 PDT <Geode Membership View Creator> tid=0x31] > rooktwo(7630)<ec>:8889 had a weight of 10 > [info 2016/04/28 22:26:25.573 PDT <Geode Membership View Creator> tid=0x31] > rooktwo(7630)<ec>:8888 had a weight of 10 > [warn 2016/04/28 22:26:25.573 PDT <Geode Membership View Creator> tid=0x31] > total weight lost in this view change is 20 of 35. Quorum has been lost! > [info 2016/04/28 22:26:25.576 PDT <Geode Membership View Creator> tid=0x31] > sending new view View[rooktwo(7630)<ec><v6>:8887|7] members: > [rooktwo(7630)<ec><v6>:8887{lead}] crashed: [rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec>:8888] > failure detection ports: -1 > [info 2016/04/28 22:26:25.884 PDT <Geode Membership View Creator> tid=0x31] > View Creator is processing 1 requests for the next membership view > [info 2016/04/28 22:26:26.604 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:26.604 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|0] members: [rooktwo(7630)<ec><v0>:8887{lead}] > old view is: null > [info 2016/04/28 22:26:26.605 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v0>:8887 > [info 2016/04/28 22:26:26.614 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec>:8890, rooktwo(7630)<ec><v0>:8887, > rooktwo(7630)<ec>:8891] > old view is: View[rooktwo(7630)<ec><v0>:8887|0] members: > [rooktwo(7630)<ec><v0>:8887{lead}] > [info 2016/04/28 22:26:26.616 PDT <Geode Membership View Creator> tid=0x34] > View Creator thread is starting > [info 2016/04/28 22:26:26.619 PDT <Geode Membership View Creator> tid=0x34] > no recipients for new view aside from myself > [info 2016/04/28 22:26:26.928 PDT <Geode Membership View Creator> tid=0x34] > View Creator is processing 3 requests for the next membership view > [info 2016/04/28 22:26:26.932 PDT <Geode Membership View Creator> tid=0x34] > rooktwo(7630)<ec>:8891 had a weight of 10 > [info 2016/04/28 22:26:26.932 PDT <Geode Membership View Creator> tid=0x34] > rooktwo(7630)<ec>:8889 had a weight of 10 > [info 2016/04/28 22:26:26.932 PDT <Geode Membership View Creator> tid=0x34] > rooktwo(7630)<ec>:8890 had a weight of 10 > [warn 2016/04/28 22:26:26.933 PDT <Geode Membership View Creator> tid=0x34] > total weight lost in this view change is 30 of 55. Quorum has been lost! > [info 2016/04/28 22:26:26.941 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:26:26.948 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:26.949 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v8>:8887 > [info 2016/04/28 22:26:26.950 PDT <Geode Membership View Creator> tid=0x36] > View Creator thread is starting > [info 2016/04/28 22:26:26.953 PDT <Geode Membership View Creator> tid=0x36] > preparing new view View[rooktwo(7630)<ec><v8>:8887|8] members: > [rooktwo(7630)<ec><v8>:8887{lead}] crashed: [rooktwo(7630)<ec>:8888] > failure detection ports: -1 > [info 2016/04/28 22:26:26.956 PDT <Geode Membership View Creator> tid=0x36] > finished waiting for responses to view preparation > [info 2016/04/28 22:26:26.956 PDT <Geode Membership View Creator> tid=0x36] > received new view: View[rooktwo(7630)<ec><v8>:8887|8] members: > [rooktwo(7630)<ec><v8>:8887{lead}] crashed: [rooktwo(7630)<ec>:8888] > old view is: View[rooktwo(7630)<ec>:8888|1] members: > [rooktwo(7630)<ec>:8888{lead}, rooktwo(7630)<ec><v8>:8887] > [info 2016/04/28 22:26:26.957 PDT <Geode Membership View Creator> tid=0x36] > rooktwo(7630)<ec>:8888 had a weight of 15 > [warn 2016/04/28 22:26:26.957 PDT <Geode Membership View Creator> tid=0x36] > total weight lost in this view change is 15 of 25. Quorum has been lost! > [info 2016/04/28 22:26:26.959 PDT <Geode Membership View Creator> tid=0x36] > sending new view View[rooktwo(7630)<ec><v8>:8887|8] members: > [rooktwo(7630)<ec><v8>:8887{lead}] crashed: [rooktwo(7630)<ec>:8888] > failure detection ports: -1 > [info 2016/04/28 22:26:27.557 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:26:27.558 PDT <Test worker> tid=0xa] Membership ignoring > removal request for rooktwo(7630)<ec>:8889 from non-member > rooktwo(7630)<ec>:9000 > [info 2016/04/28 22:26:27.563 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:27.564 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|0] members: [rooktwo(7630)<ec><v0>:8887{lead}] > old view is: null > [info 2016/04/28 22:26:27.564 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v0>:8887 > [fatal 2016/04/28 22:26:27.565 PDT <Test worker> tid=0xa] This member is no > longer in the membership view. My ID is rooktwo(7630)<ec><v0>:8887 and the > new view is View[rooktwo(7630)<ec>:8888|2] members: > [rooktwo(7630)<ec>:8888{lead}, rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec>:8890, rooktwo(7630)<ec>:8891] > [info 2016/04/28 22:26:27.565 PDT <Geode Membership View Creator> tid=0x39] > View Creator thread is starting > [info 2016/04/28 22:26:27.571 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:26:27.578 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:26:27.580 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec>:8889 for rooktwo(7630)<ec>:8889 > [info 2016/04/28 22:26:27.581 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:27.581 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v11>:8887 > [info 2016/04/28 22:26:27.584 PDT <Geode Membership View Creator> tid=0x3c] > View Creator thread is starting > [info 2016/04/28 22:26:27.589 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:27.590 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|0] members: [rooktwo(7630)<ec><v0>:8887{lead}] > old view is: null > [info 2016/04/28 22:26:27.591 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v0>:8887 > [info 2016/04/28 22:26:27.592 PDT <Geode Membership View Creator> tid=0x3e] > View Creator thread is starting > [info 2016/04/28 22:26:27.604 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:27.605 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|0] members: [rooktwo(7630)<ec><v0>:8887{lead}] > old view is: null > [info 2016/04/28 22:26:27.606 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v0>:8887 > [info 2016/04/28 22:26:27.609 PDT <Geode Membership View Creator> tid=0x40] > View Creator thread is starting > [info 2016/04/28 22:26:27.614 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:27.615 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|0] members: [rooktwo(7630)<ec><v0>:8887{lead}] > old view is: null > [info 2016/04/28 22:26:27.615 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v0>:8887 > [info 2016/04/28 22:26:27.616 PDT <Geode Membership View Creator> tid=0x42] > View Creator thread is starting > [info 2016/04/28 22:26:27.619 PDT <Geode Membership View Creator> tid=0x42] > no recipients for new view aside from myself > [info 2016/04/28 22:26:28.119 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|1] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889] > old view is: View[rooktwo(7630)<ec><v0>:8887|0] members: > [rooktwo(7630)<ec><v0>:8887{lead}] > Current viewid 1 > [info 2016/04/28 22:26:28.119 PDT <Test worker> tid=0xa] received join > request from rooktwo(7630)<ec>:8890 > [info 2016/04/28 22:26:28.429 PDT <Geode Membership View Creator> tid=0x42] > View Creator is processing 1 requests for the next membership view > [info 2016/04/28 22:26:28.430 PDT <Geode Membership View Creator> tid=0x42] > preparing new view View[rooktwo(7630)<ec><v0>:8887|2] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec><v2>:8890] > failure detection ports: -1 -1 -1 98989 > Empty sleeps 1 stoppping: false > Empty sleeps 2 stoppping: false > Empty sleeps 3 stoppping: false > Empty sleeps 4 stoppping: false > Empty sleeps 5 stoppping: false > [info 2016/04/28 22:26:33.446 PDT <Geode Membership View Creator> tid=0x42] > finished waiting for responses to view preparation > [warn 2016/04/28 22:26:33.446 PDT <Geode Membership View Creator> tid=0x42] > these members failed to respond to the view change: [rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec><v2>:8890, rooktwo(7630)<ec>:8888] > [info 2016/04/28 22:26:33.448 PDT <Geode View Creator verification thread 3> > tid=0x45] received leave request from rooktwo(7630)<ec>:8888 for > rooktwo(7630)<ec>:8888 > [info 2016/04/28 22:26:33.450 PDT <Geode Membership View Creator> tid=0x42] > received new view: View[rooktwo(7630)<ec><v0>:8887|2] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec><v2>:8890] > old view is: View[rooktwo(7630)<ec><v0>:8887|1] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889] > [info 2016/04/28 22:26:33.451 PDT <Geode Membership View Creator> tid=0x42] > sending new view View[rooktwo(7630)<ec><v0>:8887|2] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec><v2>:8890] > failure detection ports: -1 -1 -1 98989 > [info 2016/04/28 22:26:33.452 PDT <Geode Membership View Creator> tid=0x42] > View Creator is processing 1 requests for the next membership view > [info 2016/04/28 22:26:33.453 PDT <Geode Membership View Creator> tid=0x42] > preparing new view View[rooktwo(7630)<ec><v0>:8887|3] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec><v2>:8890] shutdown: [rooktwo(7630)<ec>:8888] > failure detection ports: -1 -1 98989 > Empty sleeps 6 stoppping: false > [info 2016/04/28 22:26:34.139 PDT <Geode Membership View Creator> tid=0x42] > finished waiting for responses to view preparation > [warn 2016/04/28 22:26:34.139 PDT <Geode Membership View Creator> tid=0x42] > these members failed to respond to the view change: [rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec><v2>:8890] > [info 2016/04/28 22:26:34.140 PDT <Geode View Creator verification thread 1> > tid=0x46] checking state of member rooktwo(7630)<ec>:8889 > [info 2016/04/28 22:26:34.144 PDT <Geode View Creator verification thread 1> > tid=0x46] member rooktwo(7630)<ec>:8889 failed availability check > [info 2016/04/28 22:26:34.144 PDT <Geode View Creator verification thread 2> > tid=0x47] checking state of member rooktwo(7630)<ec><v2>:8890 > [info 2016/04/28 22:26:34.146 PDT <Geode View Creator verification thread 2> > tid=0x47] member rooktwo(7630)<ec><v2>:8890 failed availability check > [info 2016/04/28 22:26:34.152 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:34.152 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|0] members: [rooktwo(7630)<ec><v0>:8887{lead}] > old view is: null > [info 2016/04/28 22:26:34.153 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v0>:8887 > [info 2016/04/28 22:26:34.155 PDT <Geode Membership View Creator> tid=0x49] > View Creator thread is starting > [info 2016/04/28 22:26:34.156 PDT <Geode Membership View Creator> tid=0x49] > no recipients for new view aside from myself > [info 2016/04/28 22:26:34.657 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|1] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec>:8890] > old view is: View[rooktwo(7630)<ec><v0>:8887|0] members: > [rooktwo(7630)<ec><v0>:8887{lead}] > [info 2016/04/28 22:26:34.657 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec>:8889 for rooktwo(7630)<ec>:8889 > [info 2016/04/28 22:26:39.702 PDT <Geode Membership View Creator> tid=0x49] > View Creator is processing 1 requests for the next membership view > [info 2016/04/28 22:26:39.703 PDT <Geode Membership View Creator> tid=0x49] > preparing new view View[rooktwo(7630)<ec><v0>:8887|2] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8890] shutdown: > [rooktwo(7630)<ec>:8889] > failure detection ports: -1 -1 > [info 2016/04/28 22:26:44.715 PDT <Geode Membership View Creator> tid=0x49] > finished waiting for responses to view preparation > [warn 2016/04/28 22:26:44.715 PDT <Geode Membership View Creator> tid=0x49] > these members failed to respond to the view change: [rooktwo(7630)<ec>:8890] > [info 2016/04/28 22:26:44.717 PDT <Geode View Creator verification thread 1> > tid=0x4a] checking state of member rooktwo(7630)<ec>:8890 > [info 2016/04/28 22:26:44.719 PDT <Geode View Creator verification thread 1> > tid=0x4a] member rooktwo(7630)<ec>:8890 passed availability check > [info 2016/04/28 22:26:44.720 PDT <Geode Membership View Creator> tid=0x49] > received new view: View[rooktwo(7630)<ec><v0>:8887|2] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8890] shutdown: > [rooktwo(7630)<ec>:8889] > old view is: View[rooktwo(7630)<ec><v0>:8887|1] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec>:8890] > [info 2016/04/28 22:26:44.722 PDT <Geode Membership View Creator> tid=0x49] > sending new view View[rooktwo(7630)<ec><v0>:8887|2] members: > [rooktwo(7630)<ec><v0>:8887{lead}, rooktwo(7630)<ec>:8890] shutdown: > [rooktwo(7630)<ec>:8889] > failure detection ports: -1 -1 > [info 2016/04/28 22:26:45.226 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8887|1] members: [rooktwo(7630)<ec>:8887{lead}, > rooktwo(7630)<ec>:8888] > old view is: null > [info 2016/04/28 22:26:45.227 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:45.227 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v6>:8887 > [info 2016/04/28 22:26:45.228 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec>:8888 for rooktwo(7630)<ec>:8888 > [info 2016/04/28 22:26:45.228 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec>:8888 for rooktwo(7630)<ec>:8888 > [info 2016/04/28 22:26:45.229 PDT <Geode Membership View Creator> tid=0x4c] > View Creator thread is starting > [info 2016/04/28 22:26:45.231 PDT <Geode Membership View Creator> tid=0x4c] > preparing new view View[rooktwo(7630)<ec><v6>:8887|6] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888] > failure detection ports: -1 -1 > [info 2016/04/28 22:26:45.233 PDT <Geode Membership View Creator> tid=0x4c] > finished waiting for responses to view preparation > [info 2016/04/28 22:26:45.234 PDT <Geode Membership View Creator> tid=0x4c] > received new view: View[rooktwo(7630)<ec><v6>:8887|6] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888] > old view is: View[rooktwo(7630)<ec><v6>:8887|1] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888] > [info 2016/04/28 22:26:45.235 PDT <Geode Membership View Creator> tid=0x4c] > sending new view View[rooktwo(7630)<ec><v6>:8887|6] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888] > failure detection ports: -1 -1 > [info 2016/04/28 22:26:45.545 PDT <Geode Membership View Creator> tid=0x4c] > View Creator is processing 2 requests for the next membership view > [info 2016/04/28 22:26:45.545 PDT <Geode Membership View Creator> tid=0x4c] > received new view: View[rooktwo(7630)<ec><v6>:8887|7] members: > [rooktwo(7630)<ec><v6>:8887{lead}] shutdown: [rooktwo(7630)<ec>:8888] > old view is: View[rooktwo(7630)<ec><v6>:8887|6] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888] > [info 2016/04/28 22:26:45.546 PDT <Geode Membership View Creator> tid=0x4c] > no recipients for new view aside from myself > [info 2016/04/28 22:26:46.236 PDT <Test worker> tid=0xa] received join > request from rooktwo(7630)<ec>:8700 > [warn 2016/04/28 22:26:46.237 PDT <Test worker> tid=0xa] detected an attempt > to start a peer using an older version of the product rooktwo(7630)<ec>:8700 > [info 2016/04/28 22:26:46.243 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:26:46.244 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec><v0>:8889 for rooktwo(7630)<ec><v0>:8889 > [info 2016/04/28 22:26:46.251 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:26:46.252 PDT <Test worker> tid=0xa] Membership received > a request to remove rooktwo(7630)<ec>:8888 from rooktwo(7630)<ec>:8888 > reason=testing > [info 2016/04/28 22:26:46.253 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:46.253 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v15>:8887 > [info 2016/04/28 22:26:46.256 PDT <Geode Membership View Creator> tid=0x50] > View Creator thread is starting > [info 2016/04/28 22:26:46.260 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:46.261 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|0] members: [rooktwo(7630)<ec><v0>:8887{lead}] > old view is: null > [info 2016/04/28 22:26:46.261 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v0>:8887 > [info 2016/04/28 22:26:46.263 PDT <Geode Membership View Creator> tid=0x52] > View Creator thread is starting > [info 2016/04/28 22:26:46.263 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec>:8890, rooktwo(7630)<ec><v0>:8887, > rooktwo(7630)<ec>:8891] > old view is: View[rooktwo(7630)<ec><v0>:8887|0] members: > [rooktwo(7630)<ec><v0>:8887{lead}] > [info 2016/04/28 22:26:46.269 PDT <Test worker> tid=0xa] received join > request from rooktwo(7630)<ec>:8888 > [info 2016/04/28 22:26:46.277 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:46.277 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec><v0>:8887|0] members: [rooktwo(7630)<ec><v0>:8887{lead}] > old view is: null > [info 2016/04/28 22:26:46.278 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v0>:8887 > [info 2016/04/28 22:26:46.283 PDT <Geode Membership View Creator> tid=0x55] > View Creator thread is starting > [info 2016/04/28 22:26:46.287 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:26:46.287 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|2] members: [rooktwo(7630)<ec>:8887{lead}, > rooktwo(7630)<ec>:8890] > old view is: View[rooktwo(7630)<ec>:8888|1] members: > [rooktwo(7630)<ec>:8888{lead}, rooktwo(7630)<ec>:8887] > [info 2016/04/28 22:26:46.288 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:46.288 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v13>:8887 > [info 2016/04/28 22:26:46.289 PDT <Geode Membership View Creator> tid=0x57] > View Creator thread is starting > [info 2016/04/28 22:26:46.293 PDT <Geode Membership View Creator> tid=0x57] > preparing new view View[rooktwo(7630)<ec><v13>:8887|13] members: > [rooktwo(7630)<ec><v13>:8887{lead}, rooktwo(7630)<ec>:8890] > failure detection ports: -1 -1 > [info 2016/04/28 22:26:51.303 PDT <Geode Membership View Creator> tid=0x57] > finished waiting for responses to view preparation > [warn 2016/04/28 22:26:51.304 PDT <Geode Membership View Creator> tid=0x57] > these members failed to respond to the view change: [rooktwo(7630)<ec>:8890] > [info 2016/04/28 22:26:51.305 PDT <Geode View Creator verification thread 1> > tid=0x58] checking state of member rooktwo(7630)<ec>:8890 > [info 2016/04/28 22:26:51.307 PDT <Geode View Creator verification thread 1> > tid=0x58] member rooktwo(7630)<ec>:8890 failed availability check > [info 2016/04/28 22:26:56.280 PDT <Geode Membership View Creator> tid=0x57] > adding these unresponsive members to the crash-set for the next view: > [rooktwo(7630)<ec>:8890] > [info 2016/04/28 22:26:56.281 PDT <Geode Membership View Creator> tid=0x57] > received new view: View[rooktwo(7630)<ec><v13>:8887|14] members: > [rooktwo(7630)<ec><v13>:8887{lead}] crashed: [rooktwo(7630)<ec>:8890] > old view is: View[rooktwo(7630)<ec>:8888|2] members: > [rooktwo(7630)<ec><v13>:8887{lead}, rooktwo(7630)<ec>:8890] > [info 2016/04/28 22:26:56.281 PDT <Geode Membership View Creator> tid=0x57] > rooktwo(7630)<ec>:8890 had a weight of 10 > [info 2016/04/28 22:26:56.283 PDT <Geode Membership View Creator> tid=0x57] > sending new view View[rooktwo(7630)<ec><v13>:8887|14] members: > [rooktwo(7630)<ec><v13>:8887{lead}] crashed: [rooktwo(7630)<ec>:8890] > failure detection ports: -1 > [info 2016/04/28 22:26:56.305 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8887|1] members: [rooktwo(7630)<ec>:8887{lead}, > rooktwo(7630)<ec>:8888] > old view is: null > [info 2016/04/28 22:26:56.307 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:26:56.308 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v6>:8887 > [info 2016/04/28 22:26:56.311 PDT <Test worker> tid=0xa] received join > request from rooktwo(7630)<ec>:8890 > [info 2016/04/28 22:26:56.311 PDT <Test worker> tid=0xa] received join > request from rooktwo(7630)<ec>:8890 > [info 2016/04/28 22:26:56.312 PDT <Geode Membership View Creator> tid=0x5a] > View Creator thread is starting > [info 2016/04/28 22:26:56.314 PDT <Geode Membership View Creator> tid=0x5a] > preparing new view View[rooktwo(7630)<ec><v6>:8887|6] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889] > failure detection ports: -1 -1 -1 > [info 2016/04/28 22:27:01.325 PDT <Geode Membership View Creator> tid=0x5a] > finished waiting for responses to view preparation > [warn 2016/04/28 22:27:01.326 PDT <Geode Membership View Creator> tid=0x5a] > these members failed to respond to the view change: [rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec>:8888] > [info 2016/04/28 22:27:01.327 PDT <Geode View Creator verification thread 1> > tid=0x5b] checking state of member rooktwo(7630)<ec>:8889 > [info 2016/04/28 22:27:01.328 PDT <Geode View Creator verification thread 2> > tid=0x5c] checking state of member rooktwo(7630)<ec>:8888 > [info 2016/04/28 22:27:01.330 PDT <Geode View Creator verification thread 2> > tid=0x5c] member rooktwo(7630)<ec>:8888 passed availability check > [info 2016/04/28 22:27:01.331 PDT <Geode View Creator verification thread 1> > tid=0x5b] member rooktwo(7630)<ec>:8889 passed availability check > [info 2016/04/28 22:27:01.333 PDT <Geode Membership View Creator> tid=0x5a] > received new view: View[rooktwo(7630)<ec><v6>:8887|6] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889] > old view is: View[rooktwo(7630)<ec><v6>:8887|1] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889] > [info 2016/04/28 22:27:01.334 PDT <Geode Membership View Creator> tid=0x5a] > sending new view View[rooktwo(7630)<ec><v6>:8887|6] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889] > failure detection ports: -1 -1 -1 > [info 2016/04/28 22:27:01.643 PDT <Geode Membership View Creator> tid=0x5a] > View Creator is processing 2 requests for the next membership view > [info 2016/04/28 22:27:01.645 PDT <Geode Membership View Creator> tid=0x5a] > preparing new view View[rooktwo(7630)<ec><v6>:8887|7] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec><v7>:8890] > failure detection ports: -1 -1 -1 -1 > [info 2016/04/28 22:27:06.660 PDT <Geode Membership View Creator> tid=0x5a] > finished waiting for responses to view preparation > [warn 2016/04/28 22:27:06.661 PDT <Geode Membership View Creator> tid=0x5a] > these members failed to respond to the view change: [rooktwo(7630)<ec>:8889, > rooktwo(7630)<ec><v7>:8890, rooktwo(7630)<ec>:8888] > [info 2016/04/28 22:27:06.662 PDT <Geode View Creator verification thread 1> > tid=0x5d] checking state of member rooktwo(7630)<ec>:8889 > [info 2016/04/28 22:27:06.663 PDT <Geode View Creator verification thread 2> > tid=0x5e] checking state of member rooktwo(7630)<ec><v7>:8890 > [info 2016/04/28 22:27:06.663 PDT <Geode View Creator verification thread 3> > tid=0x5f] checking state of member rooktwo(7630)<ec>:8888 > [info 2016/04/28 22:27:06.663 PDT <Geode View Creator verification thread 1> > tid=0x5d] member rooktwo(7630)<ec>:8889 passed availability check > [info 2016/04/28 22:27:06.665 PDT <Geode View Creator verification thread 3> > tid=0x5f] member rooktwo(7630)<ec>:8888 passed availability check > [info 2016/04/28 22:27:06.665 PDT <Geode View Creator verification thread 2> > tid=0x5e] member rooktwo(7630)<ec><v7>:8890 passed availability check > [info 2016/04/28 22:27:06.667 PDT <Geode Membership View Creator> tid=0x5a] > received new view: View[rooktwo(7630)<ec><v6>:8887|7] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec><v7>:8890] > old view is: View[rooktwo(7630)<ec><v6>:8887|6] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889] > [info 2016/04/28 22:27:06.670 PDT <Geode Membership View Creator> tid=0x5a] > sending new view View[rooktwo(7630)<ec><v6>:8887|7] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec>:8889, rooktwo(7630)<ec><v7>:8890] > failure detection ports: -1 -1 -1 -1 > [info 2016/04/28 22:27:07.340 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8887|1] members: [rooktwo(7630)<ec>:8887{lead}, > rooktwo(7630)<ec>:8888] > old view is: null > [info 2016/04/28 22:27:07.341 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:27:07.342 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v6>:8887 > [info 2016/04/28 22:27:07.343 PDT <Geode Membership View Creator> tid=0x61] > View Creator thread is starting > [info 2016/04/28 22:27:07.347 PDT <Geode Membership View Creator> tid=0x61] > preparing new view View[rooktwo(7630)<ec><v6>:8887|7] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec><v6>:8889] > failure detection ports: -1 -1 -1 > [info 2016/04/28 22:27:09.346 PDT <Geode Membership View Creator> tid=0x61] > finished waiting for responses to view preparation > [info 2016/04/28 22:27:09.348 PDT <Geode Membership View Creator> tid=0x61] > received new view: View[rooktwo(7630)<ec><v6>:8887|7] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec><v6>:8889] > old view is: View[rooktwo(7630)<ec><v6>:8887|1] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888] > [info 2016/04/28 22:27:09.352 PDT <Geode Membership View Creator> tid=0x61] > sending new view View[rooktwo(7630)<ec><v6>:8887|7] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec><v6>:8889] > failure detection ports: -1 -1 -1 > new view is View[rooktwo(7630)<ec><v6>:8887|7] members: > [rooktwo(7630)<ec><v6>:8887{lead}, rooktwo(7630)<ec>:8888, > rooktwo(7630)<ec><v6>:8889] > [info 2016/04/28 22:27:10.358 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [info 2016/04/28 22:27:10.360 PDT <Test worker> tid=0xa] received leave > request from rooktwo(7630)<ec>:8888 for rooktwo(7630)<ec>:8888 > [info 2016/04/28 22:27:10.360 PDT <Test worker> tid=0xa] This member is > becoming the membership coordinator with address rooktwo(7630)<ec>:8887 > [info 2016/04/28 22:27:10.361 PDT <Test worker> tid=0xa] ViewCreator starting > on:rooktwo(7630)<ec><v7>:8887 > [info 2016/04/28 22:27:10.363 PDT <Geode Membership View Creator> tid=0x63] > View Creator thread is starting > [info 2016/04/28 22:27:10.369 PDT <Test worker> tid=0xa] received new view: > View[rooktwo(7630)<ec>:8888|1] members: [rooktwo(7630)<ec>:8888{lead}, > rooktwo(7630)<ec>:8887] > old view is: null > [fatal 2016/04/28 22:27:10.370 PDT <Test worker> tid=0xa] This member is no > longer in the membership view. My ID is rooktwo(7630)<ec>:8887 and the new > view is View[rooktwo(7630)<ec>:8888|2] members: > [rooktwo(7630)<ec>:8889{lead}, rooktwo(7630)<ec>:8890, rooktwo(7630)<ec>:8891] > Standard Error > com.gemstone.gemfire.security.AuthenticationFailedException: we want to fail > auth here > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave.processJoinRequest(GMSJoinLeave.java:477) > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave.processMessage(GMSJoinLeave.java:1524) > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeaveJUnitTest.testProcessJoinMessageWithAuthenticationButNullCredentials(GMSJoinLeaveJUnitTest.java:245) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) > at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) > at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) > at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) > at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) > at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) > at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) > at org.junit.runners.ParentRunner.run(ParentRunner.java:363) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.runTestClass(JUnitTestClassExecuter.java:105) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.execute(JUnitTestClassExecuter.java:56) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassProcessor.processTestClass(JUnitTestClassProcessor.java:64) > at > org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:50) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.messaging.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32) > at > org.gradle.messaging.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93) > at com.sun.proxy.$Proxy2.processTestClass(Unknown Source) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:106) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.messaging.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:360) > at > org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:54) > at > org.gradle.internal.concurrent.StoppableExecutorImpl$1.run(StoppableExecutorImpl.java:40) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > [severe 2016/04/28 22:25:59.103 PDT <Geode Membership View Creator> tid=0x1e] > Uncaught exception in thread Thread[Geode Membership View Creator,5,GemFire > Membership] > java.util.ConcurrentModificationException > at java.util.HashMap$HashIterator.nextNode(HashMap.java:1429) > at java.util.HashMap$KeyIterator.next(HashMap.java:1453) > at > com.gemstone.org.apache.logging.log4j.message.GemFireParameterizedMessage.recursiveDeepToString(GemFireParameterizedMessage.java:487) > at > com.gemstone.org.apache.logging.log4j.message.GemFireParameterizedMessage.deepToString(GemFireParameterizedMessage.java:374) > at > com.gemstone.org.apache.logging.log4j.message.GemFireParameterizedMessage.parseArguments(GemFireParameterizedMessage.java:165) > at > com.gemstone.org.apache.logging.log4j.message.GemFireParameterizedMessage.<init>(GemFireParameterizedMessage.java:122) > at > com.gemstone.org.apache.logging.log4j.message.GemFireParameterizedMessageFactory.newMessage(GemFireParameterizedMessageFactory.java:52) > at > org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:1021) > at > org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:980) > at > org.apache.logging.log4j.spi.AbstractLogger.info(AbstractLogger.java:679) > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave$ViewCreator.prepareAndSendView(GMSJoinLeave.java:2222) > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave$ViewCreator.createAndSendView(GMSJoinLeave.java:2137) > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave$ViewCreator.run(GMSJoinLeave.java:1955) > com.gemstone.gemfire.security.AuthenticationFailedException: we want to fail > auth here > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave.processJoinRequest(GMSJoinLeave.java:477) > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeave.processMessage(GMSJoinLeave.java:1524) > at > com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeaveJUnitTest.testProcessJoinMessageWithBadAuthentication(GMSJoinLeaveJUnitTest.java:233) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) > at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) > at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) > at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) > at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) > at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) > at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) > at org.junit.runners.ParentRunner.run(ParentRunner.java:363) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.runTestClass(JUnitTestClassExecuter.java:105) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.execute(JUnitTestClassExecuter.java:56) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassProcessor.processTestClass(JUnitTestClassProcessor.java:64) > at > org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:50) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.messaging.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32) > at > org.gradle.messaging.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93) > at com.sun.proxy.$Proxy2.processTestClass(Unknown Source) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:106) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.messaging.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:360) > at > org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:54) > at > org.gradle.internal.concurrent.StoppableExecutorImpl$1.run(StoppableExecutorImpl.java:40) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > {noformat} -- This message was sent by Atlassian JIRA (v6.3.4#6332)