[ https://issues.apache.org/jira/browse/GEODE-6326?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16754417#comment-16754417 ]
Bruce Schuchardt commented on GEODE-6326: ----------------------------------------- This test has failed in the past with other Region configurations and the tickets were closed when the problem couldn't be reproduced: https://issues.apache.org/jira/browse/GEODE-976 (DistributedNoAckRegionCCEDUnitTest) https://issues.apache.org/jira/browse/GEODE-3451 (DistributedAckRegionCCEOffHeapDUnitTest) Comments in 3451 suggest this is a machine resource issue but I see no evidence of that in the log for this test run. The awaitlity thread shows the inconsistency over and over again. The test creates an empty Region in vm0 and non-empty replicates in vm1 and vm3. Then it starts some async puts in vm0 and vm1. During the async operations it creates a new replicate in vm2. It is vm2 that is out of sync with the other caches, so this seems to be a problem with initial-image transfer. I am adding code to dump the version information on the region in the hosting JVMs when the test fails. > versionTestConcurrentEventsOnEmptyRegion fails in CI in multiple > configurations > ------------------------------------------------------------------------------- > > Key: GEODE-6326 > URL: https://issues.apache.org/jira/browse/GEODE-6326 > Project: Geode > Issue Type: Bug > Components: regions > Reporter: Dale Emery > Assignee: Bruce Schuchardt > Priority: Major > > DistributedAckOverflowRegionCCEDUnitTest.versionTestConcurrentEventsOnEmptyRegion > failed in CI: > [https://concourse.apachegeode-ci.info/teams/main/pipelines/apache-develop-main/jobs/DistributedTestOpenJDK8/builds/335] > [http://files.apachegeode-ci.info/builds/apache-develop-main/1.9.0-SNAPSHOT.0375/test-results/distributedTest/1548465595/classes/org.apache.geode.cache30.DistributedAckOverflowRegionCCEDUnitTest.html#testConcurrentEventsOnEmptyRegion]: > {noformat} > org.awaitility.core.ConditionTimeoutException: Condition with alias 'Wait for > the members to eventually be consistent' didn't complete within 300 seconds > because assertion condition defined as a lambda expression in > org.apache.geode.cache30.MultiVMRegionTestCase that uses > org.apache.geode.test.dunit.VM, > org.apache.geode.test.dunit.VMorg.apache.geode.test.dunit.VM, > org.apache.geode.test.dunit.VMorg.apache.geode.test.dunit.VM [r2 contents are > not consistent with r1 for cckey2] expected:<"ccvalue[-100586808]"> but > was:<"ccvalue[1574152144]">. > at org.awaitility.core.ConditionAwaiter.await(ConditionAwaiter.java:145) > at > org.awaitility.core.AssertionCondition.await(AssertionCondition.java:122) > at > org.awaitility.core.AssertionCondition.await(AssertionCondition.java:32) > at org.awaitility.core.ConditionFactory.until(ConditionFactory.java:902) > at > org.awaitility.core.ConditionFactory.untilAsserted(ConditionFactory.java:723) > at > org.apache.geode.cache30.MultiVMRegionTestCase.versionTestConcurrentEventsOnEmptyRegion(MultiVMRegionTestCase.java:8282) > at > org.apache.geode.cache30.DistributedAckRegionCCEDUnitTest.testConcurrentEventsOnEmptyRegion(DistributedAckRegionCCEDUnitTest.java:421) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) > at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55) > at org.junit.rules.RunRules.evaluate(RunRules.java:20) > at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) > at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) > at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) > at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) > at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) > at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) > at > org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) > at org.junit.runners.ParentRunner.run(ParentRunner.java:363) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.runTestClass(JUnitTestClassExecutor.java:110) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:58) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:38) > at > org.gradle.api.internal.tasks.testing.junit.AbstractJUnitTestClassProcessor.processTestClass(AbstractJUnitTestClassProcessor.java:66) > at > org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:51) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32) > at > org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93) > at com.sun.proxy.$Proxy2.processTestClass(Unknown Source) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:118) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:175) > at > org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:157) > at > org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:404) > at > org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:63) > at > org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:46) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at > org.gradle.internal.concurrent.ThreadFactoryImpl$ManagedThreadRunnable.run(ThreadFactoryImpl.java:55) > at java.lang.Thread.run(Thread.java:748) > Caused by: org.junit.ComparisonFailure: [r2 contents are not consistent with > r1 for cckey2] expected:<"ccvalue[-100586808]"> but > was:<"ccvalue[1574152144]"> > at sun.reflect.GeneratedConstructorAccessor18.newInstance(Unknown > Source) > at > sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) > at > org.apache.geode.cache30.MultiVMRegionTestCase.lambda$versionTestConcurrentEventsOnEmptyRegion$5(MultiVMRegionTestCase.java:8293) > at > org.awaitility.core.AssertionCondition$1.eval(AssertionCondition.java:55) > at > org.awaitility.core.ConditionAwaiter$ConditionPoller.call(ConditionAwaiter.java:192) > at > org.awaitility.core.ConditionAwaiter$ConditionPoller.call(ConditionAwaiter.java:179) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > ... 1 more > {noformat} -- This message was sent by Atlassian JIRA (v7.6.3#76005)