[ https://issues.apache.org/jira/browse/IGNITE-11683?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Ivan Bessonov updated IGNITE-11683: ----------------------------------- Description: The problem occurs right after this line: {code:java} assertTrue(GridTestUtils.waitForCondition(() -> metastorage(1).getUpdatesCount() == expUpdatesCnt, 15_000)); {code} Client node might not be fully reconnected yet. Adding following line resolves the problem in the particular test: {code:java} grid(1).cluster().clientReconnectFuture().get(); {code} I don't consider this a proper fix. Stopping the client that hasn't finished its reconnect shouldn't result in deadlock. Client node should be stopped successfully. Stack traces of hanging client node: {code:java} Thread [name="test-runner-#673%metastorage.DistributedMetaStoragePersistentTest%", id=823, state=WAITING, blockCnt=60, waitCnt=200] Lock [object=java.lang.Object@2a2d45ba, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at o.a.i.i.util.worker.GridWorker.join(GridWorker.java:243) at o.a.i.i.util.IgniteUtils.join(IgniteUtils.java:4831) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager.onKernalStop0(GridCachePartitionExchangeManager.java:815) at o.a.i.i.processors.cache.GridCacheSharedManagerAdapter.onKernalStop(GridCacheSharedManagerAdapter.java:120) at o.a.i.i.processors.cache.GridCacheProcessor.onKernalStop(GridCacheProcessor.java:1183) at o.a.i.i.IgniteKernal.stop0(IgniteKernal.java:2321) at o.a.i.i.IgniteKernal.stop(IgniteKernal.java:2269) at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2574) - locked o.a.i.i.IgnitionEx$IgniteNamedInstance@69d9c55 at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2537) at o.a.i.i.IgnitionEx.stop(IgnitionEx.java:330) at o.a.i.Ignition.stop(Ignition.java:223) at o.a.i.testframework.junits.GridAbstractTest.stopGrid(GridAbstractTest.java:1153) at o.a.i.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1193) at o.a.i.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1174) at o.a.i.i.processors.metastorage.DistributedMetaStorageTest.after(DistributedMetaStorageTest.java:85) at o.a.i.i.processors.metastorage.DistributedMetaStoragePersistentTest.after(DistributedMetaStoragePersistentTest.java:67) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at o.a.i.testframework.junits.GridAbstractTest$6.run(GridAbstractTest.java:2044) at java.lang.Thread.run(Thread.java:748) {code} {code:java} Thread [name="exchange-worker-#893%metastorage.DistributedMetaStoragePersistentTest1%", id=1088, state=BLOCKED, blockCnt=2, waitCnt=2] Lock [object=o.a.i.i.IgnitionEx$IgniteNamedInstance@69d9c55, ownerName=test-runner-#673%metastorage.DistributedMetaStoragePersistentTest%, ownerId=823] at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2545) at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2537) at o.a.i.i.IgnitionEx.stop(IgnitionEx.java:330) at o.a.i.Ignition.stop(Ignition.java:223) at o.a.i.i.IgniteKernal.close(IgniteKernal.java:3626) at o.a.i.i.IgniteKernal$5.apply(IgniteKernal.java:4086) at o.a.i.i.IgniteKernal$5.apply(IgniteKernal.java:4066) at o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) at o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:125) at o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45) at o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$7.apply(GridCachePartitionExchangeManager.java:723) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$7.apply(GridCachePartitionExchangeManager.java:709) at o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2263) at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:145) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.init(GridDhtPartitionsExchangeFuture.java:903) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body0(GridCachePartitionExchangeManager.java:2958) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body(GridCachePartitionExchangeManager.java:2807) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) {code} was: The problem occurs right after this line: {code:java} assertTrue(GridTestUtils.waitForCondition(() -> metastorage(1).getUpdatesCount() == expUpdatesCnt, 15_000)); {code} Client node might not be fully reconnected yet. Adding following line resolves the problem in the particular test: {code:java} grid(1).cluster().clientReconnectFuture().get(); {code} I don't consider this a proper fix. Stopping the client that hasn't finished its reconnect shouldn't result in deadlock. Client node should be stopped successfully. Stack traces of hanging client node: {code:java} Thread [name="test-runner-#673%metastorage.DistributedMetaStoragePersistentTest%", id=823, state=WAITING, blockCnt=60, waitCnt=200] Lock [object=java.lang.Object@2a2d45ba, ownerName=null, ownerId=-1] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at o.a.i.i.util.worker.GridWorker.join(GridWorker.java:243) at o.a.i.i.util.IgniteUtils.join(IgniteUtils.java:4831) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager.onKernalStop0(GridCachePartitionExchangeManager.java:815) at o.a.i.i.processors.cache.GridCacheSharedManagerAdapter.onKernalStop(GridCacheSharedManagerAdapter.java:120) at o.a.i.i.processors.cache.GridCacheProcessor.onKernalStop(GridCacheProcessor.java:1183) at o.a.i.i.IgniteKernal.stop0(IgniteKernal.java:2321) at o.a.i.i.IgniteKernal.stop(IgniteKernal.java:2269) at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2574) - locked o.a.i.i.IgnitionEx$IgniteNamedInstance@69d9c55 at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2537) at o.a.i.i.IgnitionEx.stop(IgnitionEx.java:330) at o.a.i.Ignition.stop(Ignition.java:223) at o.a.i.testframework.junits.GridAbstractTest.stopGrid(GridAbstractTest.java:1153) at o.a.i.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1193) at o.a.i.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1174) at o.a.i.i.processors.metastorage.DistributedMetaStorageTest.after(DistributedMetaStorageTest.java:85) at o.a.i.i.processors.metastorage.DistributedMetaStoragePersistentTest.after(DistributedMetaStoragePersistentTest.java:67) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at o.a.i.testframework.junits.GridAbstractTest$6.run(GridAbstractTest.java:2044) at java.lang.Thread.run(Thread.java:748) {code} {code:java} Thread [name="exchange-worker-#893%metastorage.DistributedMetaStoragePersistentTest1%", id=1088, state=BLOCKED, blockCnt=2, waitCnt=2] Lock [object=o.a.i.i.IgnitionEx$IgniteNamedInstance@69d9c55, ownerName=test-runner-#673%metastorage.DistributedMetaStoragePersistentTest%, ownerId=823] at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2545) at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2537) at o.a.i.i.IgnitionEx.stop(IgnitionEx.java:330) at o.a.i.Ignition.stop(Ignition.java:223) at o.a.i.i.IgniteKernal.close(IgniteKernal.java:3626) at o.a.i.i.IgniteKernal$5.apply(IgniteKernal.java:4086) at o.a.i.i.IgniteKernal$5.apply(IgniteKernal.java:4066) at o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) at o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:125) at o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45) at o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$7.apply(GridCachePartitionExchangeManager.java:723) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$7.apply(GridCachePartitionExchangeManager.java:709) at o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) at o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2263) at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:145) at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.init(GridDhtPartitionsExchangeFuture.java:903) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body0(GridCachePartitionExchangeManager.java:2958) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body(GridCachePartitionExchangeManager.java:2807) at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at java.lang.Thread.run(Thread.java:748) {code} > DistributedMetaStoragePersistentTest#testClientReconnect hangs sometimes. > ------------------------------------------------------------------------- > > Key: IGNITE-11683 > URL: https://issues.apache.org/jira/browse/IGNITE-11683 > Project: Ignite > Issue Type: Test > Reporter: Ivan Bessonov > Assignee: Ivan Bessonov > Priority: Major > Labels: MakeTeamcityGreenAgain > > The problem occurs right after this line: > {code:java} > assertTrue(GridTestUtils.waitForCondition(() -> > metastorage(1).getUpdatesCount() == expUpdatesCnt, 15_000)); > {code} > Client node might not be fully reconnected yet. Adding following line > resolves the problem in the particular test: > {code:java} > grid(1).cluster().clientReconnectFuture().get(); > {code} > I don't consider this a proper fix. Stopping the client that hasn't finished > its reconnect shouldn't result in deadlock. Client node should be stopped > successfully. > Stack traces of hanging client node: > {code:java} > Thread > [name="test-runner-#673%metastorage.DistributedMetaStoragePersistentTest%", > id=823, state=WAITING, blockCnt=60, waitCnt=200] > Lock [object=java.lang.Object@2a2d45ba, ownerName=null, ownerId=-1] > at java.lang.Object.wait(Native Method) > at java.lang.Object.wait(Object.java:502) > at o.a.i.i.util.worker.GridWorker.join(GridWorker.java:243) > at o.a.i.i.util.IgniteUtils.join(IgniteUtils.java:4831) > at > o.a.i.i.processors.cache.GridCachePartitionExchangeManager.onKernalStop0(GridCachePartitionExchangeManager.java:815) > at > o.a.i.i.processors.cache.GridCacheSharedManagerAdapter.onKernalStop(GridCacheSharedManagerAdapter.java:120) > at > o.a.i.i.processors.cache.GridCacheProcessor.onKernalStop(GridCacheProcessor.java:1183) > at o.a.i.i.IgniteKernal.stop0(IgniteKernal.java:2321) > at o.a.i.i.IgniteKernal.stop(IgniteKernal.java:2269) > at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2574) > - locked o.a.i.i.IgnitionEx$IgniteNamedInstance@69d9c55 > at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2537) > at o.a.i.i.IgnitionEx.stop(IgnitionEx.java:330) > at o.a.i.Ignition.stop(Ignition.java:223) > at > o.a.i.testframework.junits.GridAbstractTest.stopGrid(GridAbstractTest.java:1153) > at > o.a.i.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1193) > at > o.a.i.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1174) > at > o.a.i.i.processors.metastorage.DistributedMetaStorageTest.after(DistributedMetaStorageTest.java:85) > at > o.a.i.i.processors.metastorage.DistributedMetaStoragePersistentTest.after(DistributedMetaStoragePersistentTest.java:67) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44) > at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) > at > o.a.i.testframework.junits.GridAbstractTest$6.run(GridAbstractTest.java:2044) > at java.lang.Thread.run(Thread.java:748) > {code} > {code:java} > Thread > [name="exchange-worker-#893%metastorage.DistributedMetaStoragePersistentTest1%", > id=1088, state=BLOCKED, blockCnt=2, waitCnt=2] Lock > [object=o.a.i.i.IgnitionEx$IgniteNamedInstance@69d9c55, > ownerName=test-runner-#673%metastorage.DistributedMetaStoragePersistentTest%, > ownerId=823] at > o.a.i.i.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2545) at > o.a.i.i.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2537) at > o.a.i.i.IgnitionEx.stop(IgnitionEx.java:330) at > o.a.i.Ignition.stop(Ignition.java:223) at > o.a.i.i.IgniteKernal.close(IgniteKernal.java:3626) at > o.a.i.i.IgniteKernal$5.apply(IgniteKernal.java:4086) at > o.a.i.i.IgniteKernal$5.apply(IgniteKernal.java:4066) at > o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) > at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) > at > o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) > at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) > at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) > at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) > at o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:125) > at o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45) > at > o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) > at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) > at > o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) > at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) > at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) > at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) > at > o.a.i.i.processors.cache.GridCachePartitionExchangeManager$7.apply(GridCachePartitionExchangeManager.java:723) > at > o.a.i.i.processors.cache.GridCachePartitionExchangeManager$7.apply(GridCachePartitionExchangeManager.java:709) > at > o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399) > at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347) > at > o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335) > at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) > at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) > at > o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2263) > at > o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:145) > at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) > at > o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.init(GridDhtPartitionsExchangeFuture.java:903) > at > o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body0(GridCachePartitionExchangeManager.java:2958) > at > o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body(GridCachePartitionExchangeManager.java:2807) > at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at > java.lang.Thread.run(Thread.java:748) > {code} > -- This message was sent by Atlassian JIRA (v7.6.3#76005)