Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/5232/
Java: 64bit/hotspot/jdk-21.0.1 -XX:+UseCompressedOops -XX:+UseG1GC

6 tests failed.
FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.classMethod

Error Message:
java.lang.AssertionError: ObjectTracker found 2 object(s) that were not 
released!!! [ZkCollectionTerms, ZkShardTerms]
org.apache.solr.cloud.ZkCollectionTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkCollectionTerms
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at 
org.apache.solr.cloud.ZkCollectionTerms.<init>(ZkCollectionTerms.java:36)
        at 
org.apache.solr.cloud.ZkController.lambda$getCollectionTerms$21(ZkController.java:1857)
        at java.base/java.util.HashMap.computeIfAbsent(HashMap.java:1228)
        at 
org.apache.solr.cloud.ZkController.getCollectionTerms(ZkController.java:1856)
        at org.apache.solr.cloud.ZkController.register(ZkController.java:1275)
        at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:272)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:380)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
java.base/java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:145)
        at org.apache.solr.cloud.ZkController.onReconnect(ZkController.java:486)
        at 
org.apache.solr.common.cloud.OnReconnect.stateChanged(OnReconnect.java:36)
        at 
org.apache.curator.framework.state.ConnectionStateManager.lambda$processEvents$0(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.listen.MappingListenerManager.lambda$forEach$0(MappingListenerManager.java:82)
        ... 4 more
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
org.apache.curator.framework.listen.MappingListenerManager.forEach(MappingListenerManager.java:80)
        at 
org.apache.curator.framework.listen.StandardListenerManager.forEach(StandardListenerManager.java:81)
        at 
org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.state.ConnectionStateManager.access$000(ConnectionStateManager.java:43)
        at 
org.apache.curator.framework.state.ConnectionStateManager$1.call(ConnectionStateManager.java:130)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        ... 3 more

org.apache.solr.cloud.ZkShardTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkShardTerms
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at org.apache.solr.cloud.ZkShardTerms.<init>(ZkShardTerms.java:107)
        at 
org.apache.solr.cloud.ZkCollectionTerms.lambda$getShard$0(ZkCollectionTerms.java:41)
        at java.base/java.util.HashMap.computeIfAbsent(HashMap.java:1228)
        at 
org.apache.solr.cloud.ZkCollectionTerms.getShard(ZkCollectionTerms.java:41)
        at 
org.apache.solr.cloud.ZkCollectionTerms.register(ZkCollectionTerms.java:47)
        at org.apache.solr.cloud.ZkController.register(ZkController.java:1275)
        at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:272)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:380)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
java.base/java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:145)
        at org.apache.solr.cloud.ZkController.onReconnect(ZkController.java:486)
        at 
org.apache.solr.common.cloud.OnReconnect.stateChanged(OnReconnect.java:36)
        at 
org.apache.curator.framework.state.ConnectionStateManager.lambda$processEvents$0(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.listen.MappingListenerManager.lambda$forEach$0(MappingListenerManager.java:82)
        ... 4 more
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
org.apache.curator.framework.listen.MappingListenerManager.forEach(MappingListenerManager.java:80)
        at 
org.apache.curator.framework.listen.StandardListenerManager.forEach(StandardListenerManager.java:81)
        at 
org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.state.ConnectionStateManager.access$000(ConnectionStateManager.java:43)
        at 
org.apache.curator.framework.state.ConnectionStateManager$1.call(ConnectionStateManager.java:130)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        ... 3 more

 expected null, but was:<ObjectTracker found 2 object(s) that were not 
released!!! [ZkCollectionTerms, ZkShardTerms]
org.apache.solr.cloud.ZkCollectionTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkCollectionTerms
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at 
org.apache.solr.cloud.ZkCollectionTerms.<init>(ZkCollectionTerms.java:36)
        at 
org.apache.solr.cloud.ZkController.lambda$getCollectionTerms$21(ZkController.java:1857)
        at java.base/java.util.HashMap.computeIfAbsent(HashMap.java:1228)
        at 
org.apache.solr.cloud.ZkController.getCollectionTerms(ZkController.java:1856)
        at org.apache.solr.cloud.ZkController.register(ZkController.java:1275)
        at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:272)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:380)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
java.base/java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:145)
        at org.apache.solr.cloud.ZkController.onReconnect(ZkController.java:486)
        at 
org.apache.solr.common.cloud.OnReconnect.stateChanged(OnReconnect.java:36)
        at 
org.apache.curator.framework.state.ConnectionStateManager.lambda$processEvents$0(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.listen.MappingListenerManager.lambda$forEach$0(MappingListenerManager.java:82)
        ... 4 more
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
org.apache.curator.framework.listen.MappingListenerManager.forEach(MappingListenerManager.java:80)
        at 
org.apache.curator.framework.listen.StandardListenerManager.forEach(StandardListenerManager.java:81)
        at 
org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.state.ConnectionStateManager.access$000(ConnectionStateManager.java:43)
        at 
org.apache.curator.framework.state.ConnectionStateManager$1.call(ConnectionStateManager.java:130)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        ... 3 more

org.apache.solr.cloud.ZkShardTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkShardTerms
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at org.apache.solr.cloud.ZkShardTerms.<init>(ZkShardTerms.java:107)
        at 
org.apache.solr.cloud.ZkCollectionTerms.lambda$getShard$0(ZkCollectionTerms.java:41)
        at java.base/java.util.HashMap.computeIfAbsent(HashMap.java:1228)
        at 
org.apache.solr.cloud.ZkCollectionTerms.getShard(ZkCollectionTerms.java:41)
        at 
org.apache.solr.cloud.ZkCollectionTerms.register(ZkCollectionTerms.java:47)
        at org.apache.solr.cloud.ZkController.register(ZkController.java:1275)
        at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:272)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:380)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
java.base/java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:145)
        at org.apache.solr.cloud.ZkController.onReconnect(ZkController.java:486)
        at 
org.apache.solr.common.cloud.OnReconnect.stateChanged(OnReconnect.java:36)
        at 
org.apache.curator.framework.state.ConnectionStateManager.lambda$processEvents$0(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.listen.MappingListenerManager.lambda$forEach$0(MappingListenerManager.java:82)
        ... 4 more
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
org.apache.curator.framework.listen.MappingListenerManager.forEach(MappingListenerManager.java:80)
        at 
org.apache.curator.framework.listen.StandardListenerManager.forEach(StandardListenerManager.java:81)
        at 
org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.state.ConnectionStateManager.access$000(ConnectionStateManager.java:43)
        at 
org.apache.curator.framework.state.ConnectionStateManager$1.call(ConnectionStateManager.java:130)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        ... 3 more

>

Stack Trace:
java.lang.AssertionError: ObjectTracker found 2 object(s) that were not 
released!!! [ZkCollectionTerms, ZkShardTerms]
org.apache.solr.cloud.ZkCollectionTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkCollectionTerms
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at 
org.apache.solr.cloud.ZkCollectionTerms.<init>(ZkCollectionTerms.java:36)
        at 
org.apache.solr.cloud.ZkController.lambda$getCollectionTerms$21(ZkController.java:1857)
        at java.base/java.util.HashMap.computeIfAbsent(HashMap.java:1228)
        at 
org.apache.solr.cloud.ZkController.getCollectionTerms(ZkController.java:1856)
        at org.apache.solr.cloud.ZkController.register(ZkController.java:1275)
        at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:272)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:380)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
java.base/java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:145)
        at org.apache.solr.cloud.ZkController.onReconnect(ZkController.java:486)
        at 
org.apache.solr.common.cloud.OnReconnect.stateChanged(OnReconnect.java:36)
        at 
org.apache.curator.framework.state.ConnectionStateManager.lambda$processEvents$0(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.listen.MappingListenerManager.lambda$forEach$0(MappingListenerManager.java:82)
        ... 4 more
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
org.apache.curator.framework.listen.MappingListenerManager.forEach(MappingListenerManager.java:80)
        at 
org.apache.curator.framework.listen.StandardListenerManager.forEach(StandardListenerManager.java:81)
        at 
org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.state.ConnectionStateManager.access$000(ConnectionStateManager.java:43)
        at 
org.apache.curator.framework.state.ConnectionStateManager$1.call(ConnectionStateManager.java:130)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        ... 3 more

org.apache.solr.cloud.ZkShardTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkShardTerms
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at org.apache.solr.cloud.ZkShardTerms.<init>(ZkShardTerms.java:107)
        at 
org.apache.solr.cloud.ZkCollectionTerms.lambda$getShard$0(ZkCollectionTerms.java:41)
        at java.base/java.util.HashMap.computeIfAbsent(HashMap.java:1228)
        at 
org.apache.solr.cloud.ZkCollectionTerms.getShard(ZkCollectionTerms.java:41)
        at 
org.apache.solr.cloud.ZkCollectionTerms.register(ZkCollectionTerms.java:47)
        at org.apache.solr.cloud.ZkController.register(ZkController.java:1275)
        at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:272)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:380)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
java.base/java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:145)
        at org.apache.solr.cloud.ZkController.onReconnect(ZkController.java:486)
        at 
org.apache.solr.common.cloud.OnReconnect.stateChanged(OnReconnect.java:36)
        at 
org.apache.curator.framework.state.ConnectionStateManager.lambda$processEvents$0(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.listen.MappingListenerManager.lambda$forEach$0(MappingListenerManager.java:82)
        ... 4 more
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
org.apache.curator.framework.listen.MappingListenerManager.forEach(MappingListenerManager.java:80)
        at 
org.apache.curator.framework.listen.StandardListenerManager.forEach(StandardListenerManager.java:81)
        at 
org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.state.ConnectionStateManager.access$000(ConnectionStateManager.java:43)
        at 
org.apache.curator.framework.state.ConnectionStateManager$1.call(ConnectionStateManager.java:130)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        ... 3 more

 expected null, but was:<ObjectTracker found 2 object(s) that were not 
released!!! [ZkCollectionTerms, ZkShardTerms]
org.apache.solr.cloud.ZkCollectionTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkCollectionTerms
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at 
org.apache.solr.cloud.ZkCollectionTerms.<init>(ZkCollectionTerms.java:36)
        at 
org.apache.solr.cloud.ZkController.lambda$getCollectionTerms$21(ZkController.java:1857)
        at java.base/java.util.HashMap.computeIfAbsent(HashMap.java:1228)
        at 
org.apache.solr.cloud.ZkController.getCollectionTerms(ZkController.java:1856)
        at org.apache.solr.cloud.ZkController.register(ZkController.java:1275)
        at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:272)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:380)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
java.base/java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:145)
        at org.apache.solr.cloud.ZkController.onReconnect(ZkController.java:486)
        at 
org.apache.solr.common.cloud.OnReconnect.stateChanged(OnReconnect.java:36)
        at 
org.apache.curator.framework.state.ConnectionStateManager.lambda$processEvents$0(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.listen.MappingListenerManager.lambda$forEach$0(MappingListenerManager.java:82)
        ... 4 more
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
org.apache.curator.framework.listen.MappingListenerManager.forEach(MappingListenerManager.java:80)
        at 
org.apache.curator.framework.listen.StandardListenerManager.forEach(StandardListenerManager.java:81)
        at 
org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.state.ConnectionStateManager.access$000(ConnectionStateManager.java:43)
        at 
org.apache.curator.framework.state.ConnectionStateManager$1.call(ConnectionStateManager.java:130)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        ... 3 more

org.apache.solr.cloud.ZkShardTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkShardTerms
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at org.apache.solr.cloud.ZkShardTerms.<init>(ZkShardTerms.java:107)
        at 
org.apache.solr.cloud.ZkCollectionTerms.lambda$getShard$0(ZkCollectionTerms.java:41)
        at java.base/java.util.HashMap.computeIfAbsent(HashMap.java:1228)
        at 
org.apache.solr.cloud.ZkCollectionTerms.getShard(ZkCollectionTerms.java:41)
        at 
org.apache.solr.cloud.ZkCollectionTerms.register(ZkCollectionTerms.java:47)
        at org.apache.solr.cloud.ZkController.register(ZkController.java:1275)
        at 
org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:272)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:380)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
java.base/java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:145)
        at org.apache.solr.cloud.ZkController.onReconnect(ZkController.java:486)
        at 
org.apache.solr.common.cloud.OnReconnect.stateChanged(OnReconnect.java:36)
        at 
org.apache.curator.framework.state.ConnectionStateManager.lambda$processEvents$0(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.listen.MappingListenerManager.lambda$forEach$0(MappingListenerManager.java:82)
        ... 4 more
Caused by: java.lang.Exception: Submitter stack trace
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:347)
        at 
org.apache.curator.framework.listen.MappingListenerManager.forEach(MappingListenerManager.java:80)
        at 
org.apache.curator.framework.listen.StandardListenerManager.forEach(StandardListenerManager.java:81)
        at 
org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:256)
        at 
org.apache.curator.framework.state.ConnectionStateManager.access$000(ConnectionStateManager.java:43)
        at 
org.apache.curator.framework.state.ConnectionStateManager$1.call(ConnectionStateManager.java:130)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
        ... 3 more

>
        at __randomizedtesting.SeedInfo.seed([7211FC3699F20969]:0)
        at org.junit.Assert.fail(Assert.java:89)
        at org.junit.Assert.failNotNull(Assert.java:756)
        at org.junit.Assert.assertNull(Assert.java:738)
        at 
org.apache.solr.SolrTestCase$1.afterIfSuccessful(SolrTestCase.java:103)
        at 
com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.base/java.lang.Thread.run(Thread.java:1583)


FAILED:  org.apache.solr.cloud.PeerSyncReplicationTest.test

Error Message:
java.lang.AssertionError: expected:<154> but was:<152>

Stack Trace:
java.lang.AssertionError: expected:<154> but was:<152>
        at 
__randomizedtesting.SeedInfo.seed([7211FC3699F20969:FA45C3EC370E6491]:0)
        at org.junit.Assert.fail(Assert.java:89)
        at org.junit.Assert.failNotEquals(Assert.java:835)
        at org.junit.Assert.assertEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:633)
        at 
org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:142)
        at 
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
        at java.base/java.lang.reflect.Method.invoke(Method.java:580)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1070)
        at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1041)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.base/java.lang.Thread.run(Thread.java:1583)


FAILED:  org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest

Error Message:
java.lang.AssertionError: ZK Server did not go up when expected

Stack Trace:
java.lang.AssertionError: ZK Server did not go up when expected
        at 
__randomizedtesting.SeedInfo.seed([7211FC3699F20969:F167A3C44F8B07C8]:0)
        at org.junit.Assert.fail(Assert.java:89)
        at org.junit.Assert.assertTrue(Assert.java:42)
        at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:607)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:291)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1199)
        at 
org.apache.solr.cloud.TestCloudRecovery.beforeTest(TestCloudRecovery.java:70)
        at 
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
        at java.base/java.lang.reflect.Method.invoke(Method.java:580)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.base/java.lang.Thread.run(Thread.java:1583)


FAILED:  org.apache.solr.cloud.TestCloudRecovery.classMethod

Error Message:
java.lang.AssertionError: ObjectTracker found 3 object(s) that were not 
released!!! [ZKServerMain, Thread, ZkTestServer]
org.apache.solr.cloud.ZkTestServer$ZKServerMain:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkTestServer$ZKServerMain
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at 
org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:311)
        at org.apache.solr.cloud.ZkTestServer$1.run(ZkTestServer.java:575)

org.apache.solr.cloud.ZkTestServer$1:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkTestServer$1
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:583)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:291)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1199)
        at 
org.apache.solr.cloud.TestCloudRecovery.beforeTest(TestCloudRecovery.java:70)
        at 
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
        at java.base/java.lang.reflect.Method.invoke(Method.java:580)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.base/java.lang.Thread.run(Thread.java:1583)

org.apache.solr.cloud.ZkTestServer:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkTestServer
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at org.apache.solr.cloud.ZkTestServer.<init>(ZkTestServer.java:443)
        at org.apache.solr.cloud.ZkTestServer.<init>(ZkTestServer.java:426)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:289)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1199)
        at 
org.apache.solr.cloud.TestCloudRecovery.beforeTest(TestCloudRecovery.java:70)
        at 
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
        at java.base/java.lang.reflect.Method.invoke(Method.java:580)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.base/java.lang.Thread.run(Thread.java:1583)

 expected null, but was:<ObjectTracker found 3 object(s) that were not 
released!!! [ZKServerMain, Thread, ZkTestServer]
org.apache.solr.cloud.ZkTestServer$ZKServerMain:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkTestServer$ZKServerMain
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at 
org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:311)
        at org.apache.solr.cloud.ZkTestServer$1.run(ZkTestServer.java:575)

org.apache.solr.cloud.ZkTestServer$1:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkTestServer$1
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:583)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:291)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1199)
        at 
org.apache.solr.cloud.TestCloudRecovery.beforeTest(TestCloudRecovery.java:70)
        at 
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
        at java.base/java.lang.reflect.Method.invoke(Method.java:580)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.base/java.lang.Thread.run(Thread.java:1583)

org.apache.solr.cloud.ZkTestServer:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkTestServer
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at org.apache.solr.cloud.ZkTestServer.<init>(ZkTestServer.java:443)
        at org.apache.solr.cloud.ZkTestServer.<init>(ZkTestServer.java:426)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:289)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1199)
        at 
org.apache.solr.cloud.TestCloudRecovery.beforeTest(TestCloudRecovery.java:70)
        at 
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
        at java.base/java.lang.reflect.Method.invoke(Method.java:580)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.base/java.lang.Thread.run(Thread.java:1583)

>

Stack Trace:
java.lang.AssertionError: ObjectTracker found 3 object(s) that were not 
released!!! [ZKServerMain, Thread, ZkTestServer]
org.apache.solr.cloud.ZkTestServer$ZKServerMain:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkTestServer$ZKServerMain
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at 
org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:311)
        at org.apache.solr.cloud.ZkTestServer$1.run(ZkTestServer.java:575)

org.apache.solr.cloud.ZkTestServer$1:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkTestServer$1
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:583)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:291)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1199)
        at 
org.apache.solr.cloud.TestCloudRecovery.beforeTest(TestCloudRecovery.java:70)
        at 
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
        at java.base/java.lang.reflect.Method.invoke(Method.java:580)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.base/java.lang.Thread.run(Thread.java:1583)

org.apache.solr.cloud.ZkTestServer:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkTestServer
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at org.apache.solr.cloud.ZkTestServer.<init>(ZkTestServer.java:443)
        at org.apache.solr.cloud.ZkTestServer.<init>(ZkTestServer.java:426)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:289)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1199)
        at 
org.apache.solr.cloud.TestCloudRecovery.beforeTest(TestCloudRecovery.java:70)
        at 
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
        at java.base/java.lang.reflect.Method.invoke(Method.java:580)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.base/java.lang.Thread.run(Thread.java:1583)

 expected null, but was:<ObjectTracker found 3 object(s) that were not 
released!!! [ZKServerMain, Thread, ZkTestServer]
org.apache.solr.cloud.ZkTestServer$ZKServerMain:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkTestServer$ZKServerMain
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at 
org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:311)
        at org.apache.solr.cloud.ZkTestServer$1.run(ZkTestServer.java:575)

org.apache.solr.cloud.ZkTestServer$1:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkTestServer$1
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:583)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:291)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1199)
        at 
org.apache.solr.cloud.TestCloudRecovery.beforeTest(TestCloudRecovery.java:70)
        at 
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
        at java.base/java.lang.reflect.Method.invoke(Method.java:580)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.base/java.lang.Thread.run(Thread.java:1583)

org.apache.solr.cloud.ZkTestServer:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException:
 org.apache.solr.cloud.ZkTestServer
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
        at org.apache.solr.cloud.ZkTestServer.<init>(ZkTestServer.java:443)
        at org.apache.solr.cloud.ZkTestServer.<init>(ZkTestServer.java:426)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:289)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1199)
        at 
org.apache.solr.cloud.TestCloudRecovery.beforeTest(TestCloudRecovery.java:70)
        at 
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
        at java.base/java.lang.reflect.Method.invoke(Method.java:580)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.base/java.lang.Thread.run(Thread.java:1583)

>
        at org.junit.Assert.fail(Assert.java:89)
        at org.junit.Assert.failNotNull(Assert.java:756)
        at org.junit.Assert.assertNull(Assert.java:738)
        at 
org.apache.solr.SolrTestCase$1.afterIfSuccessful(SolrTestCase.java:103)
        at 
com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
        at 
org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
        at 
org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
        at org.junit.rules.RunRules.evaluate(RunRules.java:20)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
        at java.base/java.lang.Thread.run(Thread.java:1583)


FAILED:  org.apache.solr.cloud.TestCloudRecovery.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from 
SUITE scope at org.apache.solr.cloud.TestCloudRecovery: 
   1) Thread[id=8290, name=NIOServerCxnFactory.SelectorThread-1, 
state=RUNNABLE, group=TGRP-TestCloudRecovery]
        at java.base/sun.nio.ch.KQueue.poll(Native Method)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:125)
        at 
java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   2) Thread[id=8287, name=ZkTestServer Run Thread, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/java.lang.Object.wait0(Native Method)
        at java.base/java.lang.Object.wait(Object.java:366)
        at java.base/java.lang.Thread.join(Thread.java:2078)
        at java.base/java.lang.Thread.join(Thread.java:2154)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:919)
        at 
org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:338)
        at org.apache.solr.cloud.ZkTestServer$1.run(ZkTestServer.java:575)
   3) Thread[id=8301, name=SyncThread:0, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
   4) Thread[id=8303, name=RequestThrottler, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
   5) Thread[id=8293, name=NIOWorkerThread-2, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
   6) Thread[id=8294, name=NIOWorkerThread-3, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
   7) Thread[id=8292, name=NIOWorkerThread-1, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
   8) Thread[id=8289, name=NIOServerCxnFactory.SelectorThread-0, 
state=RUNNABLE, group=TGRP-TestCloudRecovery]
        at java.base/sun.nio.ch.KQueue.poll(Native Method)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:125)
        at 
java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   9) Thread[id=8302, name=ProcessThread(sid:0 cport:60669):, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
  10) Thread[id=8291, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, 
state=RUNNABLE, group=TGRP-TestCloudRecovery]
        at java.base/sun.nio.ch.KQueue.poll(Native Method)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:125)
        at 
java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from 
SUITE scope at org.apache.solr.cloud.TestCloudRecovery: 
   1) Thread[id=8290, name=NIOServerCxnFactory.SelectorThread-1, 
state=RUNNABLE, group=TGRP-TestCloudRecovery]
        at java.base/sun.nio.ch.KQueue.poll(Native Method)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:125)
        at 
java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   2) Thread[id=8287, name=ZkTestServer Run Thread, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/java.lang.Object.wait0(Native Method)
        at java.base/java.lang.Object.wait(Object.java:366)
        at java.base/java.lang.Thread.join(Thread.java:2078)
        at java.base/java.lang.Thread.join(Thread.java:2154)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:919)
        at 
org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:338)
        at org.apache.solr.cloud.ZkTestServer$1.run(ZkTestServer.java:575)
   3) Thread[id=8301, name=SyncThread:0, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
   4) Thread[id=8303, name=RequestThrottler, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
   5) Thread[id=8293, name=NIOWorkerThread-2, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
   6) Thread[id=8294, name=NIOWorkerThread-3, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
   7) Thread[id=8292, name=NIOWorkerThread-1, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
   8) Thread[id=8289, name=NIOServerCxnFactory.SelectorThread-0, 
state=RUNNABLE, group=TGRP-TestCloudRecovery]
        at java.base/sun.nio.ch.KQueue.poll(Native Method)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:125)
        at 
java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   9) Thread[id=8302, name=ProcessThread(sid:0 cport:60669):, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
  10) Thread[id=8291, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, 
state=RUNNABLE, group=TGRP-TestCloudRecovery]
        at java.base/sun.nio.ch.KQueue.poll(Native Method)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:125)
        at 
java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
        at __randomizedtesting.SeedInfo.seed([7211FC3699F20969]:0)


FAILED:  org.apache.solr.cloud.TestCloudRecovery.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=8290, name=NIOServerCxnFactory.SelectorThread-1, 
state=RUNNABLE, group=TGRP-TestCloudRecovery]
        at java.base/sun.nio.ch.IOUtil.write1(Native Method)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:299)
        at 
java.base/java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:231)
        at 
java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
        at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:119)
        at 
java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   2) Thread[id=8292, name=NIOWorkerThread-1, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
   3) Thread[id=8293, name=NIOWorkerThread-2, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
   4) Thread[id=8289, name=NIOServerCxnFactory.SelectorThread-0, 
state=RUNNABLE, group=TGRP-TestCloudRecovery]
        at java.base/sun.nio.ch.IOUtil.write1(Native Method)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:299)
        at 
java.base/java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:231)
        at 
java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
        at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:119)
        at 
java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   5) Thread[id=8294, name=NIOWorkerThread-3, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
   6) Thread[id=8291, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, 
state=RUNNABLE, group=TGRP-TestCloudRecovery]
        at java.base/sun.nio.ch.KQueue.poll(Native Method)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:125)
        at 
java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=8290, name=NIOServerCxnFactory.SelectorThread-1, 
state=RUNNABLE, group=TGRP-TestCloudRecovery]
        at java.base/sun.nio.ch.IOUtil.write1(Native Method)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:299)
        at 
java.base/java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:231)
        at 
java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
        at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:119)
        at 
java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   2) Thread[id=8292, name=NIOWorkerThread-1, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
   3) Thread[id=8293, name=NIOWorkerThread-2, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
   4) Thread[id=8289, name=NIOServerCxnFactory.SelectorThread-0, 
state=RUNNABLE, group=TGRP-TestCloudRecovery]
        at java.base/sun.nio.ch.IOUtil.write1(Native Method)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:299)
        at 
java.base/java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:231)
        at 
java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
        at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:119)
        at 
java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
   5) Thread[id=8294, name=NIOWorkerThread-3, state=WAITING, 
group=TGRP-TestCloudRecovery]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at 
java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at 
java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3780)
        at 
java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3725)
        at 
java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
        at 
java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at 
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1583)
   6) Thread[id=8291, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, 
state=RUNNABLE, group=TGRP-TestCloudRecovery]
        at java.base/sun.nio.ch.KQueue.poll(Native Method)
        at 
java.base/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:125)
        at 
java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
        at 
org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
        at __randomizedtesting.SeedInfo.seed([7211FC3699F20969]:0)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to