Build: https://builds.apache.org/job/Lucene-Solr-Tests-6.x/457/

6 tests failed.
FAILED:  org.apache.solr.cloud.CdcrVersionReplicationTest.testCdcrDocVersions

Error Message:
Captured an uncaught exception in thread: Thread[id=54168, 
name=qtp1123913138-54168, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught 
exception in thread: Thread[id=54168, name=qtp1123913138-54168, state=RUNNABLE, 
group=TGRP-CdcrVersionReplicationTest]
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at __randomizedtesting.SeedInfo.seed([6AE7F077D0FF419D]:0)
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.startThreads(QueuedThreadPool.java:462)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$200(QueuedThreadPool.java:47)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:639)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestDetails

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
        at 
__randomizedtesting.SeedInfo.seed([6AE7F077D0FF419D:10BAD30246A76815]:0)
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1368)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:214)
        at 
java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
        at org.apache.solr.core.CoreContainer.load(CoreContainer.java:522)
        at 
org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:203)
        at 
org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:152)
        at 
org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
        at 
org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:856)
        at 
org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1542)
        at 
org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1576)
        at 
org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1268)
        at 
org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1113)
        at 
org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:447)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:258)
        at 
org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
        at 
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:342)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:315)
        at 
org.apache.solr.handler.TestReplicationHandler.createJetty(TestReplicationHandler.java:155)
        at 
org.apache.solr.handler.TestReplicationHandler.doTestDetails(TestReplicationHandler.java:336)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.solr.handler.TestReplicationHandler.doTestIndexFetchWithMasterUrl

Error Message:
Error from server at http://127.0.0.1:48809/solr/collection1: Expected mime 
type application/octet-stream but got text/html. <html> <head> <meta 
http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 
500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing 
/solr/collection1/update. Reason: <pre>    
{metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore
 &apos;collection1&apos; is not available due to init failure: JVM Error 
creating core [collection1]: unable to create new native 
thread,trace=org.apache.solr.common.SolrException: SolrCore 
&apos;collection1&apos; is not available due to init failure: JVM Error 
creating core [collection1]: unable to create new native thread  at 
org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1097)  at 
org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:251)  at 
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:413)  at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:302)
  at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:253)
  at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
  at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:108)
  at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581) 
 at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
  at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  
at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
  at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
  at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  
at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  
at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) 
 at org.eclipse.jetty.server.Server.handle(Server.java:518)  at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) 
 at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
  at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
  at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
  at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) 
 at java.lang.Thread.run(Thread.java:745) Caused by: 
org.apache.solr.common.SolrException: JVM Error creating core [collection1]: 
unable to create new native thread  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:860)  at 
org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)  at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
 ... 1 more Caused by: java.lang.OutOfMemoryError: unable to create new native 
thread  at java.lang.Thread.start0(Native Method)  at 
java.lang.Thread.start(Thread.java:714)  at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)  
at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1357)  
at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:214)
  at 
java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134)
  at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:781)  at 
org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:688)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)  ... 6 more 
,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty";>Powered by 
Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error 
from server at http://127.0.0.1:48809/solr/collection1: Expected mime type 
application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /solr/collection1/update. Reason:
<pre>    
{metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore
 &apos;collection1&apos; is not available due to init failure: JVM Error 
creating core [collection1]: unable to create new native 
thread,trace=org.apache.solr.common.SolrException: SolrCore 
&apos;collection1&apos; is not available due to init failure: JVM Error 
creating core [collection1]: unable to create new native thread
        at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1097)
        at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:251)
        at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:413)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:302)
        at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:253)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
        at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:108)
        at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
        at 
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
        at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
        at 
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
        at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
        at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
        at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
        at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
        at org.eclipse.jetty.server.Server.handle(Server.java:518)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
        at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
        at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
        at 
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: JVM Error creating core 
[collection1]: unable to create new native thread
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:860)
        at 
org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        ... 1 more
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:714)
        at 
java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:950)
        at 
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1357)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:214)
        at 
java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134)
        at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:781)
        at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:688)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
        ... 6 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty";>Powered by Jetty:// 
9.3.8.v20160314</a><hr/>
</body>
</html>

        at 
__randomizedtesting.SeedInfo.seed([6AE7F077D0FF419D:715C0108B140392B]:0)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:560)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
        at 
org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
        at 
org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
        at 
org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
        at 
org.apache.solr.handler.TestReplicationHandler.doTestIndexFetchWithMasterUrl(TestReplicationHandler.java:682)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication

Error Message:
expected:<498> but was:<0>

Stack Trace:
java.lang.AssertionError: expected:<498> but was:<0>
        at 
__randomizedtesting.SeedInfo.seed([6AE7F077D0FF419D:7EAFAB22F3F8FC83]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.failNotEquals(Assert.java:647)
        at org.junit.Assert.assertEquals(Assert.java:128)
        at org.junit.Assert.assertEquals(Assert.java:472)
        at org.junit.Assert.assertEquals(Assert.java:456)
        at 
org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication(TestReplicationHandler.java:487)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)


FAILED:  
junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler

Error Message:
15 threads leaked from SUITE scope at 
org.apache.solr.handler.TestReplicationHandler:     1) Thread[id=7125, 
name=Scheduler-1288905127, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    2) Thread[id=7119, 
name=qtp2107595542-7119-acceptor-0@64344d8b-ServerConnector@61347e6d{HTTP/1.1,[http/1.1]}{127.0.0.1:52864},
 state=RUNNABLE, group=TGRP-TestReplicationHandler]         at 
sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)     
    at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)     
    at 
org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)       
  at 
org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) 
        at java.lang.Thread.run(Thread.java:745)    3) Thread[id=7120, 
name=qtp2107595542-7120, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) 
        at java.lang.Thread.run(Thread.java:745)    4) Thread[id=7118, 
name=qtp2107595542-7118, state=RUNNABLE, group=TGRP-TestReplicationHandler]     
    at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) 
        at java.lang.Thread.run(Thread.java:745)    5) Thread[id=7124, 
name=qtp2107595542-7124, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) 
        at java.lang.Thread.run(Thread.java:745)    6) Thread[id=7126, 
name=org.eclipse.jetty.server.session.HashSessionManager@2febf8b6Timer, 
state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    7) Thread[id=7130, 
name=Thread-2068, state=WAITING, group=TGRP-TestReplicationHandler]         at 
java.lang.Object.wait(Native Method)         at 
java.lang.Object.wait(Object.java:502)         at 
org.apache.solr.core.CloserThread.run(CoreContainer.java:1274)    8) 
Thread[id=7121, name=qtp2107595542-7121, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) 
        at java.lang.Thread.run(Thread.java:745)    9) Thread[id=7117, 
name=qtp2107595542-7117, state=RUNNABLE, group=TGRP-TestReplicationHandler]     
    at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) 
        at java.lang.Thread.run(Thread.java:745)   10) Thread[id=7127, 
name=solr-idle-connections-evictor, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native 
Method)         at 
org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:187)
         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=7128, 
name=solr-idle-connections-evictor, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]         at java.lang.Thread.sleep(Native 
Method)         at 
org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:187)
         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=7135, 
name=searcherExecutor-3092-thread-1, state=WAITING, 
group=TGRP-TestReplicationHandler]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   13) Thread[id=7129, 
name=coreLoadExecutor-3089-thread-1, state=WAITING, 
group=TGRP-TestReplicationHandler]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)   14) Thread[id=7123, 
name=qtp2107595542-7123, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) 
        at java.lang.Thread.run(Thread.java:745)   15) Thread[id=7122, 
name=qtp2107595542-7122, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)     
    at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) 
        at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from 
SUITE scope at org.apache.solr.handler.TestReplicationHandler: 
   1) Thread[id=7125, name=Scheduler-1288905127, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=7119, 
name=qtp2107595542-7119-acceptor-0@64344d8b-ServerConnector@61347e6d{HTTP/1.1,[http/1.1]}{127.0.0.1:52864},
 state=RUNNABLE, group=TGRP-TestReplicationHandler]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at 
sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at 
org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)
        at 
org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=7120, name=qtp2107595542-7120, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=7118, name=qtp2107595542-7118, state=RUNNABLE, 
group=TGRP-TestReplicationHandler]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=7124, name=qtp2107595542-7124, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=7126, 
name=org.eclipse.jetty.server.session.HashSessionManager@2febf8b6Timer, 
state=TIMED_WAITING, group=TGRP-TestReplicationHandler]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=7130, name=Thread-2068, state=WAITING, 
group=TGRP-TestReplicationHandler]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1274)
   8) Thread[id=7121, name=qtp2107595542-7121, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=7117, name=qtp2107595542-7117, state=RUNNABLE, 
group=TGRP-TestReplicationHandler]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=7127, name=solr-idle-connections-evictor, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:187)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=7128, name=solr-idle-connections-evictor, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]
        at java.lang.Thread.sleep(Native Method)
        at 
org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:187)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=7135, name=searcherExecutor-3092-thread-1, state=WAITING, 
group=TGRP-TestReplicationHandler]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=7129, name=coreLoadExecutor-3089-thread-1, state=WAITING, 
group=TGRP-TestReplicationHandler]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  14) Thread[id=7123, name=qtp2107595542-7123, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  15) Thread[id=7122, name=qtp2107595542-7122, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
        at __randomizedtesting.SeedInfo.seed([6AE7F077D0FF419D]:0)


FAILED:  
junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler

Error Message:
There are still zombie threads that couldn't be terminated:    1) 
Thread[id=7125, name=Scheduler-1288905127, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    2) Thread[id=7118, 
name=qtp2107595542-7118, state=RUNNABLE, group=TGRP-TestReplicationHandler]     
    at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at 
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) 
        at java.lang.Thread.run(Thread.java:745)    3) Thread[id=7135, 
name=searcherExecutor-3092-thread-1, state=WAITING, 
group=TGRP-TestReplicationHandler]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    4) Thread[id=7129, 
name=coreLoadExecutor-3089-thread-1, state=WAITING, 
group=TGRP-TestReplicationHandler]         at sun.misc.Unsafe.park(Native 
Method)         at 
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
         at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)     
    at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    5) Thread[id=7126, 
name=org.eclipse.jetty.server.session.HashSessionManager@2febf8b6Timer, 
state=TIMED_WAITING, group=TGRP-TestReplicationHandler]         at 
sun.misc.Unsafe.park(Native Method)         at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         
at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
         at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
         at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)   
      at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) 
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
        at java.lang.Thread.run(Thread.java:745)    6) Thread[id=7130, 
name=Thread-2068, state=WAITING, group=TGRP-TestReplicationHandler]         at 
java.lang.Object.wait(Native Method)         at 
java.lang.Object.wait(Object.java:502)         at 
org.apache.solr.core.CloserThread.run(CoreContainer.java:1274)    7) 
Thread[id=7117, name=qtp2107595542-7117, state=RUNNABLE, 
group=TGRP-TestReplicationHandler]         at 
sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)         at 
sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)         at 
sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)         at 
java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)   
      at 
java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)         at 
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at 
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
         at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
         at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)  
       at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
         at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) 
        at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie 
threads that couldn't be terminated:
   1) Thread[id=7125, name=Scheduler-1288905127, state=TIMED_WAITING, 
group=TGRP-TestReplicationHandler]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=7118, name=qtp2107595542-7118, state=RUNNABLE, 
group=TGRP-TestReplicationHandler]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=7135, name=searcherExecutor-3092-thread-1, state=WAITING, 
group=TGRP-TestReplicationHandler]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=7129, name=coreLoadExecutor-3089-thread-1, state=WAITING, 
group=TGRP-TestReplicationHandler]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at 
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=7126, 
name=org.eclipse.jetty.server.session.HashSessionManager@2febf8b6Timer, 
state=TIMED_WAITING, group=TGRP-TestReplicationHandler]
        at sun.misc.Unsafe.park(Native Method)
        at 
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at 
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at 
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at 
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=7130, name=Thread-2068, state=WAITING, 
group=TGRP-TestReplicationHandler]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1274)
   7) Thread[id=7117, name=qtp2107595542-7117, state=RUNNABLE, 
group=TGRP-TestReplicationHandler]
        at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
        at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
        at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
        at 
java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at 
java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at 
org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at 
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
        at __randomizedtesting.SeedInfo.seed([6AE7F077D0FF419D]:0)




Build Log:
[...truncated 11392 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/init-core-data-001
   [junit4]   2> 1334671 INFO  
(SUITE-TestReplicationHandler-seed#[6AE7F077D0FF419D]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: 
@org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 1334683 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting doTestHandlerPathUnchanged
   [junit4]   2> 1334684 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-001/collection1
   [junit4]   2> 1334709 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1334741 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@2284995d{/solr,null,AVAILABLE}
   [junit4]   2> 1334741 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.e.j.s.ServerConnector Started 
ServerConnector@7d3a9103{HTTP/1.1,[http/1.1]}{127.0.0.1:55292}
   [junit4]   2> 1334742 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.e.j.s.Server Started @1344084ms
   [junit4]   2> 1334742 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-001/collection1/data,
 hostContext=/solr, hostPort=55292}
   [junit4]   2> 1334742 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? 
version 6.3.0
   [junit4]   2> 1334742 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone 
mode on port null
   [junit4]   2> 1334742 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1334742 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-09-28T12:47:01.874Z
   [junit4]   2> 1334743 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-001/solr.xml
   [junit4]   2> 1334791 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with 
params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1334800 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-001/.
   [junit4]   2> 1334800 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1334828 WARN  (coreLoadExecutor-2838-thread-1) [    ] 
o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use 
<mergePolicyFactory> instead.
   [junit4]   2> 1334828 INFO  (coreLoadExecutor-2838-thread-1) [    ] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 1335059 INFO  (coreLoadExecutor-2838-thread-1) [    ] 
o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1335073 INFO  (coreLoadExecutor-2838-thread-1) [    ] 
o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1335076 INFO  (coreLoadExecutor-2838-thread-1) [    ] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from 
instancedir 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-001/./collection1
   [junit4]   2> 1335076 INFO  (coreLoadExecutor-2838-thread-1) [    
x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-001/collection1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 1335081 INFO  (coreLoadExecutor-2838-thread-1) [    
x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: 
maxMergeAtOnce=23, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=5.208984375, 
floorSegmentMB=1.4287109375, forceMergeDeletesPctAllowed=10.27355101629424, 
segmentsPerTier=27.0, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.6130262309560954
   [junit4]   2> 1335097 INFO  (coreLoadExecutor-2838-thread-1) [    
x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1335098 INFO  (coreLoadExecutor-2838-thread-1) [    
x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1335103 INFO  (coreLoadExecutor-2838-thread-1) [    
x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: 
minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.0]
   [junit4]   2> 1335103 INFO  (coreLoadExecutor-2838-thread-1) [    
x:collection1] o.a.s.s.SolrIndexSearcher Opening 
[Searcher@68aa9a60[collection1] main]
   [junit4]   2> 1335103 INFO  (coreLoadExecutor-2838-thread-1) [    
x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to 
use dir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-001/collection1/conf
   [junit4]   2> 1335104 INFO  (coreLoadExecutor-2838-thread-1) [    
x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following 
config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 1335104 INFO  (coreLoadExecutor-2838-thread-1) [    
x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1335105 INFO  
(searcherExecutor-2839-thread-1-processing-x:collection1) [    x:collection1] 
o.a.s.c.SolrCore [collection1] Registered new searcher 
Searcher@68aa9a60[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1335308 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-002/collection1
   [junit4]   2> 1335311 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1335316 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1ba8d5c6{/solr,null,AVAILABLE}
   [junit4]   2> 1335316 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.e.j.s.ServerConnector Started 
ServerConnector@29ab7ec6{HTTP/1.1,[http/1.1]}{127.0.0.1:48093}
   [junit4]   2> 1335316 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.e.j.s.Server Started @1344658ms
   [junit4]   2> 1335316 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-002/collection1/data,
 hostContext=/solr, hostPort=48093}
   [junit4]   2> 1335316 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? 
version 6.3.0
   [junit4]   2> 1335317 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone 
mode on port null
   [junit4]   2> 1335317 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1335317 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-09-28T12:47:02.449Z
   [junit4]   2> 1335335 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-002/solr.xml
   [junit4]   2> 1335342 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with 
params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1335345 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-002/.
   [junit4]   2> 1335345 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1335367 WARN  (coreLoadExecutor-2848-thread-1) [    ] 
o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use 
<mergePolicyFactory> instead.
   [junit4]   2> 1335367 INFO  (coreLoadExecutor-2848-thread-1) [    ] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 1335392 INFO  (coreLoadExecutor-2848-thread-1) [    ] 
o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1335394 INFO  (coreLoadExecutor-2848-thread-1) [    ] 
o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1335402 INFO  (coreLoadExecutor-2848-thread-1) [    ] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from 
instancedir 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-002/./collection1
   [junit4]   2> 1335402 INFO  (coreLoadExecutor-2848-thread-1) [    
x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-002/collection1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-002/./collection1/data/]
   [junit4]   2> 1335411 INFO  (coreLoadExecutor-2848-thread-1) [    
x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: 
maxMergeAtOnce=23, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=5.208984375, 
floorSegmentMB=1.4287109375, forceMergeDeletesPctAllowed=10.27355101629424, 
segmentsPerTier=27.0, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.6130262309560954
   [junit4]   2> 1335425 INFO  (coreLoadExecutor-2848-thread-1) [    
x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1335425 INFO  (coreLoadExecutor-2848-thread-1) [    
x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1335425 INFO  (coreLoadExecutor-2848-thread-1) [    
x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: 
minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.0]
   [junit4]   2> 1335426 INFO  (coreLoadExecutor-2848-thread-1) [    
x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@f6914c2[collection1] 
main]
   [junit4]   2> 1335426 INFO  (coreLoadExecutor-2848-thread-1) [    
x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to 
use dir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-002/collection1/conf
   [junit4]   2> 1335428 INFO  (coreLoadExecutor-2848-thread-1) [    
x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 
1000ms
   [junit4]   2> 1335428 INFO  
(searcherExecutor-2849-thread-1-processing-x:collection1) [    x:collection1] 
o.a.s.c.SolrCore [collection1] Registered new searcher 
Searcher@f6914c2[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1335446 INFO  (qtp1294453588-6723) [    x:collection1] 
o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication 
params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 
QTime=0
   [junit4]   2> 1335446 INFO  (indexFetcher-2853-thread-1) [    x:collection1] 
o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 1335446 INFO  (indexFetcher-2853-thread-1) [    x:collection1] 
o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 1335446 INFO  (indexFetcher-2853-thread-1) [    x:collection1] 
o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 1335954 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending doTestHandlerPathUnchanged
   [junit4]   2> 1335955 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.e.j.s.ServerConnector Stopped 
ServerConnector@7d3a9103{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1335955 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1616261841
   [junit4]   2> 1335963 INFO  (coreCloseExecutor-2854-thread-1) [    
x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore 
org.apache.solr.core.SolrCore@6fdef02e
   [junit4]   2> 1335987 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@2284995d{/solr,null,UNAVAILABLE}
   [junit4]   2> 1335990 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.e.j.s.ServerConnector Stopped 
ServerConnector@29ab7ec6{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1335991 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1354755498
   [junit4]   2> 1335999 INFO  (coreCloseExecutor-2856-thread-1) [    
x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore 
org.apache.solr.core.SolrCore@77df4df1
   [junit4]   2> 1336007 INFO  
(TEST-TestReplicationHandler.doTestHandlerPathUnchanged-seed#[6AE7F077D0FF419D])
 [    ] o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@1ba8d5c6{/solr,null,UNAVAILABLE}
   [junit4]   2> 1336012 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting doTestRepeater
   [junit4]   2> 1336012 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-003/collection1
   [junit4]   2> 1336026 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1336054 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@29077f16{/solr,null,AVAILABLE}
   [junit4]   2> 1336055 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.e.j.s.ServerConnector Started 
ServerConnector@2eed3548{HTTP/1.1,[http/1.1]}{127.0.0.1:33919}
   [junit4]   2> 1336055 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.e.j.s.Server Started @1345397ms
   [junit4]   2> 1336055 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-003/collection1/data,
 hostContext=/solr, hostPort=33919}
   [junit4]   2> 1336056 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.3.0
   [junit4]   2> 1336056 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on 
port null
   [junit4]   2> 1336056 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1336056 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-09-28T12:47:03.188Z
   [junit4]   2> 1336057 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-003/solr.xml
   [junit4]   2> 1336075 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1336080 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-003/.
   [junit4]   2> 1336080 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1336100 WARN  (coreLoadExecutor-2863-thread-1) [    ] 
o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use 
<mergePolicyFactory> instead.
   [junit4]   2> 1336100 INFO  (coreLoadExecutor-2863-thread-1) [    ] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 1336109 INFO  (coreLoadExecutor-2863-thread-1) [    ] 
o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1336111 INFO  (coreLoadExecutor-2863-thread-1) [    ] 
o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1336112 INFO  (coreLoadExecutor-2863-thread-1) [    ] 
o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from 
instancedir 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-003/./collection1
   [junit4]   2> 1336112 INFO  (coreLoadExecutor-2863-thread-1) [    
x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-003/collection1],
 
dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-003/./collection1/data/]
   [junit4]   2> 1336113 INFO  (coreLoadExecutor-2863-thread-1) [    
x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: 
maxMergeAtOnce=23, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=5.208984375, 
floorSegmentMB=1.4287109375, forceMergeDeletesPctAllowed=10.27355101629424, 
segmentsPerTier=27.0, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.6130262309560954
   [junit4]   2> 1336122 INFO  (coreLoadExecutor-2863-thread-1) [    
x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1336123 INFO  (coreLoadExecutor-2863-thread-1) [    
x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1336123 INFO  (coreLoadExecutor-2863-thread-1) [    
x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class 
org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: 
minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, 
maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, 
maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, 
noCFSRatio=0.0]
   [junit4]   2> 1336123 INFO  (coreLoadExecutor-2863-thread-1) [    
x:collection1] o.a.s.s.SolrIndexSearcher Opening 
[Searcher@71f88eb8[collection1] main]
   [junit4]   2> 1336123 INFO  (coreLoadExecutor-2863-thread-1) [    
x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to 
use dir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-003/collection1/conf
   [junit4]   2> 1336124 INFO  (coreLoadExecutor-2863-thread-1) [    
x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following 
config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 1336124 INFO  (coreLoadExecutor-2863-thread-1) [    
x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1336124 INFO  
(searcherExecutor-2864-thread-1-processing-x:collection1) [    x:collection1] 
o.a.s.c.SolrCore [collection1] Registered new searcher 
Searcher@71f88eb8[collection1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1336603 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.SolrTestCaseJ4 Writing core.properties file to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-004/collection1
   [junit4]   2> 1336606 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1336612 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@7af804ea{/solr,null,AVAILABLE}
   [junit4]   2> 1336613 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.e.j.s.ServerConnector Started 
ServerConnector@4766ebcf{HTTP/1.1,[http/1.1]}{127.0.0.1:45927}
   [junit4]   2> 1336613 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.e.j.s.Server Started @1345955ms
   [junit4]   2> 1336613 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: 
{solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-004/collection1/data,
 hostContext=/solr, hostPort=45927}
   [junit4]   2> 1336613 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 
6.3.0
   [junit4]   2> 1336613 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in standalone mode on 
port null
   [junit4]   2> 1336613 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1336613 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-09-28T12:47:03.745Z
   [junit4]   2> 1336621 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.c.SolrXmlConfig Loading container configuration from 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-004/solr.xml
   [junit4]   2> 1336630 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1336633 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_6AE7F077D0FF419D-001/solr-instance-004/.
   [junit4]   2> 1336633 INFO  
(TEST-TestReplicationHandler.doTestRepeater-seed#[6AE7F077D0FF419D]) [    ] 
o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1336650 WARN  (coreLoadExecutor-2873-thread-1) [    ] 
o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use 
<mergePolicyFactory> instead.
   [junit4]   2> 1336659 INFO  (coreLoadExecutor-2873-thread-1) [    ] 
o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 1336676 INFO  (coreLoadExecutor-2873-thread-1) [    ] 
o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1336678 INFO  (coreLoadExecutor-2873-thread-1) [    ] 
o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
   [junit4]   2> 1336680 INFO  (co

[...truncated too long message...]

EST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[6AE7F077D0FF419D]) [   
 ] o.e.j.u.t.QueuedThreadPool qtp1292529395{STOPPING,8<=2656<=10000,i=0,q=0} 
Couldn't stop Thread[qtp1292529395-54167,5,]
   [junit4]   2> 1713590 WARN  
(TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[6AE7F077D0FF419D]) [ 
   ] o.e.j.u.t.QueuedThreadPool qtp1292529395{STOPPING,8<=2656<=10000,i=0,q=0} 
Couldn't stop Thread[qtp1292529395-54171,5,TGRP-CdcrVersionReplicationTest]
   [junit4]   2> 1713590 INFO  
(TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[6AE7F077D0FF419D]) [ 
   ] o.a.s.c.ChaosMonkey monkey: stop shard! 32799
   [junit4]   2> 1713590 INFO  
(TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[6AE7F077D0FF419D]) [ 
   ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=113818604
   [junit4]   2> 1713984 INFO  
(zkCallback-12878-thread-8-processing-n:127.0.0.1:32799_lr_qvd%2Ft) 
[n:127.0.0.1:32799_lr_qvd%2Ft    ] o.a.s.c.c.ZkStateReader A cluster state 
change: [WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/source_collection/state.json] for collection 
[source_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1714046 INFO  (coreCloseExecutor-3990-thread-1) 
[n:127.0.0.1:32799_lr_qvd%2Ft c:source_collection s:shard1 r:core_node3 
x:source_collection_shard1_replica1] o.a.s.c.SolrCore 
[source_collection_shard1_replica1]  CLOSING SolrCore 
org.apache.solr.core.SolrCore@26474c83
   [junit4]   2> 1714046 INFO  (coreCloseExecutor-3990-thread-1) 
[n:127.0.0.1:32799_lr_qvd%2Ft c:source_collection s:shard1 r:core_node3 
x:source_collection_shard1_replica1] o.a.s.h.CdcrRequestHandler Solr core is 
being closed - shutting down CDCR handler @ source_collection:shard1
   [junit4]   2> 1714330 INFO  
(TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[6AE7F077D0FF419D]) [ 
   ] o.a.s.c.Overseer Overseer 
(id=96669973770862605-127.0.0.1:32799_lr_qvd%2Ft-n_0000000003) closing
   [junit4]   2> 1714331 INFO  
(OverseerStateUpdate-96669973770862605-127.0.0.1:32799_lr_qvd%2Ft-n_0000000003) 
[n:127.0.0.1:32799_lr_qvd%2Ft    ] o.a.s.c.Overseer Overseer Loop exiting : 
127.0.0.1:32799_lr_qvd%2Ft
   [junit4]   2> 1715885 WARN  
(zkCallback-12878-thread-7-processing-n:127.0.0.1:32799_lr_qvd%2Ft) 
[n:127.0.0.1:32799_lr_qvd%2Ft    ] o.a.s.c.c.ZkStateReader ZooKeeper watch 
triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for 
/live_nodes]
   [junit4]   2> 1715886 INFO  
(TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[6AE7F077D0FF419D]) [ 
   ] o.e.j.s.ServerConnector Stopped 
ServerConnector@12f80805{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1715886 INFO  
(TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[6AE7F077D0FF419D]) [ 
   ] o.e.j.s.h.ContextHandler Stopped 
o.e.j.s.ServletContextHandler@6521f093{/lr_qvd/t,null,UNAVAILABLE}
   [junit4]   2> 1716214 WARN  
(TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[6AE7F077D0FF419D]) [ 
   ] o.e.j.u.t.QueuedThreadPool qtp1123913138{STOPPING,8<=2654<=10000,i=0,q=0} 
Couldn't stop Thread[qtp1123913138-54170,5,TGRP-CdcrVersionReplicationTest]
   [junit4]   2> 1716215 WARN  
(TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[6AE7F077D0FF419D]) [ 
   ] o.e.j.u.t.QueuedThreadPool qtp1123913138{STOPPING,8<=2654<=10000,i=0,q=0} 
Couldn't stop Thread[qtp1123913138-54168,5,]
   [junit4]   2> 1716215 WARN  
(TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[6AE7F077D0FF419D]) [ 
   ] o.e.j.u.t.QueuedThreadPool qtp1123913138{STOPPING,8<=2654<=10000,i=0,q=0} 
Couldn't stop Thread[qtp1123913138-54169,5,TGRP-CdcrVersionReplicationTest]
   [junit4]   2> 1716215 INFO  
(TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[6AE7F077D0FF419D]) [ 
   ] o.a.s.SolrTestCaseJ4 ###Ending testCdcrDocVersions
   [junit4]   2> 1716353 INFO  
(TEST-CdcrVersionReplicationTest.testCdcrDocVersions-seed#[6AE7F077D0FF419D]) [ 
   ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53961 53961
   [junit4]   2> 1716580 INFO  (Thread-955) [    ] o.a.s.c.ZkTestServer 
connecting to 127.0.0.1:53961 53961
   [junit4]   2> 1716607 WARN  (Thread-955) [    ] o.a.s.c.ZkTestServer Watch 
limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2>        17      /solr/aliases.json
   [junit4]   2>        17      /solr/clusterprops.json
   [junit4]   2>        8       /solr/configs/conf1
   [junit4]   2>        4       /solr/collections/tmp_collection/state.json
   [junit4]   2>        4       /solr/security.json
   [junit4]   2>        2       
/solr/collections/tmp_collection/leaders/shard2/leader
   [junit4]   2>        2       
/solr/collections/tmp_collection/leaders/shard1/leader
   [junit4]   2>        2       
/solr/collections/source_collection/leaders/shard1/leader
   [junit4]   2>        2       
/solr/collections/source_collection/leaders/shard2/leader
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2>        17      /solr/clusterstate.json
   [junit4]   2>        4       
/solr/collections/source_collection/cdcr/state/buffer
   [junit4]   2>        4       
/solr/collections/source_collection/cdcr/state/process
   [junit4]   2>        4       
/solr/collections/tmp_collection/cdcr/state/buffer
   [junit4]   2>        4       /solr/collections/source_collection/state.json
   [junit4]   2>        4       
/solr/collections/tmp_collection/cdcr/state/process
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        339     /solr/overseer/collection-queue-work
   [junit4]   2>        49      /solr/overseer/queue
   [junit4]   2>        17      /solr/live_nodes
   [junit4]   2>        17      /solr/collections
   [junit4]   2>        10      /solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  
-Dtestcase=CdcrVersionReplicationTest -Dtests.method=testCdcrDocVersions 
-Dtests.seed=6AE7F077D0FF419D -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=es -Dtests.timezone=America/Edmonton -Dtests.asserts=true 
-Dtests.file.encoding=US-ASCII
   [junit4] ERROR    465s J1 | CdcrVersionReplicationTest.testCdcrDocVersions 
<<<
   [junit4]    > Throwable #1: 
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught 
exception in thread: Thread[id=54168, name=qtp1123913138-54168, state=RUNNABLE, 
group=TGRP-CdcrVersionReplicationTest]
   [junit4]    > Caused by: java.lang.OutOfMemoryError: unable to create new 
native thread
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([6AE7F077D0FF419D]:0)
   [junit4]    >        at java.lang.Thread.start0(Native Method)
   [junit4]    >        at java.lang.Thread.start(Thread.java:714)
   [junit4]    >        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.startThreads(QueuedThreadPool.java:462)
   [junit4]    >        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$200(QueuedThreadPool.java:47)
   [junit4]    >        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:639)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)Throwable #2: 
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught 
exception in thread: Thread[id=54167, name=qtp1292529395-54167, state=RUNNABLE, 
group=TGRP-CdcrVersionReplicationTest]
   [junit4]    > Caused by: java.lang.OutOfMemoryError: unable to create new 
native thread
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([6AE7F077D0FF419D]:0)
   [junit4]    >        at java.lang.Thread.start0(Native Method)
   [junit4]    >        at java.lang.Thread.start(Thread.java:714)
   [junit4]    >        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.startThreads(QueuedThreadPool.java:462)
   [junit4]    >        at 
org.eclipse.jetty.util.thread.QueuedThreadPool.access$200(QueuedThreadPool.java:47)
   [junit4]    >        at 
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:639)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1716637 INFO  
(SUITE-CdcrVersionReplicationTest-seed#[6AE7F077D0FF419D]-worker) [    ] 
o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> Sep 28, 2016 12:53:23 PM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: 
codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST,
 chunkSize=17873, maxDocsPerChunk=7, blockSize=692), 
termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, 
chunkSize=17873, blockSize=692)), sim=ClassicSimilarity, locale=es, 
timezone=America/Edmonton
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 
1.8.0_102 (64-bit)/cpus=4,threads=1,free=81718944,total=478674944
   [junit4]   2> NOTE: All tests run in this JVM: [TestCustomSort, 
TestLocalFSCloudBackupRestore, TestSubQueryTransformerDistrib, DeleteShardTest, 
TestHdfsBackupRestoreCore, SmileWriterTest, TestStressVersions, 
AsyncMigrateRouteKeyTest, TestUniqueKeyFieldResource, TestMaxScoreQueryParser, 
AnalyticsQueryTest, SuggesterFSTTest, TestStressLiveNodes, 
HdfsBasicDistributedZk2Test, CdcrReplicationDistributedZkTest, 
TestInfoStreamLogging, ChaosMonkeySafeLeaderTest, CSVRequestHandlerTest, 
TestRawTransformer, BlobRepositoryCloudTest, TestReloadAndDeleteDocs, 
TestUpdate, TestRTimerTree, SuggestComponentContextFilterQueryTest, 
DocExpirationUpdateProcessorFactoryTest, CollectionsAPIDistributedZkTest, 
TestFieldCacheSanityChecker, SolrIndexConfigTest, 
DistribDocExpirationUpdateProcessorTest, SmileWriterTest, 
LeaderInitiatedRecoveryOnShardRestartTest, ExplicitHLLTest, SoftAutoCommitTest, 
AtomicUpdatesTest, TestElisionMultitermQuery, TestExpandComponent, 
TestDistributedGrouping, SaslZkACLProviderTest, HdfsSyncSliceTest, 
ConcurrentDeleteAndCreateCollectionTest, HdfsChaosMonkeySafeLeaderTest, 
TestFastOutputStream, TestBM25SimilarityFactory, TestConfigSetsAPIExclusivity, 
OutOfBoxZkACLAndCredentialsProvidersTest, SearchHandlerTest, 
CdcrVersionReplicationTest]
   [junit4] Completed [565/635 (2!)] on J1 in 465.84s, 1 test, 1 error <<< 
FAILURES!

[...truncated 60490 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Reply via email to