Build: https://builds.apache.org/job/Lucene-Solr-Tests-6.x/627/

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
  at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)  
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)  at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)  at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)  at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
  at sun.reflect.GeneratedConstructorAccessor187.newInstance(Unknown Source)  
at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  at java.lang.reflect.Constructor.newInstance(Constructor.java:423)  at 
org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)  at 
org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)  at 
org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)  at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)  at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)  at 
org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)  at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
 at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not 
released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
        at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
        at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
        at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
        at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
        at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
        at sun.reflect.GeneratedConstructorAccessor187.newInstance(Unknown 
Source)
        at 
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
        at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
        at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
        at org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
        at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
        at 
org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


        at __randomizedtesting.SeedInfo.seed([69CB894DA9B28985]:0)
        at org.junit.Assert.fail(Assert.java:93)
        at org.junit.Assert.assertTrue(Assert.java:43)
        at org.junit.Assert.assertNull(Assert.java:551)
        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
        at sun.reflect.GeneratedMethodAccessor22.invoke(Unknown Source)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
        at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 12137 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_69CB894DA9B28985-001/init-core-data-001
   [junit4]   2> 1298736 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] 
o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1298737 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] 
o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_69CB894DA9B28985-001/tempDir-001
   [junit4]   2> 1298737 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] 
o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1298745 INFO  (Thread-2304) [    ] o.a.s.c.ZkTestServer client 
port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1298745 INFO  (Thread-2304) [    ] o.a.s.c.ZkTestServer 
Starting server
   [junit4]   2> 1298845 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] 
o.a.s.c.ZkTestServer start zk server on port:39986
   [junit4]   2> 1298893 INFO  (jetty-launcher-1393-thread-1) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1298894 INFO  (jetty-launcher-1393-thread-2) [    ] 
o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 1298895 INFO  (jetty-launcher-1393-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@6974776d{/solr,null,AVAILABLE}
   [junit4]   2> 1298898 INFO  (jetty-launcher-1393-thread-1) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@30c2777f{HTTP/1.1,[http/1.1]}{127.0.0.1:58002}
   [junit4]   2> 1298898 INFO  (jetty-launcher-1393-thread-1) [    ] 
o.e.j.s.Server Started @1303323ms
   [junit4]   2> 1298898 INFO  (jetty-launcher-1393-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=58002}
   [junit4]   2> 1298898 ERROR (jetty-launcher-1393-thread-1) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 1298898 INFO  (jetty-launcher-1393-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
6.4.0
   [junit4]   2> 1298898 INFO  (jetty-launcher-1393-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1298898 INFO  (jetty-launcher-1393-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1298899 INFO  (jetty-launcher-1393-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-12-26T09:22:15.548Z
   [junit4]   2> 1298900 INFO  (jetty-launcher-1393-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@6582713{/solr,null,AVAILABLE}
   [junit4]   2> 1298901 INFO  (jetty-launcher-1393-thread-2) [    ] 
o.e.j.s.AbstractConnector Started 
ServerConnector@e23cff5{HTTP/1.1,[http/1.1]}{127.0.0.1:51437}
   [junit4]   2> 1298901 INFO  (jetty-launcher-1393-thread-2) [    ] 
o.e.j.s.Server Started @1303326ms
   [junit4]   2> 1298901 INFO  (jetty-launcher-1393-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=51437}
   [junit4]   2> 1298901 ERROR (jetty-launcher-1393-thread-2) [    ] 
o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be 
missing or incomplete.
   [junit4]   2> 1298901 INFO  (jetty-launcher-1393-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 
6.4.0
   [junit4]   2> 1298901 INFO  (jetty-launcher-1393-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1298901 INFO  (jetty-launcher-1393-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1298901 INFO  (jetty-launcher-1393-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 
2016-12-26T09:22:15.550Z
   [junit4]   2> 1298917 INFO  (jetty-launcher-1393-thread-1) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1298917 INFO  (jetty-launcher-1393-thread-2) [    ] 
o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1298940 INFO  (jetty-launcher-1393-thread-2) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1298941 INFO  (jetty-launcher-1393-thread-2) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39986/solr
   [junit4]   2> 1298951 INFO  (jetty-launcher-1393-thread-1) [    ] 
o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: 
socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1298952 INFO  (jetty-launcher-1393-thread-1) [    ] 
o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39986/solr
   [junit4]   2> 1298964 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] 
o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from 
client sessionid 0x1593a6ffeed0004, likely client has closed socket
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2>        at 
org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1298984 INFO  (jetty-launcher-1393-thread-2) 
[n:127.0.0.1:51437_solr    ] o.a.s.c.OverseerElectionContext I am going to be 
the leader 127.0.0.1:51437_solr
   [junit4]   2> 1298985 INFO  (jetty-launcher-1393-thread-2) 
[n:127.0.0.1:51437_solr    ] o.a.s.c.Overseer Overseer 
(id=97173119657902085-127.0.0.1:51437_solr-n_0000000000) starting
   [junit4]   2> 1298997 INFO  (jetty-launcher-1393-thread-1) 
[n:127.0.0.1:58002_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:58002_solr
   [junit4]   2> 1299013 INFO  
(zkCallback-1403-thread-1-processing-n:127.0.0.1:51437_solr) 
[n:127.0.0.1:51437_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 1299019 INFO  
(zkCallback-1404-thread-1-processing-n:127.0.0.1:58002_solr) 
[n:127.0.0.1:58002_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (0) -> (1)
   [junit4]   2> 1299022 INFO  (jetty-launcher-1393-thread-2) 
[n:127.0.0.1:51437_solr    ] o.a.s.c.ZkController Register node as live in 
ZooKeeper:/live_nodes/127.0.0.1:51437_solr
   [junit4]   2> 1299034 INFO  
(zkCallback-1404-thread-1-processing-n:127.0.0.1:58002_solr) 
[n:127.0.0.1:58002_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 1299034 INFO  
(zkCallback-1403-thread-1-processing-n:127.0.0.1:51437_solr) 
[n:127.0.0.1:51437_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from 
ZooKeeper... (1) -> (2)
   [junit4]   2> 1299206 INFO  (jetty-launcher-1393-thread-2) 
[n:127.0.0.1:51437_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_69CB894DA9B28985-001/tempDir-001/node2/.
   [junit4]   2> 1299315 INFO  (jetty-launcher-1393-thread-1) 
[n:127.0.0.1:58002_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core 
definitions underneath 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_69CB894DA9B28985-001/tempDir-001/node1/.
   [junit4]   2> 1299423 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] 
o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1299424 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] 
o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39986/solr ready
   [junit4]   2> 1300783 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] 
o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your 
platform... using builtin-java classes where applicable
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 1302355 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] 
o.a.h.m.i.MetricsConfig Cannot locate configuration: tried 
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 1302900 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] o.m.log 
Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via 
org.mortbay.log.Slf4jLog
   [junit4]   2> 1302942 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1303050 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 1303146 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] o.m.log 
Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs
 to ./temp/Jetty_localhost_36212_hdfs____arxgsa/webapp
   [junit4]   2> 1304498 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36212
   [junit4]   2> 1305701 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1305710 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 1305732 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] o.m.log 
Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_33467_datanode____.iz483o/webapp
   [junit4]   2> 1306627 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33467
   [junit4]   2> 1307706 WARN  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] 
o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1307711 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] o.m.log 
jetty-6.1.26
   [junit4]   2> 1307745 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] o.m.log 
Extract 
jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode
 to ./temp/Jetty_localhost_36793_datanode____otfdbp/webapp
   [junit4]   2> 1308792 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] o.m.log 
Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36793
   [junit4]   2> 1311579 INFO  (IPC Server handler 9 on 51476) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-616e3950-ba17-48ca-9124-05054643fb27 node 
DatanodeRegistration(127.0.0.1:53839, 
datanodeUuid=5437dfe4-3769-4bc8-a7cd-fdd6343646dd, infoPort=56585, 
infoSecurePort=0, ipcPort=52636, 
storageInfo=lv=-56;cid=testClusterID;nsid=2023701746;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 2 msecs
   [junit4]   2> 1311584 INFO  (IPC Server handler 9 on 51476) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-aa98bf58-ccf6-48f5-941e-f33a56cbf367 node 
DatanodeRegistration(127.0.0.1:53839, 
datanodeUuid=5437dfe4-3769-4bc8-a7cd-fdd6343646dd, infoPort=56585, 
infoSecurePort=0, ipcPort=52636, 
storageInfo=lv=-56;cid=testClusterID;nsid=2023701746;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1311592 INFO  (IPC Server handler 5 on 51476) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-1f5b7373-d1d7-4f4d-a0a3-ea1600fe2b0e node 
DatanodeRegistration(127.0.0.1:35460, 
datanodeUuid=581d1a47-d951-498d-b74e-dedce954d948, infoPort=34576, 
infoSecurePort=0, ipcPort=55078, 
storageInfo=lv=-56;cid=testClusterID;nsid=2023701746;c=0), blocks: 0, 
hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 1311608 INFO  (IPC Server handler 5 on 51476) [    ] 
BlockStateChange BLOCK* processReport: from storage 
DS-3b8cdd87-bba7-4881-b65d-579053a4749f node 
DatanodeRegistration(127.0.0.1:35460, 
datanodeUuid=581d1a47-d951-498d-b74e-dedce954d948, infoPort=34576, 
infoSecurePort=0, ipcPort=55078, 
storageInfo=lv=-56;cid=testClusterID;nsid=2023701746;c=0), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1312756 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[69CB894DA9B28985]) [    ] 
o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 1312767 INFO  (qtp947469995-8804) [n:127.0.0.1:58002_solr    ] 
o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params 
replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2
 and sendToOCPQueue=true
   [junit4]   2> 1312845 INFO  
(OverseerThreadFactory-4242-thread-1-processing-n:127.0.0.1:51437_solr) 
[n:127.0.0.1:51437_solr    ] o.a.s.c.CreateCollectionCmd Create collection 
recoverytest
   [junit4]   2> 1313005 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr    
] o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1313165 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr    ] 
o.a.s.h.a.CoreAdminOperation core create command 
qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1313274 INFO  
(zkCallback-1404-thread-1-processing-n:127.0.0.1:58002_solr) 
[n:127.0.0.1:58002_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 1313274 INFO  
(zkCallback-1403-thread-6-processing-n:127.0.0.1:51437_solr) 
[n:127.0.0.1:51437_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 1314190 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 6.4.0
   [junit4]   2> 1314196 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig 
Using Lucene MatchVersion: 6.4.0
   [junit4]   2> 1314278 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema 
[recoverytest_shard1_replica2] Schema name=minimal
   [junit4]   2> 1314281 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema 
Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1314282 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer 
Creating SolrCore 'recoverytest_shard1_replica2' using configuration from 
collection recoverytest
   [junit4]   2> 1314292 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema 
[recoverytest_shard1_replica1] Schema name=minimal
   [junit4]   2> 1314329 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:51476/data
   [junit4]   2> 1314329 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 1314330 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SolrCore 
[[recoverytest_shard1_replica2] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_69CB894DA9B28985-001/tempDir-001/node1/recoverytest_shard1_replica2],
 dataDir=[hdfs://localhost:51476/data/recoverytest/core_node2/data/]
   [junit4]   2> 1314331 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:51476/data/recoverytest/core_node2/data/snapshot_metadata
   [junit4]   2> 1314339 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema 
Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1314345 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer 
Creating SolrCore 'recoverytest_shard1_replica1' using configuration from 
collection recoverytest
   [junit4]   2> 1314346 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:51476/data
   [junit4]   2> 1314380 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 1314380 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.SolrCore 
[[recoverytest_shard1_replica1] ] Opening new SolrCore at 
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-6.x/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_69CB894DA9B28985-001/tempDir-001/node2/recoverytest_shard1_replica1],
 dataDir=[hdfs://localhost:51476/data/recoverytest/core_node1/data/]
   [junit4]   2> 1314380 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:51476/data/recoverytest/core_node1/data/snapshot_metadata
   [junit4]   2> 1314418 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 1314418 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 1314418 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Creating new global HDFS BlockCache
   [junit4]   2> 1314427 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 1314427 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 1316771 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1316777 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1316784 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:51476/data/recoverytest/core_node2/data
   [junit4]   2> 1316803 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:51476/data/recoverytest/core_node1/data
   [junit4]   2> 1316904 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:51476/data/recoverytest/core_node2/data/index
   [junit4]   2> 1316925 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory creating directory factory for path 
hdfs://localhost:51476/data/recoverytest/core_node1/data/index
   [junit4]   2> 1316934 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 1316934 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 1316940 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct 
memory allocation set to [true]
   [junit4]   2> 1316940 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of 
[16777216] will allocate [1] slabs and use ~[16777216] bytes
   [junit4]   2> 1316946 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1316963 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.b.BlockDirectory Block cache on write is disabled
   [junit4]   2> 1317918 INFO  (IPC Server handler 9 on 51476) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53839 is 
added to blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-1f5b7373-d1d7-4f4d-a0a3-ea1600fe2b0e:NORMAL:127.0.0.1:35460|RBW],
 
ReplicaUC[[DISK]DS-616e3950-ba17-48ca-9124-05054643fb27:NORMAL:127.0.0.1:53839|RBW]]}
 size 71
   [junit4]   2> 1317920 INFO  (IPC Server handler 3 on 51476) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:35460 is 
added to blk_1073741825_1001 size 71
   [junit4]   2> 1317920 INFO  (IPC Server handler 4 on 51476) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:53839 is 
added to blk_1073741826_1002{UCState=COMMITTED, truncateBlock=null, 
primaryNodeIndex=-1, 
replicas=[ReplicaUC[[DISK]DS-3b8cdd87-bba7-4881-b65d-579053a4749f:NORMAL:127.0.0.1:35460|RBW],
 
ReplicaUC[[DISK]DS-616e3950-ba17-48ca-9124-05054643fb27:NORMAL:127.0.0.1:53839|RBW]]}
 size 71
   [junit4]   2> 1317930 INFO  (IPC Server handler 0 on 51476) [    ] 
BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:35460 is 
added to blk_1073741826_1002 size 71
   [junit4]   2> 1318451 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 1318451 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1318452 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog 
Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 1318465 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler 
Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 1318465 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog 
Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH 
numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1318465 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog 
Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 1318488 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 1318488 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 1318616 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker 
Hard AutoCommit: disabled
   [junit4]   2> 1318616 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker 
Soft AutoCommit: disabled
   [junit4]   2> 1318740 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@212e3416[recoverytest_shard1_replica2] main]
   [junit4]   2> 1318741 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 1318742 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1318744 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1318745 INFO  
(searcherExecutor-4247-thread-1-processing-n:127.0.0.1:58002_solr 
x:recoverytest_shard1_replica2 s:shard1 c:recoverytest) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SolrCore 
[recoverytest_shard1_replica2] Registered new searcher 
Searcher@212e3416[recoverytest_shard1_replica2] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1318745 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1554769935486418944
   [junit4]   2> 1318752 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for 
shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 1318754 INFO  
(zkCallback-1404-thread-1-processing-n:127.0.0.1:58002_solr) 
[n:127.0.0.1:58002_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 1318754 INFO  
(zkCallback-1403-thread-5-processing-n:127.0.0.1:51437_solr) 
[n:127.0.0.1:51437_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 1318756 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.s.SolrIndexSearcher Opening 
[Searcher@30dee72a[recoverytest_shard1_replica1] main]
   [junit4]   2> 1318760 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: 
/configs/conf
   [junit4]   2> 1318761 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using 
ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1318761 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] 
o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1318761 INFO  
(searcherExecutor-4248-thread-1-processing-n:127.0.0.1:51437_solr 
x:recoverytest_shard1_replica1 s:shard1 c:recoverytest) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.c.SolrCore 
[recoverytest_shard1_replica1] Registered new searcher 
Searcher@30dee72a[recoverytest_shard1_replica1] 
main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1318761 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog 
Could not find max version in index or recent updates, using new clock 
1554769935503196160
   [junit4]   2> 1319253 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1319253 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1319253 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy 
Sync replicas to http://127.0.0.1:58002/solr/recoverytest_shard1_replica2/
   [junit4]   2> 1319253 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.PeerSync 
PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:58002/solr 
START replicas=[http://127.0.0.1:51437/solr/recoverytest_shard1_replica1/] 
nUpdates=100
   [junit4]   2> 1319254 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.u.PeerSync 
PeerSync: core=recoverytest_shard1_replica2 url=http://127.0.0.1:58002/solr 
DONE.  We have no versions.  sync failed.
   [junit4]   2> 1319274 INFO  (qtp1407997440-8812) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.c.S.Request [recoverytest_shard1_replica1]  webapp=/solr path=/get 
params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2}
 status=0 QTime=0
   [junit4]   2> 1319274 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.SyncStrategy 
Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 1319274 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we 
can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1319278 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] 
o.a.s.c.ShardLeaderElectionContext I am the new leader: 
http://127.0.0.1:58002/solr/recoverytest_shard1_replica2/ shard1
   [junit4]   2> 1319383 INFO  
(zkCallback-1403-thread-5-processing-n:127.0.0.1:51437_solr) 
[n:127.0.0.1:51437_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 1319383 INFO  
(zkCallback-1404-thread-1-processing-n:127.0.0.1:58002_solr) 
[n:127.0.0.1:58002_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 1319431 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.c.ZkController I 
am the leader, no recovery necessary
   [junit4]   2> 1319433 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2}
 status=0 QTime=6268
   [junit4]   2> 1319535 INFO  
(zkCallback-1404-thread-1-processing-n:127.0.0.1:58002_solr) 
[n:127.0.0.1:58002_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 1319536 INFO  
(zkCallback-1403-thread-5-processing-n:127.0.0.1:51437_solr) 
[n:127.0.0.1:51437_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 1319771 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1  x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall 
[admin] webapp=null path=/admin/cores 
params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2}
 status=0 QTime=6766
   [junit4]   2> 1319774 INFO  (qtp947469995-8804) [n:127.0.0.1:58002_solr    ] 
o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 
30 seconds. Check all shard replicas
   [junit4]   2> 1319891 INFO  
(zkCallback-1404-thread-1-processing-n:127.0.0.1:58002_solr) 
[n:127.0.0.1:58002_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 1319891 INFO  
(zkCallback-1403-thread-6-processing-n:127.0.0.1:51437_solr) 
[n:127.0.0.1:51437_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: 
[WatchedEvent state:SyncConnected type:NodeDataChanged 
path:/collections/recoverytest/state.json] for collection [recoverytest] has 
occurred - updating... (live nodes size: [2])
   [junit4]   2> 1320775 INFO  (qtp947469995-8804) [n:127.0.0.1:58002_solr    ] 
o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections 
params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2}
 status=0 QTime=8007
   [junit4]   2> 1320776 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[69CB894DA9B28985]) [    ] 
o.a.s.c.RecoveryZkTest Indexing 1350 documents
   [junit4]   2> 1320852 INFO  
(OverseerCollectionConfigSetProcessor-97173119657902085-127.0.0.1:51437_solr-n_0000000000)
 [n:127.0.0.1:51437_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: 
/overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may 
have disconnected from ZooKeeper
   [junit4]   2> 1320968 INFO  (qtp1407997440-8816) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:58002/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-0
 (1554769937650679808)]} 0 92
   [junit4]   2> 1320969 INFO  (qtp947469995-8805) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 
(1554769937650679808)]} 0 159
   [junit4]   2> 1320970 INFO  (qtp1407997440-8818) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:58002/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-0
 (1554769937654874112)]} 0 93
   [junit4]   2> 1320970 INFO  (qtp947469995-8806) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 
(1554769937654874112)]} 0 157
   [junit4]   2> 1320976 INFO  (qtp1407997440-8811) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:58002/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[1-1
 (1554769937820549120)]} 0 2
   [junit4]   2> 1320976 INFO  (qtp947469995-8801) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 
(1554769937820549120)]} 0 4
   [junit4]   2> 1320977 INFO  (qtp1407997440-8812) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:58002/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{add=[2-1
 (1554769937821597696)]} 0 3
   [junit4]   2> 1320977 INFO  (qtp947469995-8807) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 
(1554769937821597696)]} 0 5
   [junit4]   2> 1320982 INFO  (qtp1407997440-8814) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:58002/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[1-1
 (-1554769937827889152)]} 0 1
   [junit4]   2> 1320984 INFO  (qtp1407997440-8817) [n:127.0.0.1:51437_solr 
c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1]  
webapp=/solr path=/update 
params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:58002/solr/recoverytest_shard1_replica2/&wt=javabin&version=2}{delete=[2-1
 (-1554769937828937728)]} 0 2
   [junit4]   2> 1320984 INFO  (qtp947469995-8802) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-1 
(-1554769937828937728)]} 0 5
   [junit4]   2> 1320988 INFO  (qtp947469995-8808) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-1 
(-1554769937827889152)]} 0 9
   [junit4]   2> 1320990 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[69CB894DA9B28985]) [    ] 
o.e.j.s.AbstractConnector Stopped 
ServerConnector@e23cff5{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1321001 INFO  
(TEST-HdfsRecoveryZkTest.test-seed#[69CB894DA9B28985]) [    ] 
o.a.s.c.CoreContainer Shutting down CoreContainer instance=1454658856
   [junit4]   2> 1321001 ERROR 
(updateExecutor-1400-thread-1-processing-http:////127.0.0.1:51437//solr//recoverytest_shard1_replica1
 x:recoverytest_shard1_replica2 r:core_node2 n:127.0.0.1:58002_solr s:shard1 
c:recoverytest) [n:127.0.0.1:58002_solr c:recoverytest s:shard1 r:core_node2 
x:recoverytest_shard1_replica2] o.a.s.u.StreamingSolrClients error
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321002 WARN  (qtp947469995-8804) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Error sending update to 
http://127.0.0.1:51437/solr
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321002 ERROR (qtp947469995-8804) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on 
replica http://127.0.0.1:51437/solr/recoverytest_shard1_replica1/
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321007 ERROR 
(updateExecutor-1400-thread-2-processing-http:////127.0.0.1:51437//solr//recoverytest_shard1_replica1
 x:recoverytest_shard1_replica2 r:core_node2 n:127.0.0.1:58002_solr s:shard1 
c:recoverytest) [n:127.0.0.1:58002_solr c:recoverytest s:shard1 r:core_node2 
x:recoverytest_shard1_replica2] o.a.s.u.StreamingSolrClients error
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321008 WARN  (qtp947469995-8805) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Error sending update to 
http://127.0.0.1:51437/solr
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321008 ERROR (qtp947469995-8805) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on 
replica http://127.0.0.1:51437/solr/recoverytest_shard1_replica1/
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321012 INFO  (coreCloseExecutor-4257-thread-1) 
[n:127.0.0.1:51437_solr c:recoverytest s:shard1 r:core_node1 
x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] 
 CLOSING SolrCore org.apache.solr.core.SolrCore@627963d
   [junit4]   2> 1321012 INFO  (qtp947469995-8804) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.c.ZkController Put replica core=recoverytest_shard1_replica1 
coreNodeName=core_node1 on 127.0.0.1:51437_solr into leader-initiated recovery.
   [junit4]   2> 1321013 INFO  (qtp947469995-8805) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 
(1554769937840472064)]} 0 23
   [junit4]   2> 1321013 INFO  (qtp947469995-8804) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 
(1554769937850957824)]} 0 12
   [junit4]   2> 1321018 INFO  
(updateExecutor-1400-thread-2-processing-x:recoverytest_shard1_replica2 
r:core_node2 http:////127.0.0.1:51437//solr//recoverytest_shard1_replica1// 
n:127.0.0.1:58002_solr s:shard1 c:recoverytest) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.c.LeaderInitiatedRecoveryThread Put replica 
core=recoverytest_shard1_replica1 coreNodeName=core_node1 on 
127.0.0.1:51437_solr into leader-initiated recovery.
   [junit4]   2> 1321018 WARN  
(updateExecutor-1400-thread-2-processing-x:recoverytest_shard1_replica2 
r:core_node2 http:////127.0.0.1:51437//solr//recoverytest_shard1_replica1// 
n:127.0.0.1:58002_solr s:shard1 c:recoverytest) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.c.LeaderInitiatedRecoveryThread Leader is publishing 
core=recoverytest_shard1_replica1 coreNodeName =core_node1 state=down on behalf 
of un-reachable replica 
http://127.0.0.1:51437/solr/recoverytest_shard1_replica1/
   [junit4]   2> 1321020 INFO  
(updateExecutor-1400-thread-2-processing-x:recoverytest_shard1_replica2 
r:core_node2 http:////127.0.0.1:51437//solr//recoverytest_shard1_replica1// 
n:127.0.0.1:58002_solr s:shard1 c:recoverytest) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.c.LeaderInitiatedRecoveryThread 
LeaderInitiatedRecoveryThread-recoverytest_shard1_replica1 started running to 
send REQUESTRECOVERY command to 
http://127.0.0.1:51437/solr/recoverytest_shard1_replica1/; will try for a max 
of 600 secs
   [junit4]   2> 1321021 INFO  
(updateExecutor-1400-thread-2-processing-x:recoverytest_shard1_replica2 
r:core_node2 http:////127.0.0.1:51437//solr//recoverytest_shard1_replica1// 
n:127.0.0.1:58002_solr s:shard1 c:recoverytest) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.c.LeaderInitiatedRecoveryThread Asking core=recoverytest_shard1_replica1 
coreNodeName=core_node1 on http://127.0.0.1:51437/solr to recover
   [junit4]   2> 1321023 ERROR 
(updateExecutor-1400-thread-1-processing-http:////127.0.0.1:51437//solr//recoverytest_shard1_replica1
 x:recoverytest_shard1_replica2 r:core_node2 n:127.0.0.1:58002_solr s:shard1 
c:recoverytest) [n:127.0.0.1:58002_solr c:recoverytest s:shard1 r:core_node2 
x:recoverytest_shard1_replica2] o.a.s.u.StreamingSolrClients error
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321024 WARN  (qtp947469995-8806) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Error sending update to 
http://127.0.0.1:51437/solr
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321024 ERROR (qtp947469995-8806) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on 
replica http://127.0.0.1:51437/solr/recoverytest_shard1_replica1/
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321025 INFO  (qtp947469995-8806) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 
(1554769937875075072)]} 0 1
   [junit4]   2> 1321038 ERROR 
(updateExecutor-1400-thread-2-processing-x:recoverytest_shard1_replica2 
r:core_node2 http:////127.0.0.1:51437//solr//recoverytest_shard1_replica1// 
n:127.0.0.1:58002_solr s:shard1 c:recoverytest) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.c.LeaderInitiatedRecoveryThread http://127.0.0.1:51437/solr: Could not 
tell a replica to recover:org.apache.solr.client.solrj.SolrServerException: 
Server refused connection at: http://127.0.0.1:51437/solr
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:617)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:279)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:268)
   [junit4]   2>        at 
org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2>        at 
org.apache.solr.cloud.LeaderInitiatedRecoveryThread.sendRecoveryCommandWithRetry(LeaderInitiatedRecoveryThread.java:208)
   [junit4]   2>        at 
org.apache.solr.cloud.LeaderInitiatedRecoveryThread.run(LeaderInitiatedRecoveryThread.java:95)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:515)
   [junit4]   2>        ... 9 more
   [junit4]   2> 
   [junit4]   2> 1321042 ERROR 
(updateExecutor-1400-thread-1-processing-http:////127.0.0.1:51437//solr//recoverytest_shard1_replica1
 x:recoverytest_shard1_replica2 r:core_node2 n:127.0.0.1:58002_solr s:shard1 
c:recoverytest) [n:127.0.0.1:58002_solr c:recoverytest s:shard1 r:core_node2 
x:recoverytest_shard1_replica2] o.a.s.u.StreamingSolrClients error
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321043 WARN  (qtp947469995-8801) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Error sending update to 
http://127.0.0.1:51437/solr
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321043 ERROR (qtp947469995-8801) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on 
replica http://127.0.0.1:51437/solr/recoverytest_shard1_replica1/
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321044 INFO  (qtp947469995-8801) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2]  
webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 
(1554769937877172224)]} 0 19
   [junit4]   2> 1321046 ERROR 
(updateExecutor-1400-thread-3-processing-http:////127.0.0.1:51437//solr//recoverytest_shard1_replica1
 x:recoverytest_shard1_replica2 r:core_node2 n:127.0.0.1:58002_solr s:shard1 
c:recoverytest) [n:127.0.0.1:58002_solr c:recoverytest s:shard1 r:core_node2 
x:recoverytest_shard1_replica2] o.a.s.u.StreamingSolrClients error
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321047 WARN  (qtp947469995-8806) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Error sending update to 
http://127.0.0.1:51437/solr
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]   2>        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:311)
   [junit4]   2>        at 
org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:184)
   [junit4]   2>        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2>        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1321047 ERROR (qtp947469995-8806) [n:127.0.0.1:58002_solr 
c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] 
o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on 
replica http://127.0.0.1:51437/solr/recoverytest_shard1_replica1/
   [junit4]   2> java.net.ConnectException: Connection refused
   [junit4]   2>        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]   2>        at 
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]   2>        at 
java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2>        at java.net.Socket.connect(Socket.java:589)
   [junit4]   2>        at 
org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
   [junit4]   2>        at 
org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
   [junit4]   2>        at 
org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
   [junit4]   2>        at 
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
   [junit4]   2>        at 
org.apache.http.impl.client.AbstractHttpClient.doExecute(Abstrac

[...truncated too long message...]

emImpl.java:463)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2>        at 
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2>        at 
org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2>        at 
org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest.teardownClass(HdfsRecoveryZkTest.java:51)
   [junit4]   2>        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native 
Method)
   [junit4]   2>        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>        at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2>        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2>        at 
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2>        ... 54 more
   [junit4]   2> 1339821 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] 
o.a.s.c.ZkTestServer connecting to 127.0.0.1:39986 39986
   [junit4]   2> 1339822 INFO  
(SUITE-HdfsRecoveryZkTest-seed#[69CB894DA9B28985]-worker) [    ] 
o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: test params are: 
codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION,
 chunkSize=1472, maxDocsPerChunk=979, blockSize=7), 
termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION,
 chunkSize=1472, blockSize=7)), 
sim=RandomSimilarity(queryNorm=false,coord=yes): {}, locale=it-CH, 
timezone=Asia/Tbilisi
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 
1.8.0_102 (64-bit)/cpus=4,threads=5,free=199447240,total=521142272
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrDeletionPolicy2, 
AnalysisAfterCoreReloadTest, TestSolr4Spatial2, 
ParsingFieldUpdateProcessorsTest, TestFunctionQuery, 
TestCollationFieldDocValues, TestNonDefinedSimilarityFactory, 
TestPivotHelperCode, SmileWriterTest, HdfsBasicDistributedZkTest, 
XsltUpdateRequestHandlerTest, TestDistributedSearch, DateFieldTest, 
TestManagedStopFilterFactory, RegexBoostProcessorTest, ShardSplitTest, 
TestPseudoReturnFields, CreateCollectionCleanupTest, RuleEngineTest, 
TestRawTransformer, TestQueryWrapperFilter, TestSolrConfigHandler, 
TestComponentsName, DocValuesMissingTest, DistributedSuggestComponentTest, 
TestDownShardTolerantSearch, DistributedFacetPivotLongTailTest, 
HttpSolrCallGetCoreTest, StandardRequestHandlerTest, CursorMarkTest, 
TestCloudRecovery, TestTolerantUpdateProcessorCloud, CdcrUpdateLogTest, 
SegmentsInfoRequestHandlerTest, TestFieldTypeResource, ExplicitHLLTest, 
UUIDFieldTest, ScriptEngineTest, FastVectorHighlighterTest, OverseerRolesTest, 
TestUseDocValuesAsStored2, TestDocumentBuilder, TestSolr4Spatial, 
DirectoryFactoryTest, ZkNodePropsTest, TestScoreJoinQPScore, TestMissingGroups, 
BooleanFieldTest, TestRandomFlRTGCloud, SignatureUpdateProcessorFactoryTest, 
HighlighterConfigTest, TestCSVLoader, DistanceUnitsTest, 
TestLocalFSCloudBackupRestore, ZkStateReaderTest, 
ShufflingReplicaListTransformerTest, TestManagedSchemaThreadSafety, 
DistributedSpellCheckComponentTest, TestHighlightDedupGrouping, 
QueryEqualityTest, CoreMergeIndexesAdminHandlerTest, 
TestSolrConfigHandlerConcurrent, BasicAuthIntegrationTest, TestRealTimeGet, 
AsyncCallRequestStatusResponseTest, DeleteShardTest, CacheHeaderTest, TestJoin, 
TestCorePropertiesReload, MBeansHandlerTest, SolrCoreTest, 
TestPerFieldSimilarity, TestReplicationHandler, TestSQLHandler, 
BasicDistributedZkTest, TestStressLucene, TestConfigSetImmutable, 
TestManagedResource, TestHashQParserPlugin, TestOmitPositions, 
TestCodecSupport, UniqFieldsUpdateProcessorFactoryTest, 
TriLevelCompositeIdRoutingTest, SolrRequestParserTest, ClusterStateTest, 
CdcrVersionReplicationTest, CdcrReplicationDistributedZkTest, 
TestJavabinTupleStreamParser, WrapperMergePolicyFactoryTest, 
DeleteLastCustomShardedReplicaTest, TestSolrQueryParserDefaultOperatorResource, 
TestCrossCoreJoin, TestFoldingMultitermQuery, TestUninvertingReader, 
TestRemoteStreaming, CurrencyFieldOpenExchangeTest, NumericFieldsTest, 
HdfsCollectionsAPIDistributedZkTest, PeerSyncReplicationTest, 
HdfsRecoveryZkTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HdfsRecoveryZkTest 
-Dtests.seed=69CB894DA9B28985 -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=it-CH -Dtests.timezone=Asia/Tbilisi -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | HdfsRecoveryZkTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 
object(s) that were not released!!! [HdfsTransactionLog]
   [junit4]    > 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
   [junit4]    >        at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
   [junit4]    >        at 
org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
   [junit4]    >        at 
org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
   [junit4]    >        at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
   [junit4]    >        at 
org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
   [junit4]    >        at 
org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
   [junit4]    >        at 
java.lang.reflect.Constructor.newInstance(Constructor.java:423)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.createInstance(SolrCore.java:704)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:766)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1005)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:870)
   [junit4]    >        at 
org.apache.solr.core.SolrCore.<init>(SolrCore.java:774)
   [junit4]    >        at 
org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
   [junit4]    >        at 
org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
   [junit4]    >        at 
java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    >        at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    >        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    >        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4]    >        at 
__randomizedtesting.SeedInfo.seed([69CB894DA9B28985]:0)
   [junit4]    >        at 
org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:266)
   [junit4]    >        at java.lang.Thread.run(Thread.java:745)
   [junit4] Completed [445/665 (1!)] on J0 in 72.27s, 1 test, 1 failure <<< 
FAILURES!

[...truncated 63126 lines...]


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Reply via email to