Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2126/
1 tests failed. REGRESSION: org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.testDistribSearch Error Message: No live SolrServers available to handle this request:[http://127.0.0.1:33393/d/un, http://127.0.0.1:33442/d/un, http://127.0.0.1:33233/d/un] Stack Trace: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:33393/d/un, http://127.0.0.1:33442/d/un, http://127.0.0.1:33233/d/un] at __randomizedtesting.SeedInfo.seed([CD047FFD643DDB10:4CE2F1E51362BB2C]:0) at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:322) at org.apache.solr.client.solrj.impl.CloudSolrServer.sendRequest(CloudSolrServer.java:880) at org.apache.solr.client.solrj.impl.CloudSolrServer.requestWithRetryOnStaleState(CloudSolrServer.java:658) at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:601) at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.removeAndWaitForLastReplicaGone(DeleteLastCustomShardedReplicaTest.java:117) at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.doTest(DeleteLastCustomShardedReplicaTest.java:107) at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863) at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53) at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55) at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365) at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798) at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458) at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836) at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738) at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772) at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Could not remove replica : customcollreplicadeletion/a/core_node1 at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:550) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206) at org.apache.solr.client.solrj.impl.LBHttpSolrServer.doRequest(LBHttpSolrServer.java:343) at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:304) ... 46 more Build Log: [...truncated 11462 lines...] [junit4] Suite: org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest [junit4] 2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/init-core-data-001 [junit4] 2> 1534371 T3686 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true) [junit4] 2> 1534371 T3686 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /d/un [junit4] 2> 1534376 T3686 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch [junit4] 2> 1534377 T3686 oasc.ZkTestServer.run STARTING ZK TEST SERVER [junit4] 1> client port:0.0.0.0/0.0.0.0:0 [junit4] 2> 1534378 T3687 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server [junit4] 2> 1534478 T3686 oasc.ZkTestServer.run start zk server on port:33228 [junit4] 2> 1534479 T3686 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider [junit4] 2> 1534480 T3686 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper [junit4] 2> 1534483 T3693 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a0b3811 name:ZooKeeperConnection Watcher:127.0.0.1:33228 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 1534483 T3686 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper [junit4] 2> 1534483 T3686 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider [junit4] 2> 1534484 T3686 oascc.SolrZkClient.makePath makePath: /solr [junit4] 2> 1534486 T3686 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider [junit4] 2> 1534488 T3686 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper [junit4] 2> 1534489 T3695 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61a37e92 name:ZooKeeperConnection Watcher:127.0.0.1:33228/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 1534489 T3686 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper [junit4] 2> 1534489 T3686 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider [junit4] 2> 1534489 T3686 oascc.SolrZkClient.makePath makePath: /collections/collection1 [junit4] 2> 1534491 T3686 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards [junit4] 2> 1534493 T3686 oascc.SolrZkClient.makePath makePath: /collections/control_collection [junit4] 2> 1534494 T3686 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards [junit4] 2> 1534495 T3686 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml [junit4] 2> 1534496 T3686 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml [junit4] 2> 1534498 T3686 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml [junit4] 2> 1534498 T3686 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml [junit4] 2> 1534500 T3686 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml [junit4] 2> 1534500 T3686 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml [junit4] 2> 1534502 T3686 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt [junit4] 2> 1534502 T3686 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt [junit4] 2> 1534504 T3686 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt [junit4] 2> 1534504 T3686 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt [junit4] 2> 1534505 T3686 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml [junit4] 2> 1534506 T3686 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml [junit4] 2> 1534507 T3686 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml [junit4] 2> 1534507 T3686 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml [junit4] 2> 1534509 T3686 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json [junit4] 2> 1534509 T3686 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json [junit4] 2> 1534511 T3686 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt [junit4] 2> 1534511 T3686 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt [junit4] 2> 1534512 T3686 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt [junit4] 2> 1534513 T3686 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt [junit4] 2> 1534514 T3686 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt [junit4] 2> 1534515 T3686 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt [junit4] 2> 1535852 T3686 oejs.Server.doStart jetty-8.1.10.v20130312 [junit4] 2> 1535858 T3686 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33233 [junit4] 2> 1535858 T3686 oass.SolrDispatchFilter.init SolrDispatchFilter.init() [junit4] 2> 1535859 T3686 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx) [junit4] 2> 1535859 T3686 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002 [junit4] 2> 1535859 T3686 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002/' [junit4] 2> 1535892 T3686 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002/solr.xml [junit4] 2> 1535940 T3686 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002 [junit4] 2> 1535940 T3686 oasc.CoreContainer.<init> New CoreContainer 1876134725 [junit4] 2> 1535940 T3686 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002/] [junit4] 2> 1535941 T3686 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000 [junit4] 2> 1535941 T3686 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: [junit4] 2> 1535942 T3686 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000 [junit4] 2> 1535942 T3686 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20 [junit4] 2> 1535942 T3686 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0 [junit4] 2> 1535942 T3686 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647 [junit4] 2> 1535943 T3686 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5 [junit4] 2> 1535943 T3686 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1 [junit4] 2> 1535943 T3686 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false [junit4] 2> 1535943 T3686 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false [junit4] 2> 1535944 T3686 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory [junit4] 2> 1535944 T3686 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)] [junit4] 2> 1535944 T3686 oasc.CoreContainer.load Host Name: 127.0.0.1 [junit4] 2> 1535944 T3686 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33228/solr [junit4] 2> 1535945 T3686 oasc.ZkController.checkChrootPath zkHost includes chroot [junit4] 2> 1535945 T3686 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider [junit4] 2> 1535946 T3686 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper [junit4] 2> 1535947 T3706 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@295c985e name:ZooKeeperConnection Watcher:127.0.0.1:33228 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 1535948 T3686 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper [junit4] 2> 1535948 T3686 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider [junit4] 2> 1535950 T3686 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper [junit4] 2> 1535951 T3708 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9deb0db name:ZooKeeperConnection Watcher:127.0.0.1:33228/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 1535952 T3686 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper [junit4] 2> 1535953 T3686 oascc.SolrZkClient.makePath makePath: /overseer/queue [junit4] 2> 1535954 T3686 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work [junit4] 2> 1535956 T3686 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running [junit4] 2> 1535957 T3686 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed [junit4] 2> 1535959 T3686 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure [junit4] 2> 1535960 T3686 oascc.SolrZkClient.makePath makePath: /live_nodes [junit4] 2> 1535961 T3686 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33233_d%2Fun [junit4] 2> 1535961 T3686 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33233_d%2Fun [junit4] 2> 1535963 T3686 oascc.SolrZkClient.makePath makePath: /overseer_elect [junit4] 2> 1535964 T3686 oascc.SolrZkClient.makePath makePath: /overseer_elect/election [junit4] 2> 1535965 T3686 oasc.Overseer.close Overseer (id=null) closing [junit4] 2> 1535966 T3686 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:33233_d%2Fun [junit4] 2> 1535966 T3686 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader [junit4] 2> 1535967 T3686 oasc.Overseer.start Overseer (id=92492249834979331-127.0.0.1:33233_d%2Fun-n_0000000000) starting [junit4] 2> 1535968 T3686 oascc.SolrZkClient.makePath makePath: /overseer/queue-work [junit4] 2> 1535973 T3686 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000 [junit4] 2> 1535973 T3710 oasc.OverseerCollectionProcessor.run Process current queue of collection creations [junit4] 2> 1535974 T3686 oascc.SolrZkClient.makePath makePath: /clusterstate.json [junit4] 2> 1535975 T3686 oascc.SolrZkClient.makePath makePath: /aliases.json [junit4] 2> 1535976 T3686 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... [junit4] 2> 1535978 T3709 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue [junit4] 2> 1535980 T3686 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002 [junit4] 2> 1536937 T3686 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002/conf/ [junit4] 2> 1536938 T3686 oasc.CorePropertiesLocator.discover Found 1 core definitions [junit4] 2> 1536938 T3686 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3 [junit4] 2> 1536939 T3686 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done [junit4] 2> 1536939 T3686 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider [junit4] 2> 1536940 T3686 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper [junit4] 2> 1536941 T3714 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49bdcab2 name:ZooKeeperConnection Watcher:127.0.0.1:33228/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 1536942 T3686 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper [junit4] 2> 1536942 T3686 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider [junit4] 2> 1536944 T3686 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... [junit4] 2> 1536946 T3686 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false [junit4] 2> 1536946 T3686 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2 [junit4] 2> 1536946 T3686 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider [junit4] 2> 1536947 T3686 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper [junit4] 2> 1536948 T3716 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ffc27c8 name:ZooKeeperConnection Watcher:127.0.0.1:33228/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 1536948 T3686 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper [junit4] 2> 1536948 T3686 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider [junit4] 2> 1536950 T3717 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged [junit4] 2> 1536952 T3709 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: collection1 [junit4] 2> 1536952 T3709 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2] [junit4] 2> 1536952 T3709 oasc.Overseer$ClusterStateUpdater.createCollection state version collection1 2 [junit4] 2> 1536954 T3709 oasc.Overseer$ClusterStateUpdater.updateZkStates going to create_collection /collections/collection1/state.json [junit4] 2> 1536955 T3717 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1) [junit4] 2> 1536955 T3718 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1) [junit4] 2> 1537912 T3686 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 [junit4] 2> 1537913 T3686 oejs.Server.doStart jetty-8.1.10.v20130312 [junit4] 2> 1537917 T3686 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33393 [junit4] 2> 1537917 T3686 oass.SolrDispatchFilter.init SolrDispatchFilter.init() [junit4] 2> 1537918 T3686 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx) [junit4] 2> 1537918 T3686 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-003 [junit4] 2> 1537918 T3686 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-003/' [junit4] 2> 1537953 T3686 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-003/solr.xml [junit4] 2> 1537999 T3686 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-003 [junit4] 2> 1538000 T3686 oasc.CoreContainer.<init> New CoreContainer 1034917504 [junit4] 2> 1538000 T3686 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-003/] [junit4] 2> 1538001 T3686 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000 [junit4] 2> 1538002 T3686 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: [junit4] 2> 1538002 T3686 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000 [junit4] 2> 1538002 T3686 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20 [junit4] 2> 1538002 T3686 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0 [junit4] 2> 1538003 T3686 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647 [junit4] 2> 1538003 T3686 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5 [junit4] 2> 1538003 T3686 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1 [junit4] 2> 1538003 T3686 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false [junit4] 2> 1538004 T3686 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false [junit4] 2> 1538004 T3686 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory [junit4] 2> 1538004 T3686 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)] [junit4] 2> 1538005 T3686 oasc.CoreContainer.load Host Name: 127.0.0.1 [junit4] 2> 1538005 T3686 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33228/solr [junit4] 2> 1538005 T3686 oasc.ZkController.checkChrootPath zkHost includes chroot [junit4] 2> 1538005 T3686 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider [junit4] 2> 1538006 T3686 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper [junit4] 2> 1538008 T3729 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@52c1cd23 name:ZooKeeperConnection Watcher:127.0.0.1:33228 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 1538009 T3686 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper [junit4] 2> 1538009 T3686 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider [junit4] 2> 1538012 T3686 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper [junit4] 2> 1538013 T3731 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2ae44fd2 name:ZooKeeperConnection Watcher:127.0.0.1:33228/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 1538013 T3686 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper [junit4] 2> 1538018 T3686 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... [junit4] 2> 1539022 T3686 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33393_d%2Fun [junit4] 2> 1539023 T3686 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33393_d%2Fun [junit4] 2> 1539026 T3686 oasc.Overseer.close Overseer (id=null) closing [junit4] 2> 1539030 T3686 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-003 [junit4] 2> 1539034 T3686 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-003/conf/ [junit4] 2> 1539987 T3686 oasc.CorePropertiesLocator.discover Found 1 core definitions [junit4] 2> 1539988 T3686 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3 [junit4] 2> 1539989 T3686 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done [junit4] 2> 1540857 T3686 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 [junit4] 2> 1540858 T3686 oejs.Server.doStart jetty-8.1.10.v20130312 [junit4] 2> 1540861 T3686 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33442 [junit4] 2> 1540861 T3686 oass.SolrDispatchFilter.init SolrDispatchFilter.init() [junit4] 2> 1540862 T3686 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx) [junit4] 2> 1540862 T3686 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004 [junit4] 2> 1540862 T3686 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004/' [junit4] 2> 1540891 T3686 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004/solr.xml [junit4] 2> 1540932 T3686 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004 [junit4] 2> 1540932 T3686 oasc.CoreContainer.<init> New CoreContainer 2003352130 [junit4] 2> 1540932 T3686 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004/] [junit4] 2> 1540933 T3686 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000 [junit4] 2> 1540933 T3686 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: [junit4] 2> 1540934 T3686 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000 [junit4] 2> 1540934 T3686 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20 [junit4] 2> 1540934 T3686 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0 [junit4] 2> 1540934 T3686 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647 [junit4] 2> 1540935 T3686 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5 [junit4] 2> 1540935 T3686 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1 [junit4] 2> 1540935 T3686 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false [junit4] 2> 1540936 T3686 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false [junit4] 2> 1540936 T3686 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory [junit4] 2> 1540936 T3686 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)] [junit4] 2> 1540936 T3686 oasc.CoreContainer.load Host Name: 127.0.0.1 [junit4] 2> 1540937 T3686 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33228/solr [junit4] 2> 1540937 T3686 oasc.ZkController.checkChrootPath zkHost includes chroot [junit4] 2> 1540937 T3686 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider [junit4] 2> 1540938 T3686 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper [junit4] 2> 1540939 T3744 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c9c804c name:ZooKeeperConnection Watcher:127.0.0.1:33228 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 1540940 T3686 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper [junit4] 2> 1540941 T3686 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider [junit4] 2> 1540942 T3686 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper [junit4] 2> 1540944 T3746 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21c3b793 name:ZooKeeperConnection Watcher:127.0.0.1:33228/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 1540944 T3686 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper [junit4] 2> 1540948 T3686 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... [junit4] 2> 1541952 T3686 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33442_d%2Fun [junit4] 2> 1541953 T3686 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33442_d%2Fun [junit4] 2> 1541957 T3686 oasc.Overseer.close Overseer (id=null) closing [junit4] 2> 1541961 T3686 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004 [junit4] 2> 1541965 T3686 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004/conf/ [junit4] 2> 1543379 T3686 oasc.CorePropertiesLocator.discover Found 1 core definitions [junit4] 2> 1543382 T3686 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3 [junit4] 2> 1543382 T3686 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done [junit4] 2> 1543384 T3686 oasc.AbstractFullDistribZkTestBase.createCollection Creating collection with stateFormat=2: customcollreplicadeletion [junit4] 2> 1543384 T3686 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider [junit4] 2> 1543385 T3686 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper [junit4] 2> 1543386 T3750 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c10d77f name:ZooKeeperConnection Watcher:127.0.0.1:33228/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 1543386 T3686 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper [junit4] 2> 1543387 T3686 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider [junit4] 2> 1543388 T3686 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... [junit4] 2> 1543392 T3699 oasha.CollectionsHandler.handleCreateAction Creating Collection : stateFormat=2&router.name=implicit&maxShardsPerNode=5&version=2&wt=javabin&name=customcollreplicadeletion&shards=a,b&numShards=1&replicationFactor=1&action=CREATE [junit4] 2> 1543393 T3717 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged [junit4] 2> 1543394 T3710 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{ [junit4] 2> "operation":"create", [junit4] 2> "fromApi":"true", [junit4] 2> "name":"customcollreplicadeletion", [junit4] 2> "replicationFactor":"1", [junit4] 2> "numShards":"1", [junit4] 2> "maxShardsPerNode":"5", [junit4] 2> "shards":"a,b", [junit4] 2> "stateFormat":"2", [junit4] 2> "router.name":"implicit"} [junit4] 2> 1543394 T3751 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : create , { [junit4] 2> "operation":"create", [junit4] 2> "fromApi":"true", [junit4] 2> "name":"customcollreplicadeletion", [junit4] 2> "replicationFactor":"1", [junit4] 2> "numShards":"1", [junit4] 2> "maxShardsPerNode":"5", [junit4] 2> "shards":"a,b", [junit4] 2> "stateFormat":"2", [junit4] 2> "router.name":"implicit"} [junit4] 2> 1543395 T3751 oasc.OverseerCollectionProcessor.createConfNode Only one config set found in zk - using it:conf1 [junit4] 2> 1543395 T3751 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/customcollreplicadeletion [junit4] 2> 1543396 T3751 oascc.SolrZkClient.makePath makePath: /collections/customcollreplicadeletion [junit4] 2> 1543398 T3717 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged [junit4] 2> 1543400 T3709 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: customcollreplicadeletion [junit4] 2> 1543400 T3709 oasc.Overseer$ClusterStateUpdater.createCollection Create collection customcollreplicadeletion with shards [a, b] [junit4] 2> 1543400 T3709 oasc.Overseer$ClusterStateUpdater.createCollection state version customcollreplicadeletion 2 [junit4] 2> 1543402 T3709 oasc.Overseer$ClusterStateUpdater.updateZkStates going to create_collection /collections/customcollreplicadeletion/state.json [junit4] 2> 1543403 T3717 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3) [junit4] 2> 1543403 T3718 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3) [junit4] 2> 1543403 T3732 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3) [junit4] 2> 1543403 T3747 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3) [junit4] 2> 1543403 T3752 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3) [junit4] 2> 1543498 T3751 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection customcollreplicadeletion, shardNames [a, b] , replicationFactor : 1 [junit4] 2> 1543499 T3751 oasc.OverseerCollectionProcessor.createCollection Creating shard customcollreplicadeletion_a_replica1 as part of slice a of collection customcollreplicadeletion on 127.0.0.1:33233_d%2Fun [junit4] 2> 1543499 T3751 oasc.OverseerCollectionProcessor.createCollection Creating shard customcollreplicadeletion_b_replica1 as part of slice b of collection customcollreplicadeletion on 127.0.0.1:33442_d%2Fun [junit4] 2> 1543502 T3700 oasha.CoreAdminHandler.handleCreateAction core create command collection=customcollreplicadeletion&version=2&wt=javabin&collection.configName=conf1&numShards=2&name=customcollreplicadeletion_a_replica1&action=CREATE&qt=/admin/cores&shard=a [junit4] 2> 1543502 T3736 oasha.CoreAdminHandler.handleCreateAction core create command collection=customcollreplicadeletion&version=2&wt=javabin&collection.configName=conf1&numShards=2&name=customcollreplicadeletion_b_replica1&action=CREATE&qt=/admin/cores&shard=b [junit4] 2> 1543505 T3700 oasc.ZkController.publish publishing core=customcollreplicadeletion_a_replica1 state=down collection=customcollreplicadeletion [junit4] 2> 1543506 T3736 oasc.ZkController.publish publishing core=customcollreplicadeletion_b_replica1 state=down collection=customcollreplicadeletion [junit4] 2> 1543506 T3717 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged [junit4] 2> 1543507 T3700 oasc.ZkController.preRegister Registering watch for external collection customcollreplicadeletion [junit4] 2> 1543507 T3700 oascc.ZkStateReader.addZkWatch addZkWatch customcollreplicadeletion [junit4] 2> 1543507 T3700 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/customcollreplicadeletion/state.json from ZooKeeper... [junit4] 2> 1543507 T3736 oasc.ZkController.preRegister Registering watch for external collection customcollreplicadeletion [junit4] 2> 1543508 T3736 oascc.ZkStateReader.addZkWatch addZkWatch customcollreplicadeletion [junit4] 2> 1543508 T3700 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 0 [junit4] 2> 1543508 T3736 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/customcollreplicadeletion/state.json from ZooKeeper... [junit4] 2> 1543508 T3700 oasc.ZkController.waitForCoreNodeName look for our core node name [junit4] 2> 1543509 T3736 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 0 [junit4] 2> 1543510 T3736 oasc.ZkController.waitForCoreNodeName look for our core node name [junit4] 2> 1543510 T3709 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 0 [junit4] 2> 1543510 T3709 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={ [junit4] 2> "roles":null, [junit4] 2> "collection":"customcollreplicadeletion", [junit4] 2> "node_name":"127.0.0.1:33233_d%2Fun", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:33233/d/un";, [junit4] 2> "numShards":"2", [junit4] 2> "operation":"state", [junit4] 2> "shard":"a", [junit4] 2> "core":"customcollreplicadeletion_a_replica1"} [junit4] 2> 1543512 T3709 oasc.Overseer$ClusterStateUpdater.updateZkStates going to update_collection /collections/customcollreplicadeletion/state.json [junit4] 2> 1543512 T3717 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1543512 T3747 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1543513 T3747 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 1 [junit4] 2> 1543514 T3717 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 1 [junit4] 2> 1543516 T3709 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 1 [junit4] 2> 1543516 T3709 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={ [junit4] 2> "roles":null, [junit4] 2> "collection":"customcollreplicadeletion", [junit4] 2> "node_name":"127.0.0.1:33442_d%2Fun", [junit4] 2> "state":"down", [junit4] 2> "base_url":"http://127.0.0.1:33442/d/un";, [junit4] 2> "numShards":"2", [junit4] 2> "operation":"state", [junit4] 2> "shard":"b", [junit4] 2> "core":"customcollreplicadeletion_b_replica1"} [junit4] 2> 1543517 T3717 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged [junit4] 2> 1543517 T3709 oasc.Overseer$ClusterStateUpdater.updateZkStates going to update_collection /collections/customcollreplicadeletion/state.json [junit4] 2> 1543518 T3717 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1543518 T3747 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1543519 T3747 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 2 [junit4] 2> 1543519 T3717 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 2 [junit4] 2> 1544509 T3700 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for customcollreplicadeletion_a_replica1 [junit4] 2> 1544510 T3700 oasc.ZkController.createCollectionZkNode Check for collection zkNode:customcollreplicadeletion [junit4] 2> 1544510 T3700 oasc.ZkController.createCollectionZkNode Collection zkNode exists [junit4] 2> 1544510 T3736 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for customcollreplicadeletion_b_replica1 [junit4] 2> 1544511 T3700 oascc.ZkStateReader.readConfigName Load collection config from:/collections/customcollreplicadeletion [junit4] 2> 1544511 T3736 oasc.ZkController.createCollectionZkNode Check for collection zkNode:customcollreplicadeletion [junit4] 2> 1544512 T3700 oascc.ZkStateReader.readConfigName path=/collections/customcollreplicadeletion configName=conf1 specified config exists in ZooKeeper [junit4] 2> 1544512 T3736 oasc.ZkController.createCollectionZkNode Collection zkNode exists [junit4] 2> 1544512 T3700 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002/customcollreplicadeletion_a_replica1/' [junit4] 2> 1544513 T3736 oascc.ZkStateReader.readConfigName Load collection config from:/collections/customcollreplicadeletion [junit4] 2> 1544514 T3736 oascc.ZkStateReader.readConfigName path=/collections/customcollreplicadeletion configName=conf1 specified config exists in ZooKeeper [junit4] 2> 1544514 T3736 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004/customcollreplicadeletion_b_replica1/' [junit4] 2> 1544650 T3700 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0 [junit4] 2> 1544651 T3736 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0 [junit4] 2> 1544723 T3700 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml [junit4] 2> 1544725 T3736 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml [junit4] 2> 1544725 T3700 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml [junit4] 2> 1544726 T3736 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml [junit4] 2> 1544736 T3700 oass.IndexSchema.readSchema [customcollreplicadeletion_a_replica1] Schema name=test [junit4] 2> 1544736 T3736 oass.IndexSchema.readSchema [customcollreplicadeletion_b_replica1] Schema name=test [junit4] 2> 1545150 T3736 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440. [junit4] 2> 1545156 T3700 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440. [junit4] 2> 1545160 T3736 oass.IndexSchema.readSchema default search field in schema is text [junit4] 2> 1545163 T3700 oass.IndexSchema.readSchema default search field in schema is text [junit4] 2> 1545163 T3736 oass.IndexSchema.readSchema unique key field: id [junit4] 2> 1545166 T3700 oass.IndexSchema.readSchema unique key field: id [junit4] 2> 1545172 T3736 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml [junit4] 2> 1545175 T3700 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml [junit4] 2> 1545177 T3736 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml [junit4] 2> 1545180 T3700 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml [junit4] 2> 1545181 T3736 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json [junit4] 2> 1545182 T3736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE [junit4] 2> 1545184 T3736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING [junit4] 2> 1545184 T3736 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json [junit4] 2> 1545185 T3736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE [junit4] 2> 1545185 T3736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING [junit4] 2> 1545185 T3700 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json [junit4] 2> 1545186 T3736 oasc.CoreContainer.create Creating SolrCore 'customcollreplicadeletion_b_replica1' using configuration from collection customcollreplicadeletion [junit4] 2> 1545186 T3736 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory [junit4] 2> 1545186 T3700 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE [junit4] 2> 1545187 T3736 oasc.SolrCore.<init> [customcollreplicadeletion_b_replica1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004/customcollreplicadeletion_b_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004/customcollreplicadeletion_b_replica1/data/ [junit4] 2> 1545187 T3700 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING [junit4] 2> 1545187 T3736 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@207a010b [junit4] 2> 1545188 T3700 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json [junit4] 2> 1545189 T3736 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004/customcollreplicadeletion_b_replica1/data [junit4] 2> 1545189 T3700 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE [junit4] 2> 1545190 T3700 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING [junit4] 2> 1545189 T3736 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004/customcollreplicadeletion_b_replica1/data/index/ [junit4] 2> 1545190 T3700 oasc.CoreContainer.create Creating SolrCore 'customcollreplicadeletion_a_replica1' using configuration from collection customcollreplicadeletion [junit4] 2> 1545190 T3736 oasc.SolrCore.initIndex WARN [customcollreplicadeletion_b_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004/customcollreplicadeletion_b_replica1/data/index' doesn't exist. Creating new index... [junit4] 2> 1545191 T3700 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory [junit4] 2> 1545191 T3736 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-004/customcollreplicadeletion_b_replica1/data/index [junit4] 2> 1545192 T3700 oasc.SolrCore.<init> [customcollreplicadeletion_a_replica1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002/customcollreplicadeletion_a_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002/customcollreplicadeletion_a_replica1/data/ [junit4] 2> 1545192 T3736 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=36.248046875, floorSegmentMB=0.3173828125, forceMergeDeletesPctAllowed=4.066336795253838, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0 [junit4] 2> 1545192 T3700 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@207a010b [junit4] 2> 1545193 T3736 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1 [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-44bb62df)),segFN=segments_1,generation=1} [junit4] 2> 1545193 T3736 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1 [junit4] 2> 1545194 T3700 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002/customcollreplicadeletion_a_replica1/data [junit4] 2> 1545194 T3700 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002/customcollreplicadeletion_a_replica1/data/index/ [junit4] 2> 1545195 T3700 oasc.SolrCore.initIndex WARN [customcollreplicadeletion_a_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002/customcollreplicadeletion_a_replica1/data/index' doesn't exist. Creating new index... [junit4] 2> 1545195 T3700 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001/tempDir-002/customcollreplicadeletion_a_replica1/data/index [junit4] 2> 1545196 T3700 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=36.248046875, floorSegmentMB=0.3173828125, forceMergeDeletesPctAllowed=4.066336795253838, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0 [junit4] 2> 1545197 T3700 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1 [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-5289d34c)),segFN=segments_1,generation=1} [junit4] 2> 1545197 T3700 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1 [junit4] 2> 1545198 T3736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib" [junit4] 2> 1545198 T3736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe" [junit4] 2> 1545198 T3736 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe" [junit4] 2> 1545199 T3736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig" [junit4] 2> 1545199 T3736 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig" [junit4] 2> 1545200 T3736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit" [junit4] 2> 1545200 T3736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit" [junit4] 2> 1545201 T3736 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit" [junit4] 2> 1545201 T3736 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default [junit4] 2> 1545201 T3700 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib" [junit4] 2> 1545202 T3736 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler [junit4] 2> 1545202 T3700 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe" [junit4] 2> 1545202 T3736 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler [junit4] 2> 1545202 T3700 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe" [junit4] 2> 1545203 T3736 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler [junit4] 2> 1545203 T3700 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig" [junit4] 2> 1545203 T3736 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler [junit4] 2> 1545204 T3700 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig" [junit4] 2> 1545204 T3736 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler [junit4] 2> 1545204 T3736 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler [junit4] 2> 1545204 T3700 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit" [junit4] 2> 1545205 T3736 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler [junit4] 2> 1545205 T3700 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit" [junit4] 2> 1545206 T3736 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler [junit4] 2> 1545206 T3700 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit" [junit4] 2> 1545206 T3736 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers [junit4] 2> 1545206 T3700 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default [junit4] 2> 1545207 T3736 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler [junit4] 2> 1545207 T3700 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler [junit4] 2> 1545207 T3736 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler [junit4] 2> 1545208 T3700 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler [junit4] 2> 1545208 T3700 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler [junit4] 2> 1545209 T3700 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler [junit4] 2> 1545210 T3700 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler [junit4] 2> 1545210 T3700 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler [junit4] 2> 1545210 T3700 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler [junit4] 2> 1545211 T3700 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler [junit4] 2> 1545211 T3700 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers [junit4] 2> 1545212 T3700 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler [junit4] 2> 1545212 T3700 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler [junit4] 2> 1545234 T3700 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60 [junit4] 2> 1545234 T3736 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60 [junit4] 2> 1545236 T3700 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60 [junit4] 2> 1545236 T3736 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60 [junit4] 2> 1545238 T3700 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60 [junit4] 2> 1545239 T3736 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60 [junit4] 2> 1545241 T3700 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60 [junit4] 2> 1545241 T3736 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60 [junit4] 2> 1545251 T3700 oasu.CommitTracker.<init> Hard AutoCommit: disabled [junit4] 2> 1545251 T3736 oasu.CommitTracker.<init> Hard AutoCommit: disabled [junit4] 2> 1545251 T3700 oasu.CommitTracker.<init> Soft AutoCommit: disabled [junit4] 2> 1545251 T3736 oasu.CommitTracker.<init> Soft AutoCommit: disabled [junit4] 2> 1545252 T3700 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=95.412109375, floorSegmentMB=1.6845703125, forceMergeDeletesPctAllowed=19.708774968088363, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0 [junit4] 2> 1545252 T3736 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=95.412109375, floorSegmentMB=1.6845703125, forceMergeDeletesPctAllowed=19.708774968088363, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0 [junit4] 2> 1545253 T3700 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1 [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-5289d34c)),segFN=segments_1,generation=1} [junit4] 2> 1545253 T3736 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1 [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-44bb62df)),segFN=segments_1,generation=1} [junit4] 2> 1545253 T3700 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1 [junit4] 2> 1545254 T3736 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1 [junit4] 2> 1545254 T3700 oass.SolrIndexSearcher.<init> Opening Searcher@1804ed50[customcollreplicadeletion_a_replica1] main [junit4] 2> 1545254 T3736 oass.SolrIndexSearcher.<init> Opening Searcher@1e705946[customcollreplicadeletion_b_replica1] main [junit4] 2> 1545255 T3700 oascc.ZkStateReader.readConfigName Load collection config from:/collections/customcollreplicadeletion [junit4] 2> 1545255 T3736 oascc.ZkStateReader.readConfigName Load collection config from:/collections/customcollreplicadeletion [junit4] 2> 1545256 T3700 oascc.ZkStateReader.readConfigName path=/collections/customcollreplicadeletion configName=conf1 specified config exists in ZooKeeper [junit4] 2> 1545256 T3700 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1 [junit4] 2> 1545256 T3736 oascc.ZkStateReader.readConfigName path=/collections/customcollreplicadeletion configName=conf1 specified config exists in ZooKeeper [junit4] 2> 1545257 T3700 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1 [junit4] 2> 1545257 T3736 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1 [junit4] 2> 1545257 T3700 oasr.RestManager.init Initializing RestManager with initArgs: {} [junit4] 2> 1545258 T3736 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1 [junit4] 2> 1545258 T3700 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1 [junit4] 2> 1545258 T3736 oasr.RestManager.init Initializing RestManager with initArgs: {} [junit4] 2> 1545259 T3700 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json [junit4] 2> 1545259 T3736 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1 [junit4] 2> 1545259 T3700 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1 [junit4] 2> 1545260 T3736 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json [junit4] 2> 1545260 T3700 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json [junit4] 2> 1545260 T3736 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1 [junit4] 2> 1545261 T3736 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json [junit4] 2> 1545262 T3700 oasr.ManagedResourceStorage$ZooKeeperStorageIO$1.close Wrote 38 bytes to new znode /configs/conf1/_rest_managed.json [junit4] 2> 1545262 T3700 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1 [junit4] 2> 1545262 T3736 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1 [junit4] 2> 1545262 T3700 oasr.RestManager.init Initializing 0 registered ManagedResources [junit4] 2> 1545263 T3736 oasr.RestManager.init Initializing 0 registered ManagedResources [junit4] 2> 1545266 T3755 oasc.SolrCore.registerSearcher [customcollreplicadeletion_a_replica1] Registered new searcher Searcher@1804ed50[customcollreplicadeletion_a_replica1] main{UninvertingDirectoryReader()} [junit4] 2> 1545266 T3756 oasc.SolrCore.registerSearcher [customcollreplicadeletion_b_replica1] Registered new searcher Searcher@1e705946[customcollreplicadeletion_b_replica1] main{UninvertingDirectoryReader()} [junit4] 2> 1545268 T3700 oasc.CoreContainer.registerCore registering core: customcollreplicadeletion_a_replica1 [junit4] 2> 1545269 T3700 oasc.ZkController.register Register replica - core:customcollreplicadeletion_a_replica1 address:http://127.0.0.1:33233/d/un collection:customcollreplicadeletion shard:a [junit4] 2> 1545269 T3736 oasc.CoreContainer.registerCore registering core: customcollreplicadeletion_b_replica1 [junit4] 2> 1545269 T3700 oascc.SolrZkClient.makePath makePath: /collections/customcollreplicadeletion/leader_elect/a/election [junit4] 2> 1545269 T3736 oasc.ZkController.register Register replica - core:customcollreplicadeletion_b_replica1 address:http://127.0.0.1:33442/d/un collection:customcollreplicadeletion shard:b [junit4] 2> 1545271 T3736 oascc.SolrZkClient.makePath makePath: /collections/customcollreplicadeletion/leader_elect/b/election [junit4] 2> 1545273 T3700 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard a [junit4] 2> 1545274 T3736 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard b [junit4] 2> 1545275 T3717 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged [junit4] 2> 1545275 T3700 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue. [junit4] 2> 1545275 T3700 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync [junit4] 2> 1545276 T3736 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue. [junit4] 2> ASYNC NEW_CORE C794 name=customcollreplicadeletion_a_replica1 org.apache.solr.core.SolrCore@2a9ccde6 url=http://127.0.0.1:33233/d/un/customcollreplicadeletion_a_replica1 node=127.0.0.1:33233_d%2Fun C794_STATE=coll:customcollreplicadeletion core:customcollreplicadeletion_a_replica1 props:{node_name=127.0.0.1:33233_d%2Fun, state=down, base_url=http://127.0.0.1:33233/d/un, core=customcollreplicadeletion_a_replica1} [junit4] 2> 1545276 T3700 C794 P33233 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33233/d/un/customcollreplicadeletion_a_replica1/ [junit4] 2> 1545276 T3736 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync [junit4] 2> 1545277 T3700 C794 P33233 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me [junit4] 2> 1545277 T3709 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 2 [junit4] 2> ASYNC NEW_CORE C795 name=customcollreplicadeletion_b_replica1 org.apache.solr.core.SolrCore@65918288 url=http://127.0.0.1:33442/d/un/customcollreplicadeletion_b_replica1 node=127.0.0.1:33442_d%2Fun C795_STATE=coll:customcollreplicadeletion core:customcollreplicadeletion_b_replica1 props:{node_name=127.0.0.1:33442_d%2Fun, state=down, base_url=http://127.0.0.1:33442/d/un, core=customcollreplicadeletion_b_replica1} [junit4] 2> 1545277 T3736 C795 P33442 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33442/d/un/customcollreplicadeletion_b_replica1/ [junit4] 2> 1545277 T3700 C794 P33233 oasc.SyncStrategy.syncToMe http://127.0.0.1:33233/d/un/customcollreplicadeletion_a_replica1/ has no replicas [junit4] 2> 1545278 T3736 C795 P33442 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me [junit4] 2> 1545278 T3700 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33233/d/un/customcollreplicadeletion_a_replica1/ a [junit4] 2> 1545279 T3736 C795 P33442 oasc.SyncStrategy.syncToMe http://127.0.0.1:33442/d/un/customcollreplicadeletion_b_replica1/ has no replicas [junit4] 2> 1545279 T3709 oasc.Overseer$ClusterStateUpdater.updateZkStates going to update_collection /collections/customcollreplicadeletion/state.json [junit4] 2> 1545279 T3700 oascc.SolrZkClient.makePath makePath: /collections/customcollreplicadeletion/leaders/a [junit4] 2> 1545279 T3736 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33442/d/un/customcollreplicadeletion_b_replica1/ b [junit4] 2> 1545280 T3747 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1545280 T3717 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1545280 T3736 oascc.SolrZkClient.makePath makePath: /collections/customcollreplicadeletion/leaders/b [junit4] 2> 1545281 T3747 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 3 [junit4] 2> 1545282 T3717 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 3 [junit4] 2> 1545283 T3717 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged [junit4] 2> 1545284 T3709 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 3 [junit4] 2> 1545286 T3709 oasc.Overseer$ClusterStateUpdater.updateZkStates going to update_collection /collections/customcollreplicadeletion/state.json [junit4] 2> 1545287 T3747 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1545287 T3717 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1545287 T3747 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 4 [junit4] 2> 1545288 T3717 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 4 [junit4] 2> 1545290 T3709 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 4 [junit4] 2> 1545291 T3717 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged [junit4] 2> 1545292 T3709 oasc.Overseer$ClusterStateUpdater.updateZkStates going to update_collection /collections/customcollreplicadeletion/state.json [junit4] 2> 1545292 T3717 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1545292 T3747 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1545293 T3747 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 5 [junit4] 2> 1545293 T3717 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 5 [junit4] 2> 1545295 T3709 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 5 [junit4] 2> 1545297 T3717 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged [junit4] 2> 1545297 T3709 oasc.Overseer$ClusterStateUpdater.updateZkStates going to update_collection /collections/customcollreplicadeletion/state.json [junit4] 2> 1545298 T3747 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1545298 T3717 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1545298 T3747 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 6 [junit4] 2> 1545299 T3717 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 6 [junit4] 2> 1545334 T3736 oasc.ZkController.register We are http://127.0.0.1:33442/d/un/customcollreplicadeletion_b_replica1/ and leader is http://127.0.0.1:33442/d/un/customcollreplicadeletion_b_replica1/ [junit4] 2> 1545334 T3700 oasc.ZkController.register We are http://127.0.0.1:33233/d/un/customcollreplicadeletion_a_replica1/ and leader is http://127.0.0.1:33233/d/un/customcollreplicadeletion_a_replica1/ [junit4] 2> 1545335 T3736 oasc.ZkController.register No LogReplay needed for core=customcollreplicadeletion_b_replica1 baseURL=http://127.0.0.1:33442/d/un [junit4] 2> 1545335 T3736 oasc.ZkController.checkRecovery I am the leader, no recovery necessary [junit4] 2> 1545335 T3700 oasc.ZkController.register No LogReplay needed for core=customcollreplicadeletion_a_replica1 baseURL=http://127.0.0.1:33233/d/un [junit4] 2> 1545336 T3700 oasc.ZkController.checkRecovery I am the leader, no recovery necessary [junit4] 2> 1545336 T3736 oasc.ZkController.publish publishing core=customcollreplicadeletion_b_replica1 state=active collection=customcollreplicadeletion [junit4] 2> 1545336 T3700 oasc.ZkController.publish publishing core=customcollreplicadeletion_a_replica1 state=active collection=customcollreplicadeletion [junit4] 2> 1545337 T3717 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged [junit4] 2> 1545339 T3736 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 6 [junit4] 2> 1545340 T3700 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 6 [junit4] 2> 1545342 T3736 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={collection=customcollreplicadeletion&version=2&wt=javabin&collection.configName=conf1&numShards=2&name=customcollreplicadeletion_b_replica1&action=CREATE&qt=/admin/cores&shard=b} status=0 QTime=1840 [junit4] 2> 1545342 T3709 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 6 [junit4] 2> 1545342 T3700 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={collection=customcollreplicadeletion&version=2&wt=javabin&collection.configName=conf1&numShards=2&name=customcollreplicadeletion_a_replica1&action=CREATE&qt=/admin/cores&shard=a} status=0 QTime=1840 [junit4] 2> 1545342 T3709 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={ [junit4] 2> "roles":null, [junit4] 2> "collection":"customcollreplicadeletion", [junit4] 2> "node_name":"127.0.0.1:33442_d%2Fun", [junit4] 2> "state":"active", [junit4] 2> "base_url":"http://127.0.0.1:33442/d/un";, [junit4] 2> "numShards":"2", [junit4] 2> "operation":"state", [junit4] 2> "core_node_name":"core_node2", [junit4] 2> "shard":"b", [junit4] 2> "core":"customcollreplicadeletion_b_replica1"} [junit4] 2> 1545343 T3751 oasc.OverseerCollectionProcessor.createCollection Finished create command on all shards for collection: customcollreplicadeletion [junit4] 2> 1545344 T3709 oasc.Overseer$ClusterStateUpdater.updateZkStates going to update_collection /collections/customcollreplicadeletion/state.json [junit4] 2> 1545345 T3747 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1545345 T3717 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1545345 T3747 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 7 [junit4] 2> 1545346 T3717 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 7 [junit4] 2> 1545348 T3709 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 7 [junit4] 2> 1545349 T3709 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={ [junit4] 2> "roles":null, [junit4] 2> "collection":"customcollreplicadeletion", [junit4] 2> "node_name":"127.0.0.1:33233_d%2Fun", [junit4] 2> "state":"active", [junit4] 2> "base_url":"http://127.0.0.1:33233/d/un";, [junit4] 2> "numShards":"2", [junit4] 2> "operation":"state", [junit4] 2> "core_node_name":"core_node1", [junit4] 2> "shard":"a", [junit4] 2> "core":"customcollreplicadeletion_a_replica1"} [junit4] 2> 1545350 T3717 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged [junit4] 2> 1545350 T3709 oasc.Overseer$ClusterStateUpdater.updateZkStates going to update_collection /collections/customcollreplicadeletion/state.json [junit4] 2> 1545351 T3751 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:/overseer/collection-queue-work/qn-0000000000 complete, response:{success={null={responseHeader={status=0,QTime=1840},core=customcollreplicadeletion_b_replica1},null={responseHeader={status=0,QTime=1840},core=customcollreplicadeletion_a_replica1}}} [junit4] 2> 1545352 T3747 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1545352 T3717 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/customcollreplicadeletion/state.json, has occurred - updating... [junit4] 2> 1545352 T3747 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 8 [junit4] 2> 1545352 T3759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work/qnr-0000000000 state: SyncConnected type NodeDataChanged [junit4] 2> 1545353 T3760 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged [junit4] 2> 1545353 T3717 oascc.ZkStateReader.updateWatchedCollection Updating data for customcollreplicadeletion to ver 8 [junit4] 2> 1545355 T3699 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/collections params={stateFormat=2&router.name=implicit&maxShardsPerNode=5&version=2&wt=javabin&name=customcollreplicadeletion&shards=a,b&numShards=1&replicationFactor=1&action=CREATE} status=0 QTime=1963 [junit4] 2> 1545355 T3686 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: customcollreplicadeletion failOnTimeout:true timeout (sec):330 [junit4] 2> 1545357 T3686 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Recoveries finished - collection: customcollreplicadeletion [junit4] 2> 1545358 T3686 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider [junit4] 2> 1545359 T3686 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper [junit4] 2> 1545360 T3762 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f9085c0 name:ZooKeeperConnection Watcher:127.0.0.1:33228/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None [junit4] 2> 1545360 T3686 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper [junit4] 2> 1545360 T3686 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider [junit4] 2> 1545362 T3686 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... [junit4] 2> 1545374 T3699 oasha.CollectionsHandler.handleRemoveReplica Remove replica: collection=customcollreplicadeletion&version=2&wt=javabin&action=deletereplica&shard=a&replica=core_node1 [junit4] 2> 1545375 T3717 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged [junit4] 2> 1545376 T3710 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000002 message:{ [junit4] 2> "operation":"deletereplica", [junit4] 2> "collection":"customcollreplicadeletion", [junit4] 2> "shard":"a", [junit4] 2> "replica":"core_node1"} [junit4] 2> 1545376 T3763 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : deletereplica , { [junit4] 2> "operation":"deletereplica", [junit4] 2> "collection":"customcollreplicadeletion", [...truncated too long message...] 3228 33228 [junit4] 2> 1667654 T3687 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:33228 33228 [junit4] 2> 1667755 T3750 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c10d77f name:ZooKeeperConnection Watcher:127.0.0.1:33228/solr got event WatchedEvent state:Disconnected type:None path:null path:null type:None [junit4] 2> 1667755 T3750 oascc.ConnectionManager.process Client->ZooKeeper status change trigger but we are already closed [junit4] 2> 1667755 T3750 oaz.ClientCnxn$EventThread.processEvent ERROR Error while calling watcher java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@27de1949 rejected from java.util.concurrent.ThreadPoolExecutor@664e1c11[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 4] [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372) [junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:110) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:263) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498) [junit4] 2> [junit4] 2> 1667755 T3750 oaz.ClientCnxn$EventThread.processEvent ERROR Error while calling watcher java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@158f4336 rejected from java.util.concurrent.ThreadPoolExecutor@664e1c11[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 4] [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372) [junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:110) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:263) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498) [junit4] 2> [junit4] 2> 1667756 T3750 oaz.ClientCnxn$EventThread.processEvent ERROR Error while calling watcher java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@53127fae rejected from java.util.concurrent.ThreadPoolExecutor@664e1c11[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 4] [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821) [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372) [junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:110) [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:263) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522) [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498) [junit4] 2> [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=DeleteLastCustomShardedReplicaTest -Dtests.method=testDistribSearch -Dtests.seed=CD047FFD643DDB10 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_LB -Dtests.timezone=Africa/Tripoli -Dtests.file.encoding=ISO-8859-1 [junit4] ERROR 133s J3 | DeleteLastCustomShardedReplicaTest.testDistribSearch <<< [junit4] > Throwable #1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:33393/d/un, http://127.0.0.1:33442/d/un, http://127.0.0.1:33233/d/un] [junit4] > at __randomizedtesting.SeedInfo.seed([CD047FFD643DDB10:4CE2F1E51362BB2C]:0) [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:322) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrServer.sendRequest(CloudSolrServer.java:880) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrServer.requestWithRetryOnStaleState(CloudSolrServer.java:658) [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:601) [junit4] > at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.removeAndWaitForLastReplicaGone(DeleteLastCustomShardedReplicaTest.java:117) [junit4] > at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.doTest(DeleteLastCustomShardedReplicaTest.java:107) [junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869) [junit4] > at java.lang.Thread.run(Thread.java:745) [junit4] > Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Could not remove replica : customcollreplicadeletion/a/core_node1 [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:550) [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210) [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206) [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrServer.doRequest(LBHttpSolrServer.java:343) [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:304) [junit4] > ... 46 more [junit4] 2> 1667780 T3686 oas.SolrTestCaseJ4.deleteCore ###deleteCore [junit4] 2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-CD047FFD643DDB10-001 [junit4] 2> 133416 T3685 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 2 leaked thread(s). [junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=ar_LB, timezone=Africa/Tripoli [junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=134973728,total=272629760 [junit4] 2> NOTE: All tests run in this JVM: [HighlighterConfigTest, TestOmitPositions, HttpPartitionTest, TestDistribDocBasedVersion, TestSolr4Spatial2, SimpleFacetsTest, TestRawResponseWriter, RequestHandlersTest, CollectionsAPIDistributedZkTest, TestSchemaVersionResource, OpenExchangeRatesOrgProviderTest, SolrIndexConfigTest, HdfsDirectoryTest, SoftAutoCommitTest, TestQueryTypes, SpellCheckCollatorTest, TestMaxScoreQueryParser, TestClassNameShortening, TestNonDefinedSimilarityFactory, TestManagedSchemaDynamicFieldResource, TestRealTimeGet, CurrencyFieldXmlFileTest, DistributedQueryElevationComponentTest, JsonLoaderTest, EchoParamsTest, HdfsChaosMonkeySafeLeaderTest, TestLuceneMatchVersion, OverseerCollectionProcessorTest, TestTolerantSearch, TestExpandComponent, SolrCmdDistributorTest, DOMUtilTest, SpatialFilterTest, TestLMDirichletSimilarityFactory, TestNoOpRegenerator, TestSolrQueryParser, TestSchemaResource, TestMultiCoreConfBootstrap, DebugComponentTest, TestOverriddenPrefixQueryForCustomFieldType, UUIDUpdateProcessorFallbackTest, NumericFieldsTest, TestRandomMergePolicy, TestReload, SolrPluginUtilsTest, TestJmxMonitoredMap, DistanceFunctionTest, RemoteQueryErrorTest, TestDocumentBuilder, SchemaVersionSpecificBehaviorTest, AliasIntegrationTest, TestReplicationHandler, BasicDistributedZkTest, DistributedIntervalFacetingTest, TestCloudPivotFacet, TestRequestStatusCollectionAPI, TestShortCircuitedRequests, TestModifyConfFiles, DeleteLastCustomShardedReplicaTest] [junit4] Completed on J3 in 133.95s, 1 test, 1 error <<< FAILURES! [...truncated 520 lines...] BUILD FAILED /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:491: The following error occurred while executing this line: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:471: The following error occurred while executing this line: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:60: The following error occurred while executing this line: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:189: The following error occurred while executing this line: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:496: The following error occurred while executing this line: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1359: The following error occurred while executing this line: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:962: There were test failures: 429 suites, 1772 tests, 1 error, 59 ignored (24 assumptions) Total time: 51 minutes 50 seconds Build step 'Invoke Ant' marked build as failure Archiving artifacts Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2125 Archived 5 artifacts Archive block size is 32768 Received 0 blocks and 161385493 bytes Compression is 0.0% Took 29 sec Recording test results Email was triggered for: Failure Sending email for trigger: Failure
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org
