Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/55/

4 tests failed.
FAILED:  
org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.testCollectionsAPI

Error Message:
Failed while waiting for active collection Timeout waiting to see state for 
collection=awhollynewcollection_1 
:DocCollection(awhollynewcollection_1//collections/awhollynewcollection_1/state.json/12)={
   "pullReplicas":"0",   "replicationFactor":"4",   "shards":{     "shard1":{   
    "range":"80000000-bfffffff",       "state":"active",       "replicas":{     
    "core_node3":{           "core":"awhollynewcollection_1_shard1_replica_n1", 
          "base_url":"https://127.0.0.1:39398/solr";,           
"node_name":"127.0.0.1:39398_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node4":{      
     "core":"awhollynewcollection_1_shard1_replica_n2",           
"base_url":"https://127.0.0.1:37681/solr";,           
"node_name":"127.0.0.1:37681_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node7":{      
     "core":"awhollynewcollection_1_shard1_replica_n5",           
"base_url":"https://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node10":{     
      "core":"awhollynewcollection_1_shard1_replica_n6",           
"base_url":"https://127.0.0.1:39847/solr";,           
"node_name":"127.0.0.1:39847_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false",           
"leader":"true"}}},     "shard2":{       "range":"c0000000-ffffffff",       
"state":"active",       "replicas":{         "core_node12":{           
"core":"awhollynewcollection_1_shard2_replica_n8",           
"base_url":"https://127.0.0.1:39398/solr";,           
"node_name":"127.0.0.1:39398_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node15":{     
      "core":"awhollynewcollection_1_shard2_replica_n9",           
"base_url":"https://127.0.0.1:37681/solr";,           
"node_name":"127.0.0.1:37681_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false",           "leader":"true"},  
       "core_node17":{           
"core":"awhollynewcollection_1_shard2_replica_n11",           
"base_url":"https://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node21":{     
      "core":"awhollynewcollection_1_shard2_replica_n13",           
"base_url":"https://127.0.0.1:39847/solr";,           
"node_name":"127.0.0.1:39847_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard3":{       
"range":"0-3fffffff",       "state":"active",       "replicas":{         
"core_node23":{           "core":"awhollynewcollection_1_shard3_replica_n14",   
        "base_url":"https://127.0.0.1:39398/solr";,           
"node_name":"127.0.0.1:39398_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false",           "leader":"true"},  
       "core_node25":{           
"core":"awhollynewcollection_1_shard3_replica_n16",           
"base_url":"https://127.0.0.1:37681/solr";,           
"node_name":"127.0.0.1:37681_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node27":{     
      "core":"awhollynewcollection_1_shard3_replica_n18",           
"base_url":"https://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node28":{     
      "core":"awhollynewcollection_1_shard3_replica_n19",           
"base_url":"https://127.0.0.1:39847/solr";,           
"node_name":"127.0.0.1:39847_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"}}},     "shard4":{       
"range":"40000000-7fffffff",       "state":"active",       "replicas":{         
"core_node29":{           "core":"awhollynewcollection_1_shard4_replica_n20",   
        "base_url":"https://127.0.0.1:39398/solr";,           
"node_name":"127.0.0.1:39398_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false",           "leader":"true"},  
       "core_node30":{           
"core":"awhollynewcollection_1_shard4_replica_n22",           
"base_url":"https://127.0.0.1:37681/solr";,           
"node_name":"127.0.0.1:37681_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node31":{     
      "core":"awhollynewcollection_1_shard4_replica_n24",           
"base_url":"https://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node32":{     
      "core":"awhollynewcollection_1_shard4_replica_n26",           
"base_url":"https://127.0.0.1:39847/solr";,           
"node_name":"127.0.0.1:39847_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"}}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"5",   
"autoAddReplicas":"false",   "nrtReplicas":"4",   "tlogReplicas":"0"} Live 
Nodes: [127.0.0.1:37681_solr, 127.0.0.1:39398_solr, 127.0.0.1:39847_solr, 
127.0.0.1:40912_solr] Last available state: 
DocCollection(awhollynewcollection_1//collections/awhollynewcollection_1/state.json/12)={
   "pullReplicas":"0",   "replicationFactor":"4",   "shards":{     "shard1":{   
    "range":"80000000-bfffffff",       "state":"active",       "replicas":{     
    "core_node3":{           "core":"awhollynewcollection_1_shard1_replica_n1", 
          "base_url":"https://127.0.0.1:39398/solr";,           
"node_name":"127.0.0.1:39398_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node4":{      
     "core":"awhollynewcollection_1_shard1_replica_n2",           
"base_url":"https://127.0.0.1:37681/solr";,           
"node_name":"127.0.0.1:37681_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node7":{      
     "core":"awhollynewcollection_1_shard1_replica_n5",           
"base_url":"https://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node10":{     
      "core":"awhollynewcollection_1_shard1_replica_n6",           
"base_url":"https://127.0.0.1:39847/solr";,           
"node_name":"127.0.0.1:39847_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false",           
"leader":"true"}}},     "shard2":{       "range":"c0000000-ffffffff",       
"state":"active",       "replicas":{         "core_node12":{           
"core":"awhollynewcollection_1_shard2_replica_n8",           
"base_url":"https://127.0.0.1:39398/solr";,           
"node_name":"127.0.0.1:39398_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node15":{     
      "core":"awhollynewcollection_1_shard2_replica_n9",           
"base_url":"https://127.0.0.1:37681/solr";,           
"node_name":"127.0.0.1:37681_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false",           "leader":"true"},  
       "core_node17":{           
"core":"awhollynewcollection_1_shard2_replica_n11",           
"base_url":"https://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node21":{     
      "core":"awhollynewcollection_1_shard2_replica_n13",           
"base_url":"https://127.0.0.1:39847/solr";,           
"node_name":"127.0.0.1:39847_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard3":{       
"range":"0-3fffffff",       "state":"active",       "replicas":{         
"core_node23":{           "core":"awhollynewcollection_1_shard3_replica_n14",   
        "base_url":"https://127.0.0.1:39398/solr";,           
"node_name":"127.0.0.1:39398_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false",           "leader":"true"},  
       "core_node25":{           
"core":"awhollynewcollection_1_shard3_replica_n16",           
"base_url":"https://127.0.0.1:37681/solr";,           
"node_name":"127.0.0.1:37681_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node27":{     
      "core":"awhollynewcollection_1_shard3_replica_n18",           
"base_url":"https://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node28":{     
      "core":"awhollynewcollection_1_shard3_replica_n19",           
"base_url":"https://127.0.0.1:39847/solr";,           
"node_name":"127.0.0.1:39847_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"}}},     "shard4":{       
"range":"40000000-7fffffff",       "state":"active",       "replicas":{         
"core_node29":{           "core":"awhollynewcollection_1_shard4_replica_n20",   
        "base_url":"https://127.0.0.1:39398/solr";,           
"node_name":"127.0.0.1:39398_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false",           "leader":"true"},  
       "core_node30":{           
"core":"awhollynewcollection_1_shard4_replica_n22",           
"base_url":"https://127.0.0.1:37681/solr";,           
"node_name":"127.0.0.1:37681_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node31":{     
      "core":"awhollynewcollection_1_shard4_replica_n24",           
"base_url":"https://127.0.0.1:40912/solr";,           
"node_name":"127.0.0.1:40912_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"},         "core_node32":{     
      "core":"awhollynewcollection_1_shard4_replica_n26",           
"base_url":"https://127.0.0.1:39847/solr";,           
"node_name":"127.0.0.1:39847_solr",           "state":"active",           
"type":"NRT",           "force_set_state":"false"}}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"5",   
"autoAddReplicas":"false",   "nrtReplicas":"4",   "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=awhollynewcollection_1 
:DocCollection(awhollynewcollection_1//collections/awhollynewcollection_1/state.json/12)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{
    "shard1":{
      "range":"80000000-bfffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"awhollynewcollection_1_shard1_replica_n1",
          "base_url":"https://127.0.0.1:39398/solr";,
          "node_name":"127.0.0.1:39398_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"awhollynewcollection_1_shard1_replica_n2",
          "base_url":"https://127.0.0.1:37681/solr";,
          "node_name":"127.0.0.1:37681_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node7":{
          "core":"awhollynewcollection_1_shard1_replica_n5",
          "base_url":"https://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node10":{
          "core":"awhollynewcollection_1_shard1_replica_n6",
          "base_url":"https://127.0.0.1:39847/solr";,
          "node_name":"127.0.0.1:39847_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false",
          "leader":"true"}}},
    "shard2":{
      "range":"c0000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node12":{
          "core":"awhollynewcollection_1_shard2_replica_n8",
          "base_url":"https://127.0.0.1:39398/solr";,
          "node_name":"127.0.0.1:39398_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node15":{
          "core":"awhollynewcollection_1_shard2_replica_n9",
          "base_url":"https://127.0.0.1:37681/solr";,
          "node_name":"127.0.0.1:37681_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false",
          "leader":"true"},
        "core_node17":{
          "core":"awhollynewcollection_1_shard2_replica_n11",
          "base_url":"https://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node21":{
          "core":"awhollynewcollection_1_shard2_replica_n13",
          "base_url":"https://127.0.0.1:39847/solr";,
          "node_name":"127.0.0.1:39847_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard3":{
      "range":"0-3fffffff",
      "state":"active",
      "replicas":{
        "core_node23":{
          "core":"awhollynewcollection_1_shard3_replica_n14",
          "base_url":"https://127.0.0.1:39398/solr";,
          "node_name":"127.0.0.1:39398_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false",
          "leader":"true"},
        "core_node25":{
          "core":"awhollynewcollection_1_shard3_replica_n16",
          "base_url":"https://127.0.0.1:37681/solr";,
          "node_name":"127.0.0.1:37681_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node27":{
          "core":"awhollynewcollection_1_shard3_replica_n18",
          "base_url":"https://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node28":{
          "core":"awhollynewcollection_1_shard3_replica_n19",
          "base_url":"https://127.0.0.1:39847/solr";,
          "node_name":"127.0.0.1:39847_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard4":{
      "range":"40000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node29":{
          "core":"awhollynewcollection_1_shard4_replica_n20",
          "base_url":"https://127.0.0.1:39398/solr";,
          "node_name":"127.0.0.1:39398_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false",
          "leader":"true"},
        "core_node30":{
          "core":"awhollynewcollection_1_shard4_replica_n22",
          "base_url":"https://127.0.0.1:37681/solr";,
          "node_name":"127.0.0.1:37681_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node31":{
          "core":"awhollynewcollection_1_shard4_replica_n24",
          "base_url":"https://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node32":{
          "core":"awhollynewcollection_1_shard4_replica_n26",
          "base_url":"https://127.0.0.1:39847/solr";,
          "node_name":"127.0.0.1:39847_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"5",
  "autoAddReplicas":"false",
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:37681_solr, 127.0.0.1:39398_solr, 127.0.0.1:39847_solr, 
127.0.0.1:40912_solr]
Last available state: 
DocCollection(awhollynewcollection_1//collections/awhollynewcollection_1/state.json/12)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{
    "shard1":{
      "range":"80000000-bfffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"awhollynewcollection_1_shard1_replica_n1",
          "base_url":"https://127.0.0.1:39398/solr";,
          "node_name":"127.0.0.1:39398_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"awhollynewcollection_1_shard1_replica_n2",
          "base_url":"https://127.0.0.1:37681/solr";,
          "node_name":"127.0.0.1:37681_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node7":{
          "core":"awhollynewcollection_1_shard1_replica_n5",
          "base_url":"https://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node10":{
          "core":"awhollynewcollection_1_shard1_replica_n6",
          "base_url":"https://127.0.0.1:39847/solr";,
          "node_name":"127.0.0.1:39847_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false",
          "leader":"true"}}},
    "shard2":{
      "range":"c0000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node12":{
          "core":"awhollynewcollection_1_shard2_replica_n8",
          "base_url":"https://127.0.0.1:39398/solr";,
          "node_name":"127.0.0.1:39398_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node15":{
          "core":"awhollynewcollection_1_shard2_replica_n9",
          "base_url":"https://127.0.0.1:37681/solr";,
          "node_name":"127.0.0.1:37681_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false",
          "leader":"true"},
        "core_node17":{
          "core":"awhollynewcollection_1_shard2_replica_n11",
          "base_url":"https://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node21":{
          "core":"awhollynewcollection_1_shard2_replica_n13",
          "base_url":"https://127.0.0.1:39847/solr";,
          "node_name":"127.0.0.1:39847_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard3":{
      "range":"0-3fffffff",
      "state":"active",
      "replicas":{
        "core_node23":{
          "core":"awhollynewcollection_1_shard3_replica_n14",
          "base_url":"https://127.0.0.1:39398/solr";,
          "node_name":"127.0.0.1:39398_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false",
          "leader":"true"},
        "core_node25":{
          "core":"awhollynewcollection_1_shard3_replica_n16",
          "base_url":"https://127.0.0.1:37681/solr";,
          "node_name":"127.0.0.1:37681_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node27":{
          "core":"awhollynewcollection_1_shard3_replica_n18",
          "base_url":"https://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node28":{
          "core":"awhollynewcollection_1_shard3_replica_n19",
          "base_url":"https://127.0.0.1:39847/solr";,
          "node_name":"127.0.0.1:39847_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard4":{
      "range":"40000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node29":{
          "core":"awhollynewcollection_1_shard4_replica_n20",
          "base_url":"https://127.0.0.1:39398/solr";,
          "node_name":"127.0.0.1:39398_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false",
          "leader":"true"},
        "core_node30":{
          "core":"awhollynewcollection_1_shard4_replica_n22",
          "base_url":"https://127.0.0.1:37681/solr";,
          "node_name":"127.0.0.1:37681_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node31":{
          "core":"awhollynewcollection_1_shard4_replica_n24",
          "base_url":"https://127.0.0.1:40912/solr";,
          "node_name":"127.0.0.1:40912_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node32":{
          "core":"awhollynewcollection_1_shard4_replica_n26",
          "base_url":"https://127.0.0.1:39847/solr";,
          "node_name":"127.0.0.1:39847_solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"5",
  "autoAddReplicas":"false",
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
        at 
__randomizedtesting.SeedInfo.seed([34A99F1B2826B9A9:7CDCEBAF2E15963C]:0)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:758)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:764)
        at 
org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:435)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest.testCollectionsAPI

Error Message:
Failed while waiting for active collection Timeout waiting to see state for 
collection=awhollynewcollection_1 
:DocCollection(awhollynewcollection_1//collections/awhollynewcollection_1/state.json/6)={
   "pullReplicas":"0",   "replicationFactor":"4",   "shards":{     "shard1":{   
    "range":"80000000-bfffffff",       "state":"active",       "replicas":{     
    "core_node3":{           "core":"awhollynewcollection_1_shard1_replica_n1", 
          "base_url":"https://127.0.0.1:41222/solr";,           
"node_name":"127.0.0.1:41222_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node5":{      
     "core":"awhollynewcollection_1_shard1_replica_n2",           
"base_url":"https://127.0.0.1:44513/solr";,           
"node_name":"127.0.0.1:44513_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node6":{      
     "core":"awhollynewcollection_1_shard1_replica_n4",           
"base_url":"https://127.0.0.1:36569/solr";,           
"node_name":"127.0.0.1:36569_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node9":{      
     "core":"awhollynewcollection_1_shard1_replica_n7",           
"base_url":"https://127.0.0.1:35437/solr";,           
"node_name":"127.0.0.1:35437_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard2":{       
"range":"c0000000-ffffffff",       "state":"active",       "replicas":{         
"core_node12":{           "core":"awhollynewcollection_1_shard2_replica_n8",    
       "base_url":"https://127.0.0.1:41222/solr";,           
"node_name":"127.0.0.1:41222_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node14":{     
      "core":"awhollynewcollection_1_shard2_replica_n10",           
"base_url":"https://127.0.0.1:44513/solr";,           
"node_name":"127.0.0.1:44513_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node17":{     
      "core":"awhollynewcollection_1_shard2_replica_n11",           
"base_url":"https://127.0.0.1:36569/solr";,           
"node_name":"127.0.0.1:36569_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node19":{     
      "core":"awhollynewcollection_1_shard2_replica_n13",           
"base_url":"https://127.0.0.1:35437/solr";,           
"node_name":"127.0.0.1:35437_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard3":{       
"range":"0-3fffffff",       "state":"active",       "replicas":{         
"core_node22":{           "core":"awhollynewcollection_1_shard3_replica_n15",   
        "base_url":"https://127.0.0.1:41222/solr";,           
"node_name":"127.0.0.1:41222_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node24":{     
      "core":"awhollynewcollection_1_shard3_replica_n16",           
"base_url":"https://127.0.0.1:44513/solr";,           
"node_name":"127.0.0.1:44513_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node26":{     
      "core":"awhollynewcollection_1_shard3_replica_n18",           
"base_url":"https://127.0.0.1:36569/solr";,           
"node_name":"127.0.0.1:36569_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node28":{     
      "core":"awhollynewcollection_1_shard3_replica_n20",           
"base_url":"https://127.0.0.1:35437/solr";,           
"node_name":"127.0.0.1:35437_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard4":{       
"range":"40000000-7fffffff",       "state":"active",       "replicas":{         
"core_node29":{           "core":"awhollynewcollection_1_shard4_replica_n21",   
        "base_url":"https://127.0.0.1:41222/solr";,           
"node_name":"127.0.0.1:41222_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node30":{     
      "core":"awhollynewcollection_1_shard4_replica_n23",           
"base_url":"https://127.0.0.1:44513/solr";,           
"node_name":"127.0.0.1:44513_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node31":{     
      "core":"awhollynewcollection_1_shard4_replica_n25",           
"base_url":"https://127.0.0.1:36569/solr";,           
"node_name":"127.0.0.1:36569_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node32":{     
      "core":"awhollynewcollection_1_shard4_replica_n27",           
"base_url":"https://127.0.0.1:35437/solr";,           
"node_name":"127.0.0.1:35437_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"5",   
"autoAddReplicas":"false",   "nrtReplicas":"4",   "tlogReplicas":"0"} Live 
Nodes: [127.0.0.1:35437_solr, 127.0.0.1:36569_solr, 127.0.0.1:41222_solr, 
127.0.0.1:44513_solr] Last available state: 
DocCollection(awhollynewcollection_1//collections/awhollynewcollection_1/state.json/6)={
   "pullReplicas":"0",   "replicationFactor":"4",   "shards":{     "shard1":{   
    "range":"80000000-bfffffff",       "state":"active",       "replicas":{     
    "core_node3":{           "core":"awhollynewcollection_1_shard1_replica_n1", 
          "base_url":"https://127.0.0.1:41222/solr";,           
"node_name":"127.0.0.1:41222_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node5":{      
     "core":"awhollynewcollection_1_shard1_replica_n2",           
"base_url":"https://127.0.0.1:44513/solr";,           
"node_name":"127.0.0.1:44513_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node6":{      
     "core":"awhollynewcollection_1_shard1_replica_n4",           
"base_url":"https://127.0.0.1:36569/solr";,           
"node_name":"127.0.0.1:36569_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node9":{      
     "core":"awhollynewcollection_1_shard1_replica_n7",           
"base_url":"https://127.0.0.1:35437/solr";,           
"node_name":"127.0.0.1:35437_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard2":{       
"range":"c0000000-ffffffff",       "state":"active",       "replicas":{         
"core_node12":{           "core":"awhollynewcollection_1_shard2_replica_n8",    
       "base_url":"https://127.0.0.1:41222/solr";,           
"node_name":"127.0.0.1:41222_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node14":{     
      "core":"awhollynewcollection_1_shard2_replica_n10",           
"base_url":"https://127.0.0.1:44513/solr";,           
"node_name":"127.0.0.1:44513_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node17":{     
      "core":"awhollynewcollection_1_shard2_replica_n11",           
"base_url":"https://127.0.0.1:36569/solr";,           
"node_name":"127.0.0.1:36569_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node19":{     
      "core":"awhollynewcollection_1_shard2_replica_n13",           
"base_url":"https://127.0.0.1:35437/solr";,           
"node_name":"127.0.0.1:35437_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard3":{       
"range":"0-3fffffff",       "state":"active",       "replicas":{         
"core_node22":{           "core":"awhollynewcollection_1_shard3_replica_n15",   
        "base_url":"https://127.0.0.1:41222/solr";,           
"node_name":"127.0.0.1:41222_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node24":{     
      "core":"awhollynewcollection_1_shard3_replica_n16",           
"base_url":"https://127.0.0.1:44513/solr";,           
"node_name":"127.0.0.1:44513_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node26":{     
      "core":"awhollynewcollection_1_shard3_replica_n18",           
"base_url":"https://127.0.0.1:36569/solr";,           
"node_name":"127.0.0.1:36569_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node28":{     
      "core":"awhollynewcollection_1_shard3_replica_n20",           
"base_url":"https://127.0.0.1:35437/solr";,           
"node_name":"127.0.0.1:35437_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}},     "shard4":{       
"range":"40000000-7fffffff",       "state":"active",       "replicas":{         
"core_node29":{           "core":"awhollynewcollection_1_shard4_replica_n21",   
        "base_url":"https://127.0.0.1:41222/solr";,           
"node_name":"127.0.0.1:41222_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node30":{     
      "core":"awhollynewcollection_1_shard4_replica_n23",           
"base_url":"https://127.0.0.1:44513/solr";,           
"node_name":"127.0.0.1:44513_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node31":{     
      "core":"awhollynewcollection_1_shard4_replica_n25",           
"base_url":"https://127.0.0.1:36569/solr";,           
"node_name":"127.0.0.1:36569_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"},         "core_node32":{     
      "core":"awhollynewcollection_1_shard4_replica_n27",           
"base_url":"https://127.0.0.1:35437/solr";,           
"node_name":"127.0.0.1:35437_solr",           "state":"down",           
"type":"NRT",           "force_set_state":"false"}}}},   
"router":{"name":"compositeId"},   "maxShardsPerNode":"5",   
"autoAddReplicas":"false",   "nrtReplicas":"4",   "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=awhollynewcollection_1 
:DocCollection(awhollynewcollection_1//collections/awhollynewcollection_1/state.json/6)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{
    "shard1":{
      "range":"80000000-bfffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"awhollynewcollection_1_shard1_replica_n1",
          "base_url":"https://127.0.0.1:41222/solr";,
          "node_name":"127.0.0.1:41222_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node5":{
          "core":"awhollynewcollection_1_shard1_replica_n2",
          "base_url":"https://127.0.0.1:44513/solr";,
          "node_name":"127.0.0.1:44513_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"awhollynewcollection_1_shard1_replica_n4",
          "base_url":"https://127.0.0.1:36569/solr";,
          "node_name":"127.0.0.1:36569_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node9":{
          "core":"awhollynewcollection_1_shard1_replica_n7",
          "base_url":"https://127.0.0.1:35437/solr";,
          "node_name":"127.0.0.1:35437_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard2":{
      "range":"c0000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node12":{
          "core":"awhollynewcollection_1_shard2_replica_n8",
          "base_url":"https://127.0.0.1:41222/solr";,
          "node_name":"127.0.0.1:41222_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node14":{
          "core":"awhollynewcollection_1_shard2_replica_n10",
          "base_url":"https://127.0.0.1:44513/solr";,
          "node_name":"127.0.0.1:44513_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node17":{
          "core":"awhollynewcollection_1_shard2_replica_n11",
          "base_url":"https://127.0.0.1:36569/solr";,
          "node_name":"127.0.0.1:36569_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node19":{
          "core":"awhollynewcollection_1_shard2_replica_n13",
          "base_url":"https://127.0.0.1:35437/solr";,
          "node_name":"127.0.0.1:35437_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard3":{
      "range":"0-3fffffff",
      "state":"active",
      "replicas":{
        "core_node22":{
          "core":"awhollynewcollection_1_shard3_replica_n15",
          "base_url":"https://127.0.0.1:41222/solr";,
          "node_name":"127.0.0.1:41222_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node24":{
          "core":"awhollynewcollection_1_shard3_replica_n16",
          "base_url":"https://127.0.0.1:44513/solr";,
          "node_name":"127.0.0.1:44513_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node26":{
          "core":"awhollynewcollection_1_shard3_replica_n18",
          "base_url":"https://127.0.0.1:36569/solr";,
          "node_name":"127.0.0.1:36569_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node28":{
          "core":"awhollynewcollection_1_shard3_replica_n20",
          "base_url":"https://127.0.0.1:35437/solr";,
          "node_name":"127.0.0.1:35437_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard4":{
      "range":"40000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node29":{
          "core":"awhollynewcollection_1_shard4_replica_n21",
          "base_url":"https://127.0.0.1:41222/solr";,
          "node_name":"127.0.0.1:41222_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node30":{
          "core":"awhollynewcollection_1_shard4_replica_n23",
          "base_url":"https://127.0.0.1:44513/solr";,
          "node_name":"127.0.0.1:44513_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node31":{
          "core":"awhollynewcollection_1_shard4_replica_n25",
          "base_url":"https://127.0.0.1:36569/solr";,
          "node_name":"127.0.0.1:36569_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node32":{
          "core":"awhollynewcollection_1_shard4_replica_n27",
          "base_url":"https://127.0.0.1:35437/solr";,
          "node_name":"127.0.0.1:35437_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"5",
  "autoAddReplicas":"false",
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:35437_solr, 127.0.0.1:36569_solr, 127.0.0.1:41222_solr, 
127.0.0.1:44513_solr]
Last available state: 
DocCollection(awhollynewcollection_1//collections/awhollynewcollection_1/state.json/6)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{
    "shard1":{
      "range":"80000000-bfffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"awhollynewcollection_1_shard1_replica_n1",
          "base_url":"https://127.0.0.1:41222/solr";,
          "node_name":"127.0.0.1:41222_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node5":{
          "core":"awhollynewcollection_1_shard1_replica_n2",
          "base_url":"https://127.0.0.1:44513/solr";,
          "node_name":"127.0.0.1:44513_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"awhollynewcollection_1_shard1_replica_n4",
          "base_url":"https://127.0.0.1:36569/solr";,
          "node_name":"127.0.0.1:36569_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node9":{
          "core":"awhollynewcollection_1_shard1_replica_n7",
          "base_url":"https://127.0.0.1:35437/solr";,
          "node_name":"127.0.0.1:35437_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard2":{
      "range":"c0000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node12":{
          "core":"awhollynewcollection_1_shard2_replica_n8",
          "base_url":"https://127.0.0.1:41222/solr";,
          "node_name":"127.0.0.1:41222_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node14":{
          "core":"awhollynewcollection_1_shard2_replica_n10",
          "base_url":"https://127.0.0.1:44513/solr";,
          "node_name":"127.0.0.1:44513_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node17":{
          "core":"awhollynewcollection_1_shard2_replica_n11",
          "base_url":"https://127.0.0.1:36569/solr";,
          "node_name":"127.0.0.1:36569_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node19":{
          "core":"awhollynewcollection_1_shard2_replica_n13",
          "base_url":"https://127.0.0.1:35437/solr";,
          "node_name":"127.0.0.1:35437_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard3":{
      "range":"0-3fffffff",
      "state":"active",
      "replicas":{
        "core_node22":{
          "core":"awhollynewcollection_1_shard3_replica_n15",
          "base_url":"https://127.0.0.1:41222/solr";,
          "node_name":"127.0.0.1:41222_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node24":{
          "core":"awhollynewcollection_1_shard3_replica_n16",
          "base_url":"https://127.0.0.1:44513/solr";,
          "node_name":"127.0.0.1:44513_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node26":{
          "core":"awhollynewcollection_1_shard3_replica_n18",
          "base_url":"https://127.0.0.1:36569/solr";,
          "node_name":"127.0.0.1:36569_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node28":{
          "core":"awhollynewcollection_1_shard3_replica_n20",
          "base_url":"https://127.0.0.1:35437/solr";,
          "node_name":"127.0.0.1:35437_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard4":{
      "range":"40000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node29":{
          "core":"awhollynewcollection_1_shard4_replica_n21",
          "base_url":"https://127.0.0.1:41222/solr";,
          "node_name":"127.0.0.1:41222_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node30":{
          "core":"awhollynewcollection_1_shard4_replica_n23",
          "base_url":"https://127.0.0.1:44513/solr";,
          "node_name":"127.0.0.1:44513_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node31":{
          "core":"awhollynewcollection_1_shard4_replica_n25",
          "base_url":"https://127.0.0.1:36569/solr";,
          "node_name":"127.0.0.1:36569_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node32":{
          "core":"awhollynewcollection_1_shard4_replica_n27",
          "base_url":"https://127.0.0.1:35437/solr";,
          "node_name":"127.0.0.1:35437_solr",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"5",
  "autoAddReplicas":"false",
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
        at 
__randomizedtesting.SeedInfo.seed([34A99F1B2826B9A9:7CDCEBAF2E15963C]:0)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:758)
        at 
org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:764)
        at 
org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:435)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest.testSimple

Error Message:
Waiting for collection testSimple2 Timeout waiting to see state for 
collection=testSimple2 
:DocCollection(testSimple2//collections/testSimple2/state.json/22)={   
"pullReplicas":"0",   "replicationFactor":"2",   "shards":{     "shard1":{      
 "range":"80000000-ffffffff",       "state":"active",       "replicas":{        
 "core_node3":{           
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node3/data/", 
          "base_url":"http://127.0.0.1:41320/solr";,           
"node_name":"127.0.0.1:41320_solr",           "type":"NRT",           
"force_set_state":"false",           
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node3/data/tlog",
           "core":"testSimple2_shard1_replica_n1",           
"shared_storage":"true",           "state":"down"},         "core_node5":{      
     
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node5/data/", 
          "base_url":"http://127.0.0.1:34005/solr";,           
"node_name":"127.0.0.1:34005_solr",           "type":"NRT",           
"force_set_state":"false",           
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node5/data/tlog",
           "core":"testSimple2_shard1_replica_n2",           
"shared_storage":"true",           "state":"active",           
"leader":"true"}}},     "shard2":{       "range":"0-7fffffff",       
"state":"active",       "replicas":{         "core_node7":{           
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node7/data/", 
          "base_url":"http://127.0.0.1:41320/solr";,           
"node_name":"127.0.0.1:41320_solr",           "type":"NRT",           
"force_set_state":"false",           
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node7/data/tlog",
           "core":"testSimple2_shard2_replica_n4",           
"shared_storage":"true",           "state":"down"},         "core_node8":{      
     
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node8/data/", 
          "base_url":"http://127.0.0.1:34005/solr";,           
"node_name":"127.0.0.1:34005_solr",           "type":"NRT",           
"force_set_state":"false",           
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node8/data/tlog",
           "core":"testSimple2_shard2_replica_n6",           
"shared_storage":"true",           "state":"active",           
"leader":"true"}}}},   "router":{"name":"compositeId"},   
"maxShardsPerNode":"2",   "autoAddReplicas":"true",   "nrtReplicas":"2",   
"tlogReplicas":"0"} Live Nodes: [127.0.0.1:34005_solr, 127.0.0.1:46769_solr] 
Last available state: 
DocCollection(testSimple2//collections/testSimple2/state.json/22)={   
"pullReplicas":"0",   "replicationFactor":"2",   "shards":{     "shard1":{      
 "range":"80000000-ffffffff",       "state":"active",       "replicas":{        
 "core_node3":{           
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node3/data/", 
          "base_url":"http://127.0.0.1:41320/solr";,           
"node_name":"127.0.0.1:41320_solr",           "type":"NRT",           
"force_set_state":"false",           
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node3/data/tlog",
           "core":"testSimple2_shard1_replica_n1",           
"shared_storage":"true",           "state":"down"},         "core_node5":{      
     
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node5/data/", 
          "base_url":"http://127.0.0.1:34005/solr";,           
"node_name":"127.0.0.1:34005_solr",           "type":"NRT",           
"force_set_state":"false",           
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node5/data/tlog",
           "core":"testSimple2_shard1_replica_n2",           
"shared_storage":"true",           "state":"active",           
"leader":"true"}}},     "shard2":{       "range":"0-7fffffff",       
"state":"active",       "replicas":{         "core_node7":{           
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node7/data/", 
          "base_url":"http://127.0.0.1:41320/solr";,           
"node_name":"127.0.0.1:41320_solr",           "type":"NRT",           
"force_set_state":"false",           
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node7/data/tlog",
           "core":"testSimple2_shard2_replica_n4",           
"shared_storage":"true",           "state":"down"},         "core_node8":{      
     
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node8/data/", 
          "base_url":"http://127.0.0.1:34005/solr";,           
"node_name":"127.0.0.1:34005_solr",           "type":"NRT",           
"force_set_state":"false",           
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node8/data/tlog",
           "core":"testSimple2_shard2_replica_n6",           
"shared_storage":"true",           "state":"active",           
"leader":"true"}}}},   "router":{"name":"compositeId"},   
"maxShardsPerNode":"2",   "autoAddReplicas":"true",   "nrtReplicas":"2",   
"tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Waiting for collection testSimple2
Timeout waiting to see state for collection=testSimple2 
:DocCollection(testSimple2//collections/testSimple2/state.json/22)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{
    "shard1":{
      "range":"80000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node3/data/",
          "base_url":"http://127.0.0.1:41320/solr";,
          "node_name":"127.0.0.1:41320_solr",
          "type":"NRT",
          "force_set_state":"false",
          
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node3/data/tlog",
          "core":"testSimple2_shard1_replica_n1",
          "shared_storage":"true",
          "state":"down"},
        "core_node5":{
          
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node5/data/",
          "base_url":"http://127.0.0.1:34005/solr";,
          "node_name":"127.0.0.1:34005_solr",
          "type":"NRT",
          "force_set_state":"false",
          
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node5/data/tlog",
          "core":"testSimple2_shard1_replica_n2",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{
        "core_node7":{
          
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node7/data/",
          "base_url":"http://127.0.0.1:41320/solr";,
          "node_name":"127.0.0.1:41320_solr",
          "type":"NRT",
          "force_set_state":"false",
          
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node7/data/tlog",
          "core":"testSimple2_shard2_replica_n4",
          "shared_storage":"true",
          "state":"down"},
        "core_node8":{
          
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node8/data/",
          "base_url":"http://127.0.0.1:34005/solr";,
          "node_name":"127.0.0.1:34005_solr",
          "type":"NRT",
          "force_set_state":"false",
          
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node8/data/tlog",
          "core":"testSimple2_shard2_replica_n6",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"2",
  "autoAddReplicas":"true",
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34005_solr, 127.0.0.1:46769_solr]
Last available state: 
DocCollection(testSimple2//collections/testSimple2/state.json/22)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{
    "shard1":{
      "range":"80000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node3/data/",
          "base_url":"http://127.0.0.1:41320/solr";,
          "node_name":"127.0.0.1:41320_solr",
          "type":"NRT",
          "force_set_state":"false",
          
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node3/data/tlog",
          "core":"testSimple2_shard1_replica_n1",
          "shared_storage":"true",
          "state":"down"},
        "core_node5":{
          
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node5/data/",
          "base_url":"http://127.0.0.1:34005/solr";,
          "node_name":"127.0.0.1:34005_solr",
          "type":"NRT",
          "force_set_state":"false",
          
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node5/data/tlog",
          "core":"testSimple2_shard1_replica_n2",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{
        "core_node7":{
          
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node7/data/",
          "base_url":"http://127.0.0.1:41320/solr";,
          "node_name":"127.0.0.1:41320_solr",
          "type":"NRT",
          "force_set_state":"false",
          
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node7/data/tlog",
          "core":"testSimple2_shard2_replica_n4",
          "shared_storage":"true",
          "state":"down"},
        "core_node8":{
          
"dataDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node8/data/",
          "base_url":"http://127.0.0.1:34005/solr";,
          "node_name":"127.0.0.1:34005_solr",
          "type":"NRT",
          "force_set_state":"false",
          
"ulogDir":"hdfs://localhost:34971/solr_hdfs_home/testSimple2/core_node8/data/tlog",
          "core":"testSimple2_shard2_replica_n6",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"2",
  "autoAddReplicas":"true",
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
        at 
__randomizedtesting.SeedInfo.seed([34A99F1B2826B9A9:C1ABBE50FD56D78]:0)
        at org.junit.Assert.fail(Assert.java:88)
        at 
org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:310)
        at 
org.apache.solr.cloud.autoscaling.AutoAddReplicasIntegrationTest.testSimple(AutoAddReplicasIntegrationTest.java:169)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)


FAILED:  
org.apache.solr.update.processor.CategoryRoutedAliasUpdateProcessorTest.testSliceRouting

Error Message:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while 
waiting response from server at: 
https://127.0.0.1:45091/solr/myAlias__CRA__Heart_of_Gold_shard3_replica_n16

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: 
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while 
waiting response from server at: 
https://127.0.0.1:45091/solr/myAlias__CRA__Heart_of_Gold_shard3_replica_n16
        at 
org.apache.solr.client.solrj.impl.BaseCloudSolrClient.directUpdate(BaseCloudSolrClient.java:499)
        at 
org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:964)
        at 
org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:830)
        at 
org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:763)
        at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
        at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:177)
        at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
        at 
org.apache.solr.update.processor.CategoryRoutedAliasUpdateProcessorTest.testSliceRouting(CategoryRoutedAliasUpdateProcessorTest.java:366)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
        at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
        at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
        at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
        at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
        at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
        at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
        at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occurred 
while waiting response from server at: 
https://127.0.0.1:45091/solr/myAlias__CRA__Heart_of_Gold_shard3_replica_n16
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:660)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
        at 
org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:368)
        at 
org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:296)
        at 
org.apache.solr.client.solrj.impl.BaseCloudSolrClient.directUpdate(BaseCloudSolrClient.java:493)
        ... 46 more
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
        at sun.security.ssl.InputRecord.read(InputRecord.java:503)
        at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:975)
        at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:933)
        at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at 
org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at 
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at 
org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at 
org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at 
org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
        at 
org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at 
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at 
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at 
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
        at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at 
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
        at 
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at 
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:548)
        ... 51 more




Build Log:
[...truncated 13732 lines...]
   [junit4] Suite: 
org.apache.solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest
   [junit4]   2> 984092 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: 
test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest_34A99F1B2826B9A9-001/init-core-data-001
   [junit4]   2> 984092 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 984093 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) 
w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 984095 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: 
@org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 984168 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried 
hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 984181 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 984183 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: 
c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 984184 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 984184 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 984184 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 984185 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@168de593{static,/static,jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 984351 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.h.ContextHandler Started 
o.e.j.w.WebAppContext@25687b02{hdfs,/,file:///x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/jetty-localhost-45356-hdfs-_-any-4140541894573996506.dir/webapp/,AVAILABLE}{/hdfs}
   [junit4]   2> 984353 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.AbstractConnector Started 
ServerConnector@401ed4a7{HTTP/1.1,[http/1.1]}{localhost:45356}
   [junit4]   2> 984353 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.Server Started @984405ms
   [junit4]   2> 984454 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 984455 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: 
c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 984456 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 984456 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 984456 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 984457 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1ed29f3b{static,/static,jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 984611 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.h.ContextHandler Started 
o.e.j.w.WebAppContext@4df9e978{datanode,/,file:///x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/jetty-localhost-44271-datanode-_-any-4820557892762476689.dir/webapp/,AVAILABLE}{/datanode}
   [junit4]   2> 984612 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.AbstractConnector Started 
ServerConnector@7c11dbfb{HTTP/1.1,[http/1.1]}{localhost:44271}
   [junit4]   2> 984612 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.Server Started @984664ms
   [junit4]   2> 984651 WARN  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 984653 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: 
c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 984654 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 984654 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 984654 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 984654 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@b55a609{static,/static,jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 984725 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0x9535e623b766b49f: Processing first storage report for 
DS-cbe6907e-e808-4c5c-b2ad-d056ee5269af from datanode 
cf70b6ec-c058-4adf-83e2-23aebdb43418
   [junit4]   2> 984725 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0x9535e623b766b49f: from storage 
DS-cbe6907e-e808-4c5c-b2ad-d056ee5269af node 
DatanodeRegistration(127.0.0.1:39486, 
datanodeUuid=cf70b6ec-c058-4adf-83e2-23aebdb43418, infoPort=46719, 
infoSecurePort=0, ipcPort=42500, 
storageInfo=lv=-57;cid=testClusterID;nsid=29087641;c=1553747226890), blocks: 0, 
hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 984725 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0x9535e623b766b49f: Processing first storage report for 
DS-bafee986-27f7-4467-a3d9-b390fa2279e1 from datanode 
cf70b6ec-c058-4adf-83e2-23aebdb43418
   [junit4]   2> 984725 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0x9535e623b766b49f: from storage 
DS-bafee986-27f7-4467-a3d9-b390fa2279e1 node 
DatanodeRegistration(127.0.0.1:39486, 
datanodeUuid=cf70b6ec-c058-4adf-83e2-23aebdb43418, infoPort=46719, 
infoSecurePort=0, ipcPort=42500, 
storageInfo=lv=-57;cid=testClusterID;nsid=29087641;c=1553747226890), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 984822 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.h.ContextHandler Started 
o.e.j.w.WebAppContext@4b5ac621{datanode,/,file:///x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/jetty-localhost-43644-datanode-_-any-7294392830446133966.dir/webapp/,AVAILABLE}{/datanode}
   [junit4]   2> 984823 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.AbstractConnector Started 
ServerConnector@71d24188{HTTP/1.1,[http/1.1]}{localhost:43644}
   [junit4]   2> 984823 INFO  
(SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[34A99F1B2826B9A9]-worker) [    
] o.e.j.s.Server Started @984875ms
   [junit4]   2> 984952 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0xd9961f34a3bb9f8a: Processing first storage report for 
DS-899f6bb6-b40f-4490-9718-6672cd502fa5 from datanode 
bfd3c584-d06e-4d07-97b0-1af584f6b491
   [junit4]   2> 984952 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0xd9961f34a3bb9f8a: from storage 
DS-899f6bb6-b40f-4490-9718-6672cd502fa5 node 
DatanodeRegistration(127.0.0.1:40746, 
datanodeUuid=bfd3c584-d06e-4d07-97b0-1af584f6b491, infoPort=33314, 
infoSecurePort=0, ipcPort=43378, 
storageInfo=lv=-57;cid=testClusterID;nsid=29087641;c=1553747226890), blocks: 0, 
hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 984952 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0xd9961f34a3bb9f8a: Processing first storage report for 
DS-d56f1b0d-fe75-45fe-b80e-0428b0cf246a from datanode 
bfd3c584-d06e-4d07-97b0-1af584f6b491
   [junit4]   2> 984952 INFO  (Block report processor) [    ] BlockStateChange 
BLOCK* processReport 0xd9961f34a3bb9f8a: from storage 
DS-d56f1b0d-fe75-45fe-b80e-0428b0cf246a node 
DatanodeRegistration(127.0.0.1:40746, 
datanodeUuid=bfd3c584-d06e-4d07-97b0-1af584f6b491, infoPort=33314, 
infoSecurePort=0, ipcPort=43378, 
storageInfo=lv=-57;cid=testClusterID;nsid=29087641;c=1553747226890), blocks: 0, 
hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 985021 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testDeleteNonExistentCollection-seed#[34A99F1B2826B9A9])
 [    ] o.a.s.SolrTestCaseJ4 ###Starting testDeleteNonExistentCollection
   [junit4]   2> 985023 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testDeleteNonExistentCollection-seed#[34A99F1B2826B9A9])
 [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.HdfsCollectionsAPIDistributedZkTest_34A99F1B2826B9A9-001/tempDir-002
   [junit4]   2> 985023 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testDeleteNonExistentCollection-seed#[34A99F1B2826B9A9])
 [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 985023 INFO  (ZkTestServer Run Thread) [    ] 
o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 985023 INFO  (ZkTestServer Run Thread) [    ] 
o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 985123 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testDeleteNonExistentCollection-seed#[34A99F1B2826B9A9])
 [    ] o.a.s.c.ZkTestServer start zk server on port:40959
   [junit4]   2> 985124 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testDeleteNonExistentCollection-seed#[34A99F1B2826B9A9])
 [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:40959
   [junit4]   2> 985124 INFO  
(TEST-HdfsCollectionsAPIDistributedZkTest.testDeleteNonExistentCollection-seed#[34A99F1B2826B9A9])
 [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 40959
   [junit4]   2> 985129 INFO  (zkConnectionManagerCallback-4358-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 985133 INFO  (zkConnectionManagerCallback-4360-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 985134 INFO  (zkConnectionManagerCallback-4362-thread-1) [    
] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 985138 WARN  (jetty-launcher-4363-thread-1) [    ] 
o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 985138 INFO  (jetty-launcher-4363-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 985139 INFO  (jetty-launcher-4363-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 985139 INFO  (jetty-launcher-4363-thread-1) [    ] 
o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: 
c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 985140 WARN  (jetty-launcher-4363-thread-3) [    ] 
o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 985140 INFO  (jetty-launcher-4363-thread-3) [    ] 
o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 985140 INFO  (jetty-launcher-4363-thread-3) [    ] 
o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 985140 INFO  (jetty-launcher-4363-thread-3) [    ] 
o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: 
c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 985141 WARN  (jetty-launcher-4363-thread-2) [    ] 
o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 985141 INFO  (jetty-launcher-4363-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 985141 INFO  (jetty-launcher-4363-thread-2) [    ] 
o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 985141 INFO  (jetty-launcher-4363-thread-2) [    ] 
o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: 
c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 985141 WARN  (jetty-launcher-4363-thread-4) [    ] 
o.e.j.s.AbstractConnector Ignoring deprecated socket close linger time
   [junit4]   2> 985142 INFO  (jetty-launcher-4363-thread-4) [    ] 
o.a.s.c.s.e.JettySolrRunner Start Jetty (original configured port=0)
   [junit4]   2> 985142 INFO  (jetty-launcher-4363-thread-4) [    ] 
o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 1 ...
   [junit4]   2> 985142 INFO  (jetty-launcher-4363-thread-4) [    ] 
o.e.j.s.Server jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: 
c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_191-b12
   [junit4]   2> 985143 INFO  (jetty-launcher-4363-thread-1) [    ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 985143 INFO  (jetty-launcher-4363-thread-1) [    ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 985143 INFO  (jetty-launcher-4363-thread-1) [    ] 
o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 985144 INFO  (jetty-launcher-4363-thread-1) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@6eb2841f{/solr,null,AVAILABLE}
   [junit4]   2> 985144 WARN  (jetty-launcher-4363-thread-1) [    ] 
o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for 
SslContextFactory@11826f47[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 985145 INFO  (jetty-launcher-4363-thread-2) [    ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 985145 INFO  (jetty-launcher-4363-thread-2) [    ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 985145 INFO  (jetty-launcher-4363-thread-2) [    ] 
o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 985146 INFO  (jetty-launcher-4363-thread-2) [    ] 
o.e.j.s.h.ContextHandler Started 
o.e.j.s.ServletContextHandler@1ab04227{/solr,null,AVAILABLE}
   [junit4]   2> 985146 WARN  (jetty-launcher-4363-thread-2) [    ] 
o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for 
SslContextFactory@4345ee51[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 985146 INFO  (jetty-launcher-4363-thread-4) [    ] 
o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 985146 INFO  (jetty-launcher-4363-thread-4) [    ] 
o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 985146 INFO  (jetty-launcher-4363-thread-4) [    ] 
o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 985146 INFO  (jetty-launcher-4363-thread-1) [    ] 
o.e.j.s.AbstractConnector Started ServerConnector@48cf2593{SSL,[ssl, 
http/1.1]}{127.0.0.1:34919}
   [junit4]   2> 985146 INFO  (jetty-launcher-4363-thread-1) [    ] 
o.e.j.s.Server Started @985198ms
   [junit4]   2> 985146 INFO  (jetty-launcher-4363-thread-1) [    ] 
o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, 
hostPort=34919}
   [junit4]   2> 985146 INFO  (jetty-launcher-4363-thread-3) [    ] 
o.e.j.s.session DefaultSessionIdManager workerName=nod

[...truncated too long message...]

ops.json
   [junit4]   2>        11      /solr/clusterstate.json
   [junit4]   2>        9       /solr/collections/testMaxCardinality/state.json
   [junit4]   2>        8       
/solr/collections/myAlias__CRA__Heart_of_Gold/state.json
   [junit4]   2>        8       
/solr/collections/myAlias__CRA__Constructor/state.json
   [junit4]   2>        8       
/solr/collections/myAlias__CRA__NEW_CATEGORY_ROUTED_ALIAS_WAITING_FOR_DATA__TEMP/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2>        11      /solr/live_nodes
   [junit4]   2>        11      /solr/collections
   [junit4]   2> 
   [junit4]   2> 6976411 INFO  
(TEST-CategoryRoutedAliasUpdateProcessorTest.testMaxCardinality-seed#[34A99F1B2826B9A9])
 [    ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:41357
   [junit4]   2> 6976411 INFO  
(TEST-CategoryRoutedAliasUpdateProcessorTest.testMaxCardinality-seed#[34A99F1B2826B9A9])
 [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 41357
   [junit4]   2> 6976412 INFO  
(TEST-CategoryRoutedAliasUpdateProcessorTest.testMaxCardinality-seed#[34A99F1B2826B9A9])
 [    ] o.a.s.SolrTestCaseJ4 ###Ending testMaxCardinality
   [junit4]   2> NOTE: leaving temporary files on disk at: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J0/temp/solr.update.processor.CategoryRoutedAliasUpdateProcessorTest_34A99F1B2826B9A9-001
   [junit4]   2> Mar 28, 2019 6:06:59 AM 
com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 6 leaked 
thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene80): {}, 
docValues:{}, maxPointsInLeafNode=1498, maxMBSortInHeap=7.734142975871658, 
sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@1aec601c),
 locale=ar-AE, timezone=Africa/Harare
   [junit4]   2> NOTE: Linux 4.4.0-137-generic amd64/Oracle Corporation 
1.8.0_191 (64-bit)/cpus=4,threads=1,free=195344408,total=476577792
   [junit4]   2> NOTE: All tests run in this JVM: [ForceLeaderTest, 
TestJmxIntegration, RankQueryTest, CoreAdminOperationTest, 
DistribCursorPagingTest, ImplicitSnitchTest, TestBulkSchemaAPI, 
TestCloudPseudoReturnFields, OrderedExecutorTest, 
TriLevelCompositeIdRoutingTest, DistanceUnitsTest, TestExactStatsCache, 
CursorPagingTest, SSLMigrationTest, ByteBuffersDirectoryFactoryTest, 
ChaosMonkeySafeLeaderTest, MaxSizeAutoCommitTest, TestDownShardTolerantSearch, 
SOLR749Test, TestLegacyBM25SimilarityFactory, DeleteInactiveReplicaTest, 
TestPhraseSuggestions, TestReloadDeadlock, TestConfig, TestRangeQuery, 
TestPostingsSolrHighlighter, NoCacheHeaderTest, 
FieldMutatingUpdateProcessorTest, TestSurroundQueryParser, 
TestRandomRequestDistribution, ReplicationFactorTest, TestManagedSchemaAPI, 
TestSchemaResource, TestSolrIndexConfig, OverseerModifyCollectionTest, 
DistributedFacetPivotSmallTest, SimpleCollectionCreateDeleteTest, 
SparseHLLTest, DistributedQueryComponentCustomSortTest, 
TestRequestStatusCollectionAPI, TestSimDistribStateManager, 
TestScoreJoinQPNoScore, SimpleFacetsTest, TestWordDelimiterFilterFactory, 
SolrIndexConfigTest, TestClusterStateMutator, TestSolr4Spatial2, 
SmileWriterTest, TestCloudSearcherWarming, TestHdfsCloudBackupRestore, 
TestIndexSearcher, CollectionsAPISolrJTest, TestAnalyzedSuggestions, 
CurrencyRangeFacetCloudTest, TestSimExtremeIndexing, HdfsRecoverLeaseTest, 
SearchHandlerTest, FieldAnalysisRequestHandlerTest, CloudMLTQParserTest, 
ZkCLITest, MergeStrategyTest, ClassificationUpdateProcessorIntegrationTest, 
MBeansHandlerTest, CustomHighlightComponentTest, AddReplicaTest, 
FacetPivotSmallTest, PropertiesRequestHandlerTest, 
AutoscalingHistoryHandlerTest, TestFieldTypeCollectionResource, 
TestManagedResource, CopyFieldTest, TestConfigSetProperties, 
TestReplicationHandler, HighlighterConfigTest, TestNoOpRegenerator, 
HdfsChaosMonkeyNothingIsSafeTest, TestApiFramework, TestSolrQueryResponse, 
TestCustomDocTransformer, TestStreamBody, CustomCollectionTest, 
TestExportWriter, TestMiniSolrCloudClusterSSL, BadIndexSchemaTest, 
JvmMetricsTest, AnalysisAfterCoreReloadTest, TestFieldCacheVsDocValues, 
AlternateDirectoryTest, ShowFileRequestHandlerTest, TestFieldCacheReopen, 
SearchHandlerTest, TestDeprecatedFilters, ShardRoutingTest, 
SolrJmxReporterTest, TestTolerantUpdateProcessorCloud, TestFaceting, 
NodeMarkersRegistrationTest, AutoScalingHandlerTest, 
SchemaVersionSpecificBehaviorTest, GraphQueryTest, TestUtils, 
UninvertDocValuesMergePolicyTest, CircularListTest, SampleTest, EchoParamsTest, 
TestJettySolrRunner, TestDocTermOrds, HdfsDirectoryFactoryTest, 
AddBlockUpdateTest, TestRuleBasedAuthorizationPlugin, 
TestSerializedLuceneMatchVersion, RAMDirectoryFactoryTest, 
ParsingFieldUpdateProcessorsTest, TestDirectoryFactory, 
TestNumericRangeQuery64, TestSystemCollAutoCreate, TestIndexingPerformance, 
TestDocBasedVersionConstraints, TestInPlaceUpdatesDistrib, 
TriggerSetPropertiesIntegrationTest, TestLegacyField, TermsComponentTest, 
PingRequestHandlerTest, DateRangeFieldTest, HdfsSyncSliceTest, 
TestPerFieldSimilarityWithDefaultOverride, DistributedFacetPivotLargeTest, 
TestDynamicFieldResource, UUIDUpdateProcessorFallbackTest, 
DefaultValueUpdateProcessorTest, TestPayloadScoreQParserPlugin, RootFieldTest, 
TestCrossCoreJoin, SplitShardTest, IndexSchemaRuntimeFieldTest, 
SolrMetricManagerTest, IndexBasedSpellCheckerTest, 
CollectionsAPIAsyncDistributedZkTest, DistributedIntervalFacetingTest, 
SolrTestCaseJ4Test, TestDistributedMissingSort, TestDistributedSearch, 
TestGroupingSearch, TestHighlightDedupGrouping, TestRandomDVFaceting, 
TestRandomFaceting, TestSimpleTrackingShardHandler, TestSolrCoreProperties, 
TestTolerantSearch, PathHierarchyTokenizerFactoryTest, 
ProtectedTermFilterFactoryTest, TestLuceneIndexBackCompat, 
TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, 
TestEmbeddedSolrServerSchemaAPI, ActionThrottleTest, 
AsyncCallRequestStatusResponseTest, BasicDistributedZk2Test, 
BasicDistributedZkTest, BasicZkTest, ChaosMonkeyNothingIsSafeTest, 
CleanupOldIndexTest, CloudExitableDirectoryReaderTest, ClusterStateUpdateTest, 
CollectionPropsTest, CollectionStateFormat2Test, ConfigSetsAPITest, 
ConnectionManagerTest, CreateCollectionCleanupTest, DeleteNodeTest, 
DeleteReplicaTest, DistributedQueueTest, DocValuesNotIndexedTest, 
FullSolrCloudDistribCmdsTest, HealthCheckHandlerTest, 
HttpPartitionOnCommitTest, HttpPartitionWithTlogReplicasTest, 
LeaderFailoverAfterPartitionTest, LeaderTragicEventTest, 
LeaderVoteWaitTimeoutTest, MigrateRouteKeyTest, MissingSegmentRecoveryTest, 
MoveReplicaHDFSFailoverTest, MultiSolrCloudTestCaseTest, MultiThreadedOCPTest, 
NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, 
OverriddenZkACLAndCredentialsProvidersTest, OverseerStatusTest, 
OverseerTaskQueueTest, RecoveryAfterSoftCommitTest, ReindexCollectionTest, 
TestConfigSetsAPIZkFailure, TestCryptoKeys, TestLeaderElectionZkExpiry, 
TestRandomFlRTGCloud, AssignTest, CollectionReloadTest, 
CollectionTooManyReplicasTest, ExecutePlanActionTest, HttpTriggerListenerTest, 
IndexSizeTriggerTest, MetricTriggerIntegrationTest, MetricTriggerTest, 
NodeAddedTriggerTest, NodeLostTriggerTest, SearchRateTriggerIntegrationTest, 
TriggerEventQueueTest, TriggerIntegrationTest, CdcrReplicationHandlerTest, 
CdcrRequestHandlerTest, CdcrVersionReplicationTest, HDFSCollectionsAPITest, 
HdfsNNFailoverTest, HdfsTlogReplayBufferedWhileIndexingTest, 
HdfsUnloadDistributedZkTest, StressHdfsTest, TestDynamicURP, 
TestImplicitCoreProperties, TestMergePolicyConfig, TestReloadAndDeleteDocs, 
TestSolrDeletionPolicy1, TestSolrDeletionPolicy2, TestSolrCloudSnapshots, 
TestReqParamsAPI, TestSQLHandlerNonCloud, TestSolrConfigHandlerCloud, 
DistributedTermsComponentTest, StatsComponentTest, TestPivotHelperCode, 
EmbeddedSolrNoSerializeTest, RandomizedTaggerTest, TaggingAttributeTest, 
FastVectorHighlighterTest, CheckHdfsIndexTest, ExternalFileFieldSortTest, 
NumericFieldsTest, PreAnalyzedFieldManagedSchemaCloudTest, 
SchemaApiFailureTest, TestCloudSchemaless, UUIDFieldTest, LargeFieldTest, 
QueryParsingTest, SortSpecParsingTest, SpatialFilterTest, TestComponentsName, 
TestElisionMultitermQuery, TestFilteredDocIdSet, TestGraphTermsQParserPlugin, 
TestLFUCache, TestSolrFieldCacheBean, TestSolrJ, TestSolrQueryParser, 
TestStressRecovery, TestStressReorder, TestXmlQParser, RangeFacetCloudTest, 
TestCloudJSONFacetSKG, TestLMDirichletSimilarityFactory, 
BasicAuthStandaloneTest, JWTAuthPluginIntegrationTest, JWTAuthPluginTest, 
TestAuthorizationFramework, FileBasedSpellCheckerTest, SuggesterFSTTest, 
SuggesterWFSTTest, BufferStoreTest, HdfsDirectoryTest, TransactionLogTest, 
UpdateLogTest, VersionInfoTest, AtomicUpdateProcessorFactoryTest, 
AtomicUpdatesTest, CategoryRoutedAliasUpdateProcessorTest]
   [junit4] Completed [845/845 (4!)] on J0 in 2165.90s, 6 tests, 1 error <<< 
FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J0: stdout was not empty, see: 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/temp/junit4-J0-20190328_041042_6364203380397744544727.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/heapdumps/java_pid22527.hprof
 ...
   [junit4] Heap dump file created [693990308 bytes in 12.716 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 8953 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/build.xml:651:
 The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/build.xml:585:
 Some of the tests produced a heap dump, but did not fail. Maybe a suppressed 
OutOfMemoryError? Dumps created:
* java_pid22527.hprof

Total time: 305 minutes 23 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to