FAQ
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/655/

2 tests failed.
REGRESSION: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1
  at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
  at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
  at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
  at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:583)
  at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:205)
  at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  at java.lang.reflect.Method.invoke(Method.java:606)
  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
  at java.lang.Thread.run(Thread.java:745)


REGRESSION: org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.testDistribSearch

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:45467/gkx/d, http://127.0.0.1:45475/gkx/d, http://127.0.0.1:45472/gkx/d]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:45467/gkx/d, http://127.0.0.1:45475/gkx/d, http://127.0.0.1:45472/gkx/d]
  at __randomizedtesting.SeedInfo.seed([F6FA1D3B30CEE5AD:771C932347918591]:0)
  at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:333)
  at org.apache.solr.client.solrj.impl.CloudSolrServer.sendRequest(CloudSolrServer.java:1015)
  at org.apache.solr.client.solrj.impl.CloudSolrServer.requestWithRetryOnStaleState(CloudSolrServer.java:793)
  at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:736)
  at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.removeAndWaitForLastReplicaGone(DeleteLastCustomShardedReplicaTest.java:117)
  at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.doTest(DeleteLastCustomShardedReplicaTest.java:107)
  at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  at java.lang.reflect.Method.invoke(Method.java:606)
  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
  at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Could not remove replica : customcollreplicadeletion/a/core_node1
  at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
  at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
  at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
  at org.apache.solr.client.solrj.impl.LBHttpSolrServer.doRequest(LBHttpSolrServer.java:354)
  at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:313)
  ... 46 more




Build Log:
[...truncated 12523 lines...]
    [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
    [junit4] 2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/init-core-data-001
    [junit4] 2> 1680130 T4419 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
    [junit4] 2> 1680131 T4419 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
    [junit4] 2> 1680135 T4419 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
    [junit4] 2> 1680136 T4419 oasc.ZkTestServer.run STARTING ZK TEST SERVER
    [junit4] 1> client port:0.0.0.0/0.0.0.0:0
    [junit4] 2> 1680137 T4420 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
    [junit4] 2> 1680236 T4419 oasc.ZkTestServer.run start zk server on port:18019
    [junit4] 2> 1680237 T4419 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
    [junit4] 2> 1680238 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1680241 T4426 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@39530c47 name:ZooKeeperConnection Watcher:127.0.0.1:18019 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1680241 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1680242 T4419 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
    [junit4] 2> 1680242 T4419 oascc.SolrZkClient.makePath makePath: /solr
    [junit4] 2> 1680245 T4419 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
    [junit4] 2> 1680246 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1680247 T4428 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45e408e1 name:ZooKeeperConnection Watcher:127.0.0.1:18019/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1680247 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1680248 T4419 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
    [junit4] 2> 1680248 T4419 oascc.SolrZkClient.makePath makePath: /collections/collection1
    [junit4] 2> 1680250 T4419 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
    [junit4] 2> 1680252 T4419 oascc.SolrZkClient.makePath makePath: /collections/control_collection
    [junit4] 2> 1680253 T4419 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
    [junit4] 2> 1680255 T4419 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
    [junit4] 2> 1680256 T4419 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
    [junit4] 2> 1680258 T4419 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
    [junit4] 2> 1680259 T4419 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
    [junit4] 2> 1680361 T4419 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
    [junit4] 2> 1680361 T4419 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
    [junit4] 2> 1680363 T4419 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
    [junit4] 2> 1680364 T4419 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
    [junit4] 2> 1680365 T4419 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
    [junit4] 2> 1680366 T4419 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
    [junit4] 2> 1680367 T4419 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
    [junit4] 2> 1680368 T4419 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
    [junit4] 2> 1680369 T4419 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
    [junit4] 2> 1680370 T4419 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
    [junit4] 2> 1680372 T4419 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
    [junit4] 2> 1680372 T4419 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
    [junit4] 2> 1680374 T4419 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
    [junit4] 2> 1680374 T4419 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
    [junit4] 2> 1680376 T4419 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
    [junit4] 2> 1680376 T4419 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
    [junit4] 2> 1680378 T4419 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
    [junit4] 2> 1680378 T4419 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
    [junit4] 2> Using old style solr.xml
    [junit4] 2> 1681261 T4419 oejs.Server.doStart jetty-8.1.10.v20130312
    [junit4] 2> 1681266 T4419 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:18022
    [junit4] 2> 1681266 T4419 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
    [junit4] 2> 1681266 T4419 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
    [junit4] 2> 1681267 T4419 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-002
    [junit4] 2> 1681267 T4419 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-002/'
    [junit4] 2> 1681296 T4419 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-002/solr.xml
    [junit4] 2> 1681349 T4419 oasc.CoreContainer.<init> New CoreContainer 218078697
    [junit4] 2> 1681350 T4419 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-002/]
    [junit4] 2> 1681351 T4419 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
    [junit4] 2> 1681351 T4419 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
    [junit4] 2> 1681351 T4419 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
    [junit4] 2> 1681351 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
    [junit4] 2> 1681351 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
    [junit4] 2> 1681352 T4419 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
    [junit4] 2> 1681352 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
    [junit4] 2> 1681352 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
    [junit4] 2> 1681352 T4419 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
    [junit4] 2> 1681353 T4419 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
    [junit4] 2> 1681353 T4419 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
    [junit4] 2> 1681353 T4419 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
    [junit4] 2> 1681354 T4419 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
    [junit4] 2> 1681354 T4419 oasc.CoreContainer.load Host Name: 127.0.0.1
    [junit4] 2> 1681354 T4419 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18019/solr
    [junit4] 2> 1681354 T4419 oasc.ZkController.checkChrootPath zkHost includes chroot
    [junit4] 2> 1681355 T4419 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
    [junit4] 2> 1681355 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1681357 T4439 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61d832a9 name:ZooKeeperConnection Watcher:127.0.0.1:18019 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1681358 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1681358 T4419 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
    [junit4] 2> 1681360 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1681361 T4441 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2d8a4039 name:ZooKeeperConnection Watcher:127.0.0.1:18019/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1681362 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1681364 T4419 oascc.SolrZkClient.makePath makePath: /overseer/queue
    [junit4] 2> 1681366 T4419 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
    [junit4] 2> 1681368 T4419 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
    [junit4] 2> 1681370 T4419 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
    [junit4] 2> 1681372 T4419 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
    [junit4] 2> 1681373 T4419 oascc.SolrZkClient.makePath makePath: /live_nodes
    [junit4] 2> 1681375 T4419 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:18022_
    [junit4] 2> 1681375 T4419 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:18022_
    [junit4] 2> 1681377 T4419 oascc.SolrZkClient.makePath makePath: /overseer_elect
    [junit4] 2> 1681378 T4419 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
    [junit4] 2> 1681380 T4419 oasc.Overseer.close Overseer (id=null) closing
    [junit4] 2> 1681381 T4419 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:18022_
    [junit4] 2> 1681382 T4419 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
    [junit4] 2> 1681383 T4419 oasc.Overseer.start Overseer (id=92665024459571203-127.0.0.1:18022_-n_0000000000) starting
    [junit4] 2> 1681385 T4419 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
    [junit4] 2> 1681390 T4419 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
    [junit4] 2> 1681391 T4443 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
    [junit4] 2> 1681391 T4419 oascc.SolrZkClient.makePath makePath: /clusterstate.json
    [junit4] 2> 1681393 T4419 oascc.SolrZkClient.makePath makePath: /aliases.json
    [junit4] 2> 1681394 T4419 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
    [junit4] 2> 1681396 T4442 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
    [junit4] 2> 1681400 T4419 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2
    [junit4] 2> 1681401 T4419 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
    [junit4] 2> 1681402 T4419 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
    [junit4] 2> 1681402 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1681403 T4447 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@70e66856 name:ZooKeeperConnection Watcher:127.0.0.1:18019/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1681404 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1681404 T4419 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
    [junit4] 2> 1681405 T4419 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
    [junit4] 2> 1681408 T4419 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
    [junit4] 2> 1681408 T4419 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
    [junit4] 2> 1681409 T4419 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
    [junit4] 2> 1681409 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1681410 T4449 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6d412ec6 name:ZooKeeperConnection Watcher:127.0.0.1:18019/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1681411 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1681411 T4419 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
    [junit4] 2> 1681413 T4450 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
    [junit4] 2> 1681415 T4442 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: collection1
    [junit4] 2> 1681415 T4442 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
    [junit4] 2> 1681415 T4442 oasc.Overseer$ClusterStateUpdater.createCollection state version collection1 2
    [junit4] 2> 1681417 T4442 oasc.Overseer$ClusterStateUpdater.updateZkStates going to create_collection /collections/collection1/state.json
    [junit4] 2> 1681418 T4450 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
    [junit4] 2> 1681418 T4451 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
    [junit4] 2> 1682622 T4419 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
    [junit4] 2> 1682622 T4419 oejs.Server.doStart jetty-8.1.10.v20130312
    [junit4] 2> 1682625 T4419 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:18027
    [junit4] 2> 1682626 T4419 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
    [junit4] 2> 1682626 T4419 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
    [junit4] 2> 1682626 T4419 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-003
    [junit4] 2> 1682626 T4419 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-003/'
    [junit4] 2> 1682656 T4419 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-003/solr.xml
    [junit4] 2> 1682720 T4419 oasc.CoreContainer.<init> New CoreContainer 805463138
    [junit4] 2> 1682721 T4419 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-003/]
    [junit4] 2> 1682722 T4419 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
    [junit4] 2> 1682722 T4419 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
    [junit4] 2> 1682722 T4419 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
    [junit4] 2> 1682722 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
    [junit4] 2> 1682722 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
    [junit4] 2> 1682723 T4419 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
    [junit4] 2> 1682723 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
    [junit4] 2> 1682723 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
    [junit4] 2> 1682723 T4419 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
    [junit4] 2> 1682724 T4419 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
    [junit4] 2> 1682724 T4419 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
    [junit4] 2> 1682724 T4419 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
    [junit4] 2> 1682725 T4419 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
    [junit4] 2> 1682725 T4419 oasc.CoreContainer.load Host Name: 127.0.0.1
    [junit4] 2> 1682725 T4419 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18019/solr
    [junit4] 2> 1682725 T4419 oasc.ZkController.checkChrootPath zkHost includes chroot
    [junit4] 2> 1682726 T4419 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
    [junit4] 2> 1682727 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1682728 T4462 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6d0c768d name:ZooKeeperConnection Watcher:127.0.0.1:18019 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1682729 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1682730 T4419 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
    [junit4] 2> 1682732 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1682733 T4464 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61db2e42 name:ZooKeeperConnection Watcher:127.0.0.1:18019/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1682733 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1682738 T4419 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
    [junit4] 2> 1683742 T4419 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:18027_
    [junit4] 2> 1683743 T4419 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:18027_
    [junit4] 2> 1683746 T4419 oasc.Overseer.close Overseer (id=null) closing
    [junit4] 2> 1683751 T4419 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2
    [junit4] 2> 1683751 T4419 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
    [junit4] 2> 1684853 T4419 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
    [junit4] 2> 1684855 T4419 oejs.Server.doStart jetty-8.1.10.v20130312
    [junit4] 2> 1684858 T4419 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:18030
    [junit4] 2> 1684858 T4419 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
    [junit4] 2> 1684858 T4419 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
    [junit4] 2> 1684859 T4419 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-004
    [junit4] 2> 1684859 T4419 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-004/'
    [junit4] 2> 1684888 T4419 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-004/solr.xml
    [junit4] 2> 1684940 T4419 oasc.CoreContainer.<init> New CoreContainer 1895587558
    [junit4] 2> 1684940 T4419 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-004/]
    [junit4] 2> 1684941 T4419 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
    [junit4] 2> 1684941 T4419 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
    [junit4] 2> 1684941 T4419 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
    [junit4] 2> 1684942 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
    [junit4] 2> 1684942 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
    [junit4] 2> 1684942 T4419 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
    [junit4] 2> 1684942 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
    [junit4] 2> 1684943 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
    [junit4] 2> 1684943 T4419 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
    [junit4] 2> 1684943 T4419 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
    [junit4] 2> 1684944 T4419 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
    [junit4] 2> 1684944 T4419 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
    [junit4] 2> 1684944 T4419 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
    [junit4] 2> 1684944 T4419 oasc.CoreContainer.load Host Name: 127.0.0.1
    [junit4] 2> 1684945 T4419 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18019/solr
    [junit4] 2> 1684945 T4419 oasc.ZkController.checkChrootPath zkHost includes chroot
    [junit4] 2> 1684945 T4419 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
    [junit4] 2> 1684946 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1684947 T4477 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48ba87fd name:ZooKeeperConnection Watcher:127.0.0.1:18019 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1684948 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1684949 T4419 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
    [junit4] 2> 1684951 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1684952 T4479 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@474976f6 name:ZooKeeperConnection Watcher:127.0.0.1:18019/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1684952 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1684957 T4419 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
    [junit4] 2> 1685961 T4419 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:18030_
    [junit4] 2> 1685962 T4419 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:18030_
    [junit4] 2> 1685965 T4419 oasc.Overseer.close Overseer (id=null) closing
    [junit4] 2> 1685970 T4419 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2
    [junit4] 2> 1685970 T4419 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
    [junit4] 2> 1687170 T4419 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
    [junit4] 2> 1687171 T4419 oejs.Server.doStart jetty-8.1.10.v20130312
    [junit4] 2> 1687174 T4419 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:18033
    [junit4] 2> 1687174 T4419 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
    [junit4] 2> 1687175 T4419 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
    [junit4] 2> 1687175 T4419 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-005
    [junit4] 2> 1687175 T4419 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-005/'
    [junit4] 2> 1687204 T4419 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-005/solr.xml
    [junit4] 2> 1687256 T4419 oasc.CoreContainer.<init> New CoreContainer 231547783
    [junit4] 2> 1687257 T4419 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-005/]
    [junit4] 2> 1687257 T4419 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
    [junit4] 2> 1687258 T4419 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
    [junit4] 2> 1687258 T4419 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
    [junit4] 2> 1687258 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
    [junit4] 2> 1687258 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
    [junit4] 2> 1687259 T4419 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
    [junit4] 2> 1687259 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
    [junit4] 2> 1687259 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
    [junit4] 2> 1687259 T4419 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
    [junit4] 2> 1687259 T4419 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
    [junit4] 2> 1687260 T4419 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
    [junit4] 2> 1687260 T4419 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
    [junit4] 2> 1687260 T4419 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
    [junit4] 2> 1687261 T4419 oasc.CoreContainer.load Host Name: 127.0.0.1
    [junit4] 2> 1687261 T4419 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18019/solr
    [junit4] 2> 1687261 T4419 oasc.ZkController.checkChrootPath zkHost includes chroot
    [junit4] 2> 1687262 T4419 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
    [junit4] 2> 1687262 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1687264 T4492 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16d77b25 name:ZooKeeperConnection Watcher:127.0.0.1:18019 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1687265 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1687265 T4419 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
    [junit4] 2> 1687268 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1687268 T4494 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@68ca3777 name:ZooKeeperConnection Watcher:127.0.0.1:18019/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1687269 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1687274 T4419 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
    [junit4] 2> 1688278 T4419 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:18033_
    [junit4] 2> 1688279 T4419 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:18033_
    [junit4] 2> 1688283 T4419 oasc.Overseer.close Overseer (id=null) closing
    [junit4] 2> 1688288 T4419 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2
    [junit4] 2> 1688288 T4419 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
    [junit4] 2> 1689389 T4419 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
    [junit4] 2> 1689390 T4419 oejs.Server.doStart jetty-8.1.10.v20130312
    [junit4] 2> 1689393 T4419 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:18037
    [junit4] 2> 1689394 T4419 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
    [junit4] 2> 1689394 T4419 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
    [junit4] 2> 1689394 T4419 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-006
    [junit4] 2> 1689395 T4419 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-006/'
    [junit4] 2> 1689424 T4419 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-006/solr.xml
    [junit4] 2> 1689477 T4419 oasc.CoreContainer.<init> New CoreContainer 1472260477
    [junit4] 2> 1689477 T4419 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-006/]
    [junit4] 2> 1689478 T4419 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
    [junit4] 2> 1689478 T4419 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
    [junit4] 2> 1689478 T4419 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
    [junit4] 2> 1689479 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
    [junit4] 2> 1689479 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
    [junit4] 2> 1689479 T4419 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
    [junit4] 2> 1689479 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
    [junit4] 2> 1689480 T4419 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
    [junit4] 2> 1689480 T4419 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
    [junit4] 2> 1689480 T4419 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
    [junit4] 2> 1689481 T4419 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
    [junit4] 2> 1689481 T4419 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
    [junit4] 2> 1689481 T4419 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
    [junit4] 2> 1689481 T4419 oasc.CoreContainer.load Host Name: 127.0.0.1
    [junit4] 2> 1689482 T4419 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18019/solr
    [junit4] 2> 1689482 T4419 oasc.ZkController.checkChrootPath zkHost includes chroot
    [junit4] 2> 1689482 T4419 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
    [junit4] 2> 1689483 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1689484 T4507 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3a213a8d name:ZooKeeperConnection Watcher:127.0.0.1:18019 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1689485 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1689486 T4419 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
    [junit4] 2> 1689488 T4419 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
    [junit4] 2> 1689489 T4509 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4492315f name:ZooKeeperConnection Watcher:127.0.0.1:18019/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1689489 T4419 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
    [junit4] 2> 1689494 T4419 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
    [junit4] 2> 1690498 T4419 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:18037_
    [junit4] 2> 1690499 T4419 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:18037_
    [junit4] 2> 1690503 T4419 oasc.Overseer.close Overseer (id=null) closing
    [junit4] 2> 1690507 T4419 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2
    [junit4] 2> 1690508 T4419 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
    [junit4] 2> 1690511 T4456 oasha.CollectionsHandler.handleCreateAction Creating Collection : replicationFactor=2&collection.configName=conf1&wt=javabin&version=2&numShards=2&router.field=myOwnField&action=CREATE&name=solrj_collection
    [junit4] 2> 1690512 T4450 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
    [junit4] 2> 1690513 T4443 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
    [junit4] 2> "operation":"create",
    [junit4] 2> "fromApi":"true",
    [junit4] 2> "name":"solrj_collection",
    [junit4] 2> "replicationFactor":"2",
    [junit4] 2> "collection.configName":"conf1",
    [junit4] 2> "numShards":"2",
    [junit4] 2> "router.field":"myOwnField"}
    [junit4] 2> 1690515 T4512 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : create , {
    [junit4] 2> "operation":"create",
    [junit4] 2> "fromApi":"true",
    [junit4] 2> "name":"solrj_collection",
    [junit4] 2> "replicationFactor":"2",
    [junit4] 2> "collection.configName":"conf1",
    [junit4] 2> "numShards":"2",
    [junit4] 2> "router.field":"myOwnField"}
    [junit4] 2> 1690515 T4512 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection
    [junit4] 2> 1690516 T4512 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
    [junit4] 2> 1690518 T4450 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
    [junit4] 2> 1690520 T4442 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
    [junit4] 2> 1690521 T4442 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
    [junit4] 2> 1690521 T4442 oasc.Overseer$ClusterStateUpdater.createCollection state version solrj_collection 1
    [junit4] 2> 1690523 T4510 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1690523 T4465 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1690523 T4450 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1690523 T4495 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1690523 T4451 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1690523 T4480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1690618 T4512 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection solrj_collection, shardNames [shard1, shard2] , replicationFactor : 2
    [junit4] 2> 1690619 T4512 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:18022_
    [junit4] 2> 1690619 T4512 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:18027_
    [junit4] 2> 1690622 T4512 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:18037_
    [junit4] 2> 1690624 T4434 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection=solrj_collection&collection.configName=conf1&wt=javabin&version=2&numShards=2&action=CREATE&name=solrj_collection_shard1_replica1&shard=shard1
    [junit4] 2> 1690624 T4459 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection=solrj_collection&collection.configName=conf1&wt=javabin&version=2&numShards=2&action=CREATE&name=solrj_collection_shard1_replica2&shard=shard1
    [junit4] 2> 1690624 T4512 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:18033_
    [junit4] 2> 1690625 T4499 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection=solrj_collection&collection.configName=conf1&wt=javabin&version=2&numShards=2&action=CREATE&name=solrj_collection_shard2_replica1&shard=shard2
    [junit4] 2> 1690626 T4434 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down collection=solrj_collection
    [junit4] 2> 1690627 T4484 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection=solrj_collection&collection.configName=conf1&wt=javabin&version=2&numShards=2&action=CREATE&name=solrj_collection_shard2_replica2&shard=shard2
    [junit4] 2> 1690627 T4434 oasc.ZkController.waitForCoreNodeName look for our core node name
    [junit4] 2> 1690627 T4450 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
    [junit4] 2> 1690628 T4459 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down collection=solrj_collection
    [junit4] 2> 1690628 T4499 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down collection=solrj_collection
    [junit4] 2> 1690629 T4459 oasc.ZkController.waitForCoreNodeName look for our core node name
    [junit4] 2> 1690629 T4499 oasc.ZkController.waitForCoreNodeName look for our core node name
    [junit4] 2> 1690629 T4484 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down collection=solrj_collection
    [junit4] 2> 1690631 T4484 oasc.ZkController.waitForCoreNodeName look for our core node name
    [junit4] 2> 1690631 T4442 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
    [junit4] 2> "operation":"state",
    [junit4] 2> "collection":"solrj_collection",
    [junit4] 2> "node_name":"127.0.0.1:18022_",
    [junit4] 2> "state":"down",
    [junit4] 2> "numShards":"2",
    [junit4] 2> "core":"solrj_collection_shard1_replica1",
    [junit4] 2> "roles":null,
    [junit4] 2> "shard":"shard1",
    [junit4] 2> "base_url":"http://127.0.0.1:18022"}
    [junit4] 2> 1690633 T4442 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
    [junit4] 2> "operation":"state",
    [junit4] 2> "collection":"solrj_collection",
    [junit4] 2> "node_name":"127.0.0.1:18027_",
    [junit4] 2> "state":"down",
    [junit4] 2> "numShards":"2",
    [junit4] 2> "core":"solrj_collection_shard1_replica2",
    [junit4] 2> "roles":null,
    [junit4] 2> "shard":"shard1",
    [junit4] 2> "base_url":"http://127.0.0.1:18027"}
    [junit4] 2> 1690634 T4450 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
    [junit4] 2> 1690635 T4442 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
    [junit4] 2> "operation":"state",
    [junit4] 2> "collection":"solrj_collection",
    [junit4] 2> "node_name":"127.0.0.1:18037_",
    [junit4] 2> "state":"down",
    [junit4] 2> "numShards":"2",
    [junit4] 2> "core":"solrj_collection_shard2_replica1",
    [junit4] 2> "roles":null,
    [junit4] 2> "shard":"shard2",
    [junit4] 2> "base_url":"http://127.0.0.1:18037"}
    [junit4] 2> 1690637 T4442 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
    [junit4] 2> "operation":"state",
    [junit4] 2> "collection":"solrj_collection",
    [junit4] 2> "node_name":"127.0.0.1:18033_",
    [junit4] 2> "state":"down",
    [junit4] 2> "numShards":"2",
    [junit4] 2> "core":"solrj_collection_shard2_replica2",
    [junit4] 2> "roles":null,
    [junit4] 2> "shard":"shard2",
    [junit4] 2> "base_url":"http://127.0.0.1:18033"}
    [junit4] 2> 1690638 T4450 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
    [junit4] 2> 1690639 T4450 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1690639 T4451 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1690639 T4510 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1690639 T4495 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1690639 T4480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1690639 T4465 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1691769 T4459 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
    [junit4] 2> 1691769 T4459 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
    [junit4] 2> 1691770 T4459 oasc.ZkController.createCollectionZkNode Collection zkNode exists
    [junit4] 2> 1691770 T4459 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
    [junit4] 2> 1691771 T4459 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
    [junit4] 2> 1691771 T4459 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-003/solrj_collection_shard1_replica2/'
    [junit4] 2> 1691821 T4459 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
    [junit4] 2> 1691868 T4459 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
    [junit4] 2> 1691882 T4434 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
    [junit4] 2> 1691882 T4499 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
    [junit4] 2> 1691882 T4484 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
    [junit4] 2> 1691883 T4499 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
    [junit4] 2> 1691882 T4434 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
    [junit4] 2> 1691884 T4499 oasc.ZkController.createCollectionZkNode Collection zkNode exists
    [junit4] 2> 1691884 T4434 oasc.ZkController.createCollectionZkNode Collection zkNode exists
    [junit4] 2> 1691884 T4434 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
    [junit4] 2> 1691883 T4484 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
    [junit4] 2> 1691884 T4499 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
    [junit4] 2> 1691885 T4484 oasc.ZkController.createCollectionZkNode Collection zkNode exists
    [junit4] 2> 1691886 T4484 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
    [junit4] 2> 1691885 T4434 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
    [junit4] 2> 1691886 T4499 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
    [junit4] 2> 1691886 T4484 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
    [junit4] 2> 1691886 T4434 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-002/solrj_collection_shard1_replica1/'
    [junit4] 2> 1691887 T4484 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-005/solrj_collection_shard2_replica2/'
    [junit4] 2> 1691887 T4499 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-006/solrj_collection_shard2_replica1/'
    [junit4] 2> 1691942 T4434 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
    [junit4] 2> 1691947 T4499 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
    [junit4] 2> 1691953 T4484 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
    [junit4] 2> 1691969 T4459 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
    [junit4] 2> 1691980 T4459 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
    [junit4] 2> 1692006 T4434 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
    [junit4] 2> 1692007 T4434 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
    [junit4] 2> 1692012 T4499 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
    [junit4] 2> 1692016 T4484 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
    [junit4] 2> 1692018 T4434 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
    [junit4] 2> 1692113 T4499 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
    [junit4] 2> 1692117 T4484 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
    [junit4] 2> 1692125 T4499 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
    [junit4] 2> 1692128 T4484 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
    [junit4] 2> 1692410 T4459 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
    [junit4] 2> 1692423 T4459 oass.IndexSchema.readSchema default search field in schema is text
    [junit4] 2> 1692426 T4459 oass.IndexSchema.readSchema unique key field: id
    [junit4] 2> 1692435 T4434 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
    [junit4] 2> 1692436 T4459 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
    [junit4] 2> 1692441 T4459 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
    [junit4] 2> 1692443 T4434 oass.IndexSchema.readSchema default search field in schema is text
    [junit4] 2> 1692445 T4459 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
    [junit4] 2> 1692446 T4434 oass.IndexSchema.readSchema unique key field: id
    [junit4] 2> 1692446 T4459 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
    [junit4] 2> 1692449 T4459 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
    [junit4] 2> 1692449 T4459 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
    [junit4] 2> 1692450 T4459 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
    [junit4] 2> 1692451 T4459 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
    [junit4] 2> 1692451 T4459 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using configuration from collection solrj_collection
    [junit4] 2> 1692451 T4459 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
    [junit4] 2> 1692452 T4459 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-003/solrj_collection_shard1_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-003/solrj_collection_shard1_replica2/data/
    [junit4] 2> 1692452 T4459 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@33313e93
    [junit4] 2> 1692454 T4459 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-003/solrj_collection_shard1_replica2/data
    [junit4] 2> 1692454 T4459 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-003/solrj_collection_shard1_replica2/data/index/
    [junit4] 2> 1692455 T4459 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-003/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
    [junit4] 2> 1692455 T4459 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-003/solrj_collection_shard1_replica2/data/index
    [junit4] 2> 1692456 T4459 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@39d29be6
    [junit4] 2> 1692457 T4459 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
    [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-5d52b347)),segFN=segments_1,generation=1}
    [junit4] 2> 1692457 T4459 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
    [junit4] 2> 1692458 T4434 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
    [junit4] 2> 1692463 T4459 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
    [junit4] 2> 1692463 T4434 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
    [junit4] 2> 1692463 T4459 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
    [junit4] 2> 1692464 T4459 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
    [junit4] 2> 1692464 T4459 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
    [junit4] 2> 1692465 T4459 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
    [junit4] 2> 1692465 T4459 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
    [junit4] 2> 1692466 T4459 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
    [junit4] 2> 1692466 T4459 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
    [junit4] 2> 1692466 T4459 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
    [junit4] 2> 1692467 T4459 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1692468 T4434 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
    [junit4] 2> 1692468 T4459 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1692468 T4434 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
    [junit4] 2> 1692469 T4434 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
    [junit4] 2> 1692469 T4459 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1692469 T4434 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
    [junit4] 2> 1692470 T4459 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
    [junit4] 2> 1692470 T4434 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
    [junit4] 2> 1692471 T4434 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
    [junit4] 2> 1692471 T4459 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
    [junit4] 2> 1692471 T4434 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using configuration from collection solrj_collection
    [junit4] 2> 1692471 T4459 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
    [junit4] 2> 1692472 T4434 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
    [junit4] 2> 1692472 T4459 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
    [junit4] 2> 1692472 T4434 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-002/solrj_collection_shard1_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-002/solrj_collection_shard1_replica1/data/
    [junit4] 2> 1692473 T4459 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
    [junit4] 2> 1692473 T4434 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@33313e93
    [junit4] 2> 1692474 T4459 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
    [junit4] 2> 1692474 T4459 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
    [junit4] 2> 1692474 T4434 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-002/solrj_collection_shard1_replica1/data
    [junit4] 2> 1692475 T4459 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
    [junit4] 2> 1692475 T4434 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-002/solrj_collection_shard1_replica1/data/index/
    [junit4] 2> 1692476 T4434 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-002/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
    [junit4] 2> 1692476 T4434 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest-F6FA1D3B30CEE5AD-001/tempDir-002/solrj_collection_shard1_replica1/data/index
    [junit4] 2> 1692477 T4434 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@700a4fdc
    [junit4] 2> 1692478 T4434 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
    [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-70b698aa)),segFN=segments_1,generation=1}
    [junit4] 2> 1692478 T4434 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
    [junit4] 2> 1692484 T4434 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
    [junit4] 2> 1692484 T4434 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
    [junit4] 2> 1692485 T4434 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
    [junit4] 2> 1692485 T4434 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
    [junit4] 2> 1692485 T4434 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
    [junit4] 2> 1692486 T4434 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
    [junit4] 2> 1692486 T4434 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
    [junit4] 2> 1692487 T4434 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
    [junit4] 2> 1692487 T4434 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
    [junit4] 2> 1692488 T4434 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1692488 T4434 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1692489 T4434 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1692490 T4434 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
    [junit4] 2> 1692490 T4434 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
    [junit4] 2> 1692491 T4434 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
    [junit4] 2> 1692491 T4434 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
    [junit4] 2> 1692492 T4434 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
    [junit4] 2> 1692492 T4434 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
    [junit4] 2> 1692493 T4434 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
    [junit4] 2> 1692494 T4434 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
    [junit4] 2> 1692499 T4459 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
    [junit4] 2> 1692500 T4434 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
    [junit4] 2> 1692502 T4459 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
    [junit4] 2> 1692510 T4434 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
    [junit4] 2> 1692512 T4459 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
    [junit4] 2> 1692513 T4434 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
    [junit4] 2> 1692515 T4459 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
    [junit4] 2> 1692516 T4434 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
    [junit4] 2> 1692524 T4434 oasu.CommitTracker.<init> Hard AutoCommit: disabled
    [junit4] 2> 1692524 T4459 oasu.CommitTracker.<init> Hard AutoCommit: disabled
    [junit4] 2> 1692524 T4434 oasu.CommitTracker.<init> Soft AutoCommit: disabled
    [junit4] 2> 1692525 T4459 oasu.CommitTracker.<init> Soft AutoCommit: disabled
    [junit4] 2> 1692526 T4434 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=47.4541015625, floorSegmentMB=1.3125, forceMergeDeletesPctAllowed=14.847135228332702, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
    [junit4] 2> 1692526 T4459 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=47.4541015625, floorSegmentMB=1.3125, forceMergeDeletesPctAllowed=14.847135228332702, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
    [junit4] 2> 1692526 T4434 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
    [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-70b698aa)),segFN=segments_1,generation=1}
    [junit4] 2> 1692527 T4459 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
    [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-5d52b347)),segFN=segments_1,generation=1}
    [junit4] 2> 1692527 T4434 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
    [junit4] 2> 1692527 T4459 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
    [junit4] 2> 1692528 T4434 oass.SolrIndexSearcher.<init> Opening Searcher@6c24e1fe[solrj_collection_shard1_replica1] main
    [junit4] 2> 1692528 T4459 oass.SolrIndexSearcher.<init> Opening Searcher@279e7d4[solrj_collection_shard1_replica2] main
    [junit4] 2> 1692529 T4434 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
    [junit4] 2> 1692529 T4459 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
    [junit4] 2> 1692530 T4434 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
    [junit4] 2> 1692530 T4459 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
    [junit4] 2> 1692530 T4434 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
    [junit4] 2> 1692530 T4459 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
    [junit4] 2> 1692531 T4434 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
    [junit4] 2> 1692531 T4459 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
    [junit4] 2> 1692532 T4434 oasr.RestManager.init Initializing RestManager with initArgs: {}
    [junit4] 2> 1692532 T4459 oasr.RestManager.init Initializing RestManager with initArgs: {}
    [junit4] 2> 1692532 T4434 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
    [junit4] 2> 1692533 T4459 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
    [junit4] 2> 1692533 T4434 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
    [junit4] 2> 1692534 T4459 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
    [junit4] 2> 1692534 T4434 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
    [junit4] 2> 1692534 T4459 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
    [junit4] 2> 1692535 T4434 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json
    [junit4] 2> 1692535 T4459 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json
    [junit4] 2> 1692537 T4434 oasr.ManagedResourceStorage$ZooKeeperStorageIO$1.close Wrote 38 bytes to new znode /configs/conf1/_rest_managed.json
    [junit4] 2> 1692537 T4434 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
    [junit4] 2> 1692538 T4434 oasr.RestManager.init Initializing 0 registered ManagedResources
    [junit4] 2> 1692538 T4459 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
    [junit4] 2> 1692538 T4459 oasr.RestManager.init Initializing 0 registered ManagedResources
    [junit4] 2> 1692541 T4518 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica1] Registered new searcher Searcher@6c24e1fe[solrj_collection_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
    [junit4] 2> 1692542 T4517 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica2] Registered new searcher Searcher@279e7d4[solrj_collection_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
    [junit4] 2> 1692545 T4434 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica1
    [junit4] 2> 1692546 T4434 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica1 address:http://127.0.0.1:18022 collection:solrj_collection shard:shard1
    [junit4] 2> 1692546 T4459 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica2
    [junit4] 2> 1692547 T4434 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
    [junit4] 2> 1692547 T4459 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica2 address:http://127.0.0.1:18027 collection:solrj_collection shard:shard1
    [junit4] 2> 1692548 T4459 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
    [junit4] 2> 1692551 T4434 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
    [junit4] 2> 1692553 T4450 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
    [junit4] 2> 1692553 T4434 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
    [junit4] 2> 1692553 T4434 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
    [junit4] 2> ASYNC NEW_CORE C722 name=solrj_collection_shard1_replica1 org.apache.solr.core.SolrCore@3357631e url=http://127.0.0.1:18022/solrj_collection_shard1_replica1 node=127.0.0.1:18022_ C722_STATE=coll:solrj_collection core:solrj_collection_shard1_replica1 props:{node_name=127.0.0.1:18022_, state=down, core=solrj_collection_shard1_replica1, base_url=http://127.0.0.1:18022}
    [junit4] 2> 1692554 T4434 C722 P18022 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:18022/solrj_collection_shard1_replica1/
    [junit4] 2> 1692554 T4434 C722 P18022 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard1_replica1 url=http://127.0.0.1:18022 START replicas=[http://127.0.0.1:18027/solrj_collection_shard1_replica2/] nUpdates=100
    [junit4] 2> 1692555 T4434 C722 P18022 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard1_replica1 url=http://127.0.0.1:18022 DONE. We have no versions. sync failed.
    [junit4] 2> 1692556 T4434 C722 P18022 oasc.SyncStrategy.syncReplicas Leader's attempt to sync with shard failed, moving to the next candidate
    [junit4] 2> 1692556 T4434 oasc.ShardLeaderElectionContext.runLeaderProcess We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
    [junit4] 2> 1692557 T4434 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:18022/solrj_collection_shard1_replica1/ shard1
    [junit4] 2> 1692557 T4434 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leaders/shard1
    [junit4] 2> ASYNC NEW_CORE C723 name=solrj_collection_shard1_replica2 org.apache.solr.core.SolrCore@53a1e519 url=http://127.0.0.1:18027/solrj_collection_shard1_replica2 node=127.0.0.1:18027_ C723_STATE=coll:solrj_collection core:solrj_collection_shard1_replica2 props:{node_name=127.0.0.1:18027_, state=down, core=solrj_collection_shard1_replica2, base_url=http://127.0.0.1:18027}
    [junit4] 2> 1692558 T4460 C723 P18027 oasc.SolrCore.execute [solrj_collection_shard1_replica2] webapp= path=/get params={qt=/get&distrib=false&wt=javabin&version=2&getVersions=100} status=0 QTime=1
    [junit4] 2> 1692559 T4450 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1692559 T4451 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1692559 T4510 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1692559 T4465 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:No

[...truncated too long message...]

  T10228 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
    [junit4] 2> 2928521 T10228 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:45459 45459
    [junit4] 2> 2928725 T10229 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:45459 45459
    [junit4] 2> 2928824 T10292 oaz.ClientCnxn$EventThread.processEvent ERROR Error while calling watcher java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@1e6b173a rejected from java.util.concurrent.ThreadPoolExecutor@8c1f070[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 5]
    [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
    [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
    [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
    [junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:110)
    [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:263)
    [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
    [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
    [junit4] 2>
    [junit4] 2> 2928825 T10292 oaz.ClientCnxn$EventThread.processEvent ERROR Error while calling watcher java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@2dd39fc4 rejected from java.util.concurrent.ThreadPoolExecutor@8c1f070[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 5]
    [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
    [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
    [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
    [junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:110)
    [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:263)
    [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
    [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
    [junit4] 2>
    [junit4] 2> 2928825 T10292 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@52c4149d name:ZooKeeperConnection Watcher:127.0.0.1:45459/solr got event WatchedEvent state:Disconnected type:None path:null path:null type:None
    [junit4] 2> 2928825 T10292 oascc.ConnectionManager.process Client->ZooKeeper status change trigger but we are already closed
    [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
    [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=DeleteLastCustomShardedReplicaTest -Dtests.method=testDistribSearch -Dtests.seed=F6FA1D3B30CEE5AD -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ga_IE -Dtests.timezone=Asia/Ust-Nera -Dtests.file.encoding=US-ASCII
    [junit4] ERROR 135s J2 | DeleteLastCustomShardedReplicaTest.testDistribSearch <<<
    [junit4] > Throwable #1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:45467/gkx/d, http://127.0.0.1:45475/gkx/d, http://127.0.0.1:45472/gkx/d]
    [junit4] > at __randomizedtesting.SeedInfo.seed([F6FA1D3B30CEE5AD:771C932347918591]:0)
    [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:333)
    [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrServer.sendRequest(CloudSolrServer.java:1015)
    [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrServer.requestWithRetryOnStaleState(CloudSolrServer.java:793)
    [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:736)
    [junit4] > at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.removeAndWaitForLastReplicaGone(DeleteLastCustomShardedReplicaTest.java:117)
    [junit4] > at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.doTest(DeleteLastCustomShardedReplicaTest.java:107)
    [junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
    [junit4] > at java.lang.Thread.run(Thread.java:745)
    [junit4] > Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Could not remove replica : customcollreplicadeletion/a/core_node1
    [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
    [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
    [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
    [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrServer.doRequest(LBHttpSolrServer.java:354)
    [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:313)
    [junit4] > ... 46 more
    [junit4] 2> 2928835 T10228 oas.SolrTestCaseJ4.deleteCore ###deleteCore
    [junit4] 2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-F6FA1D3B30CEE5AD-001
    [junit4] 2> 134677 T10227 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 2 leaked thread(s).
    [junit4] 2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=ga_IE, timezone=Asia/Ust-Nera
    [junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=124428160,total=422051840
    [junit4] 2> NOTE: All tests run in this JVM: [DefaultValueUpdateProcessorTest, ModifyConfFileTest, TestDistributedSearch, DateMathParserTest, TestStandardQParsers, TestRandomDVFaceting, TestLuceneMatchVersion, TestReversedWildcardFilterFactory, EnumFieldTest, RegexBoostProcessorTest, OverseerRolesTest, CollectionsAPIAsyncDistributedZkTest, QueryResultKeyTest, TestManagedSynonymFilterFactory, TestPerFieldSimilarity, UpdateRequestProcessorFactoryTest, RemoteQueryErrorTest, TestCloudInspectUtil, TestSolrQueryParserResource, ZkNodePropsTest, TestReplicaProperties, TestSortingResponseWriter, RAMDirectoryFactoryTest, CurrencyFieldXmlFileTest, AliasIntegrationTest, TestManagedSchema, HdfsWriteToMultipleCollectionsTest, OpenExchangeRatesOrgProviderTest, BasicFunctionalityTest, ShardSplitTest, TestMultiCoreConfBootstrap, TestFieldTypeResource, IndexSchemaTest, CopyFieldTest, TestPseudoReturnFields, SolrXmlInZkTest, LeaderInitiatedRecoveryOnCommitTest, LeaderElectionIntegrationTest, TestSolrQueryParser, TestInfoStreamLogging, TestMaxScoreQueryParser, TestDistributedMissingSort, TestDocBasedVersionConstraints, TestIndexingPerformance, BinaryUpdateRequestHandlerTest, TestStressRecovery, PolyFieldTest, TestHighlightDedupGrouping, HdfsChaosMonkeySafeLeaderTest, TestAnalyzeInfixSuggestions, TestSolr4Spatial, TestJmxMonitoredMap, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, JSONWriterTest, QueryParsingTest, TestAnalyzedSuggestions, TestPartialUpdateDeduplication, PingRequestHandlerTest, SearchHandlerTest, HighlighterConfigTest, UpdateParamsTest, SolrIndexConfigTest, MultiTermTest, NumericFieldsTest, TestElisionMultitermQuery, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, TestPostingsSolrHighlighter, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCharFilters, EchoParamsTest, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestIBSimilarityFactory, TestDefaultSimilarityFactory, LegacyHTMLStripCharFilterTest, PluginInfoTest, TestFastLRUCache, TestSuggestSpellingConverter, DOMUtilTest, TestDocumentBuilder, TestRTGBase, DistributedIntervalFacetingTest, DeleteLastCustomShardedReplicaTest]
    [junit4] Completed on J2 in 135.47s, 1 test, 1 error <<< FAILURES!

[...truncated 310 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:531: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:472: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1360: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:962: There were test failures: 437 suites, 1797 tests, 2 errors, 54 ignored (9 assumptions)

Total time: 230 minutes 17 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #654
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 679426569 bytes
Compression is 0.0%
Took 2 min 0 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure

Search Discussions

  • Apache Jenkins Server at Oct 23, 2014 at 8:14 am
    Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/656/

    3 tests failed.
    REGRESSION: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testDistribSearch

    Error Message:
    Captured an uncaught exception in thread: Thread[id=7042, name=Thread-3262, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

    Stack Trace:
    com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=7042, name=Thread-3262, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
      at __randomizedtesting.SeedInfo.seed([EA393A22F1FBC792:6BDFB43A86A4A7AE]:0)
    Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:26091/hg_qmd/collection1
      at __randomizedtesting.SeedInfo.seed([EA393A22F1FBC792]:0)
      at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:380)
    Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:26091/hg_qmd/collection1
      at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:579)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
      at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
      at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:378)
    Caused by: java.net.SocketTimeoutException: Read timed out
      at java.net.SocketInputStream.socketRead0(Native Method)
      at java.net.SocketInputStream.read(SocketInputStream.java:152)
      at java.net.SocketInputStream.read(SocketInputStream.java:122)
      at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
      at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
      at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
      at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
      at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
      at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
      at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
      at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
      at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
      at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
      at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
      at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
      at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
      at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
      at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
      at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
      at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:465)
      ... 4 more


    FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

    Error Message:
    Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1

    Stack Trace:
    org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1
      at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
      at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:583)
      at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:205)
      at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:606)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at java.lang.Thread.run(Thread.java:745)


    FAILED: org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.testDistribSearch

    Error Message:
    No live SolrServers available to handle this request:[http://127.0.0.1:20635/zmsxn, http://127.0.0.1:20640/zmsxn, http://127.0.0.1:20615/zmsxn]

    Stack Trace:
    org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:20635/zmsxn, http://127.0.0.1:20640/zmsxn, http://127.0.0.1:20615/zmsxn]
      at __randomizedtesting.SeedInfo.seed([EA393A22F1FBC792:6BDFB43A86A4A7AE]:0)
      at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:333)
      at org.apache.solr.client.solrj.impl.CloudSolrServer.sendRequest(CloudSolrServer.java:1015)
      at org.apache.solr.client.solrj.impl.CloudSolrServer.requestWithRetryOnStaleState(CloudSolrServer.java:793)
      at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:736)
      at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.removeAndWaitForLastReplicaGone(DeleteLastCustomShardedReplicaTest.java:117)
      at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.doTest(DeleteLastCustomShardedReplicaTest.java:107)
      at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:606)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at java.lang.Thread.run(Thread.java:745)
    Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Could not remove replica : customcollreplicadeletion/a/core_node2
      at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
      at org.apache.solr.client.solrj.impl.LBHttpSolrServer.doRequest(LBHttpSolrServer.java:354)
      at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:313)
      ... 46 more




    Build Log:
    [...truncated 11977 lines...]
        [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
        [junit4] 2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/init-core-data-001
        [junit4] 2> 350699 T988 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
        [junit4] 2> 350699 T988 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
        [junit4] 2> 350707 T988 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
        [junit4] 2> 350708 T988 oasc.ZkTestServer.run STARTING ZK TEST SERVER
        [junit4] 1> client port:0.0.0.0/0.0.0.0:0
        [junit4] 2> 350709 T989 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
        [junit4] 2> 350809 T988 oasc.ZkTestServer.run start zk server on port:61302
        [junit4] 2> 350810 T988 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 350811 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 350815 T995 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f2af4a3 name:ZooKeeperConnection Watcher:127.0.0.1:61302 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 350815 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 350816 T988 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 350816 T988 oascc.SolrZkClient.makePath makePath: /solr
        [junit4] 2> 350819 T988 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 350820 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 350822 T997 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@256582ad name:ZooKeeperConnection Watcher:127.0.0.1:61302/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 350823 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 350823 T988 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 350823 T988 oascc.SolrZkClient.makePath makePath: /collections/collection1
        [junit4] 2> 350826 T988 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
        [junit4] 2> 350828 T988 oascc.SolrZkClient.makePath makePath: /collections/control_collection
        [junit4] 2> 350830 T988 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
        [junit4] 2> 350832 T988 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
        [junit4] 2> 350833 T988 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
        [junit4] 2> 350836 T988 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
        [junit4] 2> 350837 T988 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
        [junit4] 2> 350840 T988 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 350841 T988 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 350843 T988 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
        [junit4] 2> 350844 T988 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
        [junit4] 2> 350846 T988 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
        [junit4] 2> 350847 T988 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
        [junit4] 2> 350849 T988 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
        [junit4] 2> 350849 T988 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
        [junit4] 2> 350851 T988 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
        [junit4] 2> 350852 T988 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
        [junit4] 2> 350854 T988 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
        [junit4] 2> 350855 T988 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
        [junit4] 2> 350857 T988 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
        [junit4] 2> 350857 T988 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
        [junit4] 2> 350859 T988 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
        [junit4] 2> 350860 T988 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
        [junit4] 2> 350862 T988 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
        [junit4] 2> 350863 T988 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
        [junit4] 2> Using old style solr.xml
        [junit4] 2> 352294 T988 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 352299 T988 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:61321
        [junit4] 2> 352300 T988 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 352300 T988 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 352300 T988 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-002
        [junit4] 2> 352301 T988 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-002/'
        [junit4] 2> 352332 T988 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-002/solr.xml
        [junit4] 2> 352387 T988 oasc.CoreContainer.<init> New CoreContainer 1578828210
        [junit4] 2> 352387 T988 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-002/]
        [junit4] 2> 352388 T988 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 352389 T988 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 352389 T988 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 352389 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 352389 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 352390 T988 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 352390 T988 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 352390 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 352391 T988 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 352391 T988 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 352391 T988 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 352392 T988 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 352392 T988 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 352392 T988 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 352393 T988 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:61302/solr
        [junit4] 2> 352393 T988 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 352393 T988 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 352394 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 352396 T1008 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f0c7d62 name:ZooKeeperConnection Watcher:127.0.0.1:61302 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 352397 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 352397 T988 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 352400 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 352401 T1010 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@30a3a5a3 name:ZooKeeperConnection Watcher:127.0.0.1:61302/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 352401 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 352403 T988 oascc.SolrZkClient.makePath makePath: /overseer/queue
        [junit4] 2> 352405 T988 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
        [junit4] 2> 352407 T988 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
        [junit4] 2> 352409 T988 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
        [junit4] 2> 352411 T988 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
        [junit4] 2> 352413 T988 oascc.SolrZkClient.makePath makePath: /live_nodes
        [junit4] 2> 352414 T988 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61321_
        [junit4] 2> 352415 T988 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:61321_
        [junit4] 2> 352417 T988 oascc.SolrZkClient.makePath makePath: /overseer_elect
        [junit4] 2> 352418 T988 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
        [junit4] 2> 352419 T988 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 352421 T988 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:61321_
        [junit4] 2> 352422 T988 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
        [junit4] 2> 352423 T988 oasc.Overseer.start Overseer (id=92670987772559363-127.0.0.1:61321_-n_0000000000) starting
        [junit4] 2> 352425 T988 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
        [junit4] 2> 352431 T988 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
        [junit4] 2> 352432 T1012 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
        [junit4] 2> 352432 T988 oascc.SolrZkClient.makePath makePath: /clusterstate.json
        [junit4] 2> 352435 T988 oascc.SolrZkClient.makePath makePath: /aliases.json
        [junit4] 2> 352436 T988 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 352439 T1011 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
        [junit4] 2> 352441 T988 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 352442 T988 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 352442 T988 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 352443 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 352445 T1016 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@390ff1eb name:ZooKeeperConnection Watcher:127.0.0.1:61302/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 352445 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 352445 T988 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 352446 T988 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 352448 T988 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
        [junit4] 2> 352449 T988 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
        [junit4] 2> 352449 T988 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 352450 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 352451 T1018 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1076021c name:ZooKeeperConnection Watcher:127.0.0.1:61302/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 352451 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 352451 T988 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 352454 T1019 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 352456 T1011 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: collection1
        [junit4] 2> 352457 T1011 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
        [junit4] 2> 352457 T1011 oasc.Overseer$ClusterStateUpdater.createCollection state version collection1 2
        [junit4] 2> 352459 T1011 oasc.Overseer$ClusterStateUpdater.updateZkStates going to create_collection /collections/collection1/state.json
        [junit4] 2> 352460 T1019 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 352461 T1020 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 353400 T988 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
        [junit4] 2> 353401 T988 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 353404 T988 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:61326
        [junit4] 2> 353404 T988 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 353405 T988 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 353405 T988 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-003
        [junit4] 2> 353405 T988 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-003/'
        [junit4] 2> 353436 T988 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-003/solr.xml
        [junit4] 2> 353523 T988 oasc.CoreContainer.<init> New CoreContainer 2123948909
        [junit4] 2> 353524 T988 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-003/]
        [junit4] 2> 353525 T988 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 353525 T988 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 353525 T988 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 353525 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 353526 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 353526 T988 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 353526 T988 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 353527 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 353527 T988 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 353527 T988 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 353528 T988 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 353528 T988 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 353528 T988 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 353529 T988 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 353529 T988 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:61302/solr
        [junit4] 2> 353529 T988 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 353530 T988 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 353530 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 353532 T1031 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@677b4a09 name:ZooKeeperConnection Watcher:127.0.0.1:61302 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 353533 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 353534 T988 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 353536 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 353538 T1033 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@629ec43d name:ZooKeeperConnection Watcher:127.0.0.1:61302/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 353538 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 353544 T988 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 354549 T988 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61326_
        [junit4] 2> 354550 T988 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:61326_
        [junit4] 2> 354555 T988 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 354563 T988 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 354564 T988 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 356210 T988 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
        [junit4] 2> 356211 T988 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 356214 T988 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:61337
        [junit4] 2> 356215 T988 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 356215 T988 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 356216 T988 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-004
        [junit4] 2> 356216 T988 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-004/'
        [junit4] 2> 356251 T988 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-004/solr.xml
        [junit4] 2> 356310 T988 oasc.CoreContainer.<init> New CoreContainer 1850568814
        [junit4] 2> 356310 T988 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-004/]
        [junit4] 2> 356312 T988 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 356312 T988 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 356312 T988 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 356312 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 356313 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 356313 T988 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 356313 T988 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 356314 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 356314 T988 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 356314 T988 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 356315 T988 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 356315 T988 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 356315 T988 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 356316 T988 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 356316 T988 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:61302/solr
        [junit4] 2> 356316 T988 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 356317 T988 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 356317 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 356319 T1046 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e107e55 name:ZooKeeperConnection Watcher:127.0.0.1:61302 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 356320 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 356321 T988 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 356323 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 356324 T1048 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@50a31981 name:ZooKeeperConnection Watcher:127.0.0.1:61302/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 356324 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 356329 T988 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 357334 T988 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61337_
        [junit4] 2> 357335 T988 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:61337_
        [junit4] 2> 357339 T988 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 357345 T988 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 357345 T988 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 358574 T988 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
        [junit4] 2> 358575 T988 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 358578 T988 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:61344
        [junit4] 2> 358578 T988 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 358579 T988 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 358579 T988 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-005
        [junit4] 2> 358579 T988 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-005/'
        [junit4] 2> 358610 T988 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-005/solr.xml
        [junit4] 2> 358665 T988 oasc.CoreContainer.<init> New CoreContainer 608407948
        [junit4] 2> 358665 T988 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-005/]
        [junit4] 2> 358666 T988 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 358666 T988 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 358667 T988 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 358667 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 358667 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 358668 T988 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 358668 T988 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 358668 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 358668 T988 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 358669 T988 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 358669 T988 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 358670 T988 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 358670 T988 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 358670 T988 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 358671 T988 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:61302/solr
        [junit4] 2> 358671 T988 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 358671 T988 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 358672 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 358673 T1061 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@500890ed name:ZooKeeperConnection Watcher:127.0.0.1:61302 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 358674 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 358675 T988 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 358677 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 358679 T1063 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b2dc371 name:ZooKeeperConnection Watcher:127.0.0.1:61302/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 358679 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 358684 T988 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 359688 T988 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61344_
        [junit4] 2> 359689 T988 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:61344_
        [junit4] 2> 359693 T988 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 359699 T988 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 359700 T988 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 361014 T988 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
        [junit4] 2> 361016 T988 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 361019 T988 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:61352
        [junit4] 2> 361019 T988 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 361019 T988 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 361020 T988 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-006
        [junit4] 2> 361020 T988 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-006/'
        [junit4] 2> 361051 T988 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-006/solr.xml
        [junit4] 2> 361105 T988 oasc.CoreContainer.<init> New CoreContainer 921279017
        [junit4] 2> 361106 T988 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-006/]
        [junit4] 2> 361107 T988 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 361107 T988 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 361107 T988 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 361108 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 361108 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 361108 T988 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 361109 T988 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 361109 T988 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 361109 T988 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 361109 T988 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 361110 T988 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 361110 T988 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 361111 T988 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 361111 T988 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 361111 T988 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:61302/solr
        [junit4] 2> 361112 T988 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 361112 T988 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 361113 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 361114 T1076 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@228815e9 name:ZooKeeperConnection Watcher:127.0.0.1:61302 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 361115 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 361116 T988 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 361118 T988 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 361119 T1078 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@56138b49 name:ZooKeeperConnection Watcher:127.0.0.1:61302/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 361119 T988 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 361124 T988 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 362129 T988 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61352_
        [junit4] 2> 362130 T988 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:61352_
        [junit4] 2> 362134 T988 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 362139 T988 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 362140 T988 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 362152 T1024 oasha.CollectionsHandler.handleCreateAction Creating Collection : router.field=myOwnField&action=CREATE&collection.configName=conf1&version=2&wt=javabin&numShards=2&name=solrj_collection&replicationFactor=2
        [junit4] 2> 362154 T1019 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 362155 T1012 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
        [junit4] 2> "operation":"create",
        [junit4] 2> "fromApi":"true",
        [junit4] 2> "name":"solrj_collection",
        [junit4] 2> "replicationFactor":"2",
        [junit4] 2> "collection.configName":"conf1",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "router.field":"myOwnField"}
        [junit4] 2> 362157 T1081 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : create , {
        [junit4] 2> "operation":"create",
        [junit4] 2> "fromApi":"true",
        [junit4] 2> "name":"solrj_collection",
        [junit4] 2> "replicationFactor":"2",
        [junit4] 2> "collection.configName":"conf1",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "router.field":"myOwnField"}
        [junit4] 2> 362158 T1081 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection
        [junit4] 2> 362158 T1081 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
        [junit4] 2> 362161 T1019 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 362163 T1011 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
        [junit4] 2> 362164 T1011 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
        [junit4] 2> 362164 T1011 oasc.Overseer$ClusterStateUpdater.createCollection state version solrj_collection 1
        [junit4] 2> 362166 T1019 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 362166 T1020 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 362166 T1064 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 362167 T1049 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 362167 T1034 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 362166 T1079 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 362261 T1081 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection solrj_collection, shardNames [shard1, shard2] , replicationFactor : 2
        [junit4] 2> 362262 T1081 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:61321_
        [junit4] 2> 362263 T1081 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:61344_
        [junit4] 2> 362265 T1081 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:61337_
        [junit4] 2> 362266 T1081 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:61352_
        [junit4] 2> 362267 T1002 oasha.CoreAdminHandler.handleCreateAction core create command collection=solrj_collection&action=CREATE&collection.configName=conf1&version=2&qt=/admin/cores&shard=shard1&wt=javabin&numShards=2&name=solrj_collection_shard1_replica1
        [junit4] 2> 362268 T1038 oasha.CoreAdminHandler.handleCreateAction core create command collection=solrj_collection&action=CREATE&collection.configName=conf1&version=2&qt=/admin/cores&shard=shard2&wt=javabin&numShards=2&name=solrj_collection_shard2_replica1
        [junit4] 2> 362267 T1053 oasha.CoreAdminHandler.handleCreateAction core create command collection=solrj_collection&action=CREATE&collection.configName=conf1&version=2&qt=/admin/cores&shard=shard1&wt=javabin&numShards=2&name=solrj_collection_shard1_replica2
        [junit4] 2> 362269 T1068 oasha.CoreAdminHandler.handleCreateAction core create command collection=solrj_collection&action=CREATE&collection.configName=conf1&version=2&qt=/admin/cores&shard=shard2&wt=javabin&numShards=2&name=solrj_collection_shard2_replica2
        [junit4] 2> 362272 T1053 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down collection=solrj_collection
        [junit4] 2> 362272 T1002 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down collection=solrj_collection
        [junit4] 2> 362272 T1068 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down collection=solrj_collection
        [junit4] 2> 362273 T1019 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 362272 T1038 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down collection=solrj_collection
        [junit4] 2> 362274 T1068 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 362273 T1002 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 362273 T1053 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 362275 T1038 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 362276 T1011 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "roles":null,
        [junit4] 2> "state":"down",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "operation":"state",
        [junit4] 2> "core":"solrj_collection_shard1_replica2",
        [junit4] 2> "shard":"shard1",
        [junit4] 2> "base_url":"http://127.0.0.1:61344",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "node_name":"127.0.0.1:61344_"}
        [junit4] 2> 362279 T1011 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "roles":null,
        [junit4] 2> "state":"down",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "operation":"state",
        [junit4] 2> "core":"solrj_collection_shard1_replica1",
        [junit4] 2> "shard":"shard1",
        [junit4] 2> "base_url":"http://127.0.0.1:61321",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "node_name":"127.0.0.1:61321_"}
        [junit4] 2> 362280 T1019 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 362281 T1011 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "roles":null,
        [junit4] 2> "state":"down",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "operation":"state",
        [junit4] 2> "core":"solrj_collection_shard2_replica2",
        [junit4] 2> "shard":"shard2",
        [junit4] 2> "base_url":"http://127.0.0.1:61352",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "node_name":"127.0.0.1:61352_"}
        [junit4] 2> 362283 T1011 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "roles":null,
        [junit4] 2> "state":"down",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "operation":"state",
        [junit4] 2> "core":"solrj_collection_shard2_replica1",
        [junit4] 2> "shard":"shard2",
        [junit4] 2> "base_url":"http://127.0.0.1:61337",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "node_name":"127.0.0.1:61337_"}
        [junit4] 2> 362285 T1019 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 362286 T1019 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 362286 T1049 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 362286 T1020 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 362286 T1034 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 362286 T1079 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 362286 T1064 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 363275 T1002 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
        [junit4] 2> 363275 T1068 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
        [junit4] 2> 363276 T1038 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
        [junit4] 2> 363276 T1053 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
        [junit4] 2> 363276 T1002 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 363277 T1053 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 363277 T1038 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 363277 T1068 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 363279 T1038 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 363278 T1053 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 363278 T1002 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 363280 T1053 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 363279 T1038 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 363279 T1068 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 363281 T1053 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 363280 T1002 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 363282 T1053 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-005/solrj_collection_shard1_replica2/'
        [junit4] 2> 363283 T1002 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 363282 T1038 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 363281 T1068 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 363284 T1038 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-004/solrj_collection_shard2_replica1/'
        [junit4] 2> 363283 T1002 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-002/solrj_collection_shard1_replica1/'
        [junit4] 2> 363284 T1068 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 363286 T1068 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-006/solrj_collection_shard2_replica2/'
        [junit4] 2> 363342 T1053 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 363347 T1038 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 363355 T1068 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 363359 T1002 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 363404 T1053 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 363406 T1053 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
        [junit4] 2> 363409 T1038 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 363411 T1038 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
        [junit4] 2> 363419 T1053 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
        [junit4] 2> 363420 T1068 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 363422 T1068 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
        [junit4] 2> 363423 T1002 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 363424 T1038 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
        [junit4] 2> 363424 T1002 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
        [junit4] 2> 363434 T1068 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
        [junit4] 2> 363438 T1002 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
        [junit4] 2> 363876 T1053 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 363885 T1038 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 363889 T1053 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 363893 T1053 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 363894 T1038 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 363898 T1038 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 363899 T1002 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 363900 T1068 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 363907 T1002 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 363909 T1053 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 363911 T1002 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 363913 T1068 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 363914 T1038 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 363916 T1053 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 363917 T1068 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 363919 T1038 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 363921 T1053 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 363923 T1053 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 363925 T1053 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 363926 T1053 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 363926 T1038 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 363927 T1053 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 363927 T1038 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 363928 T1053 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 363928 T1038 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 363929 T1053 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using configuration from collection solrj_collection
        [junit4] 2> 363929 T1038 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 363930 T1002 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 363930 T1053 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 363931 T1038 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 363931 T1038 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 363931 T1053 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-005/solrj_collection_shard1_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-005/solrj_collection_shard1_replica2/data/
        [junit4] 2> 363932 T1038 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using configuration from collection solrj_collection
        [junit4] 2> 363932 T1053 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3cc5a37d
        [junit4] 2> 363933 T1038 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 363934 T1038 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-004/solrj_collection_shard2_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-004/solrj_collection_shard2_replica1/data/
        [junit4] 2> 363934 T1038 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3cc5a37d
        [junit4] 2> 363936 T1053 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-005/solrj_collection_shard1_replica2/data
        [junit4] 2> 363936 T1068 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 363936 T1002 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 363937 T1038 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-004/solrj_collection_shard2_replica1/data
        [junit4] 2> 363937 T1053 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-005/solrj_collection_shard1_replica2/data/index/
        [junit4] 2> 363938 T1053 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-005/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
        [junit4] 2> 363938 T1038 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-004/solrj_collection_shard2_replica1/data/index/
        [junit4] 2> 363939 T1038 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-004/solrj_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
        [junit4] 2> 363940 T1053 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-005/solrj_collection_shard1_replica2/data/index
        [junit4] 2> 363941 T1053 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=53.876953125, floorSegmentMB=1.5595703125, forceMergeDeletesPctAllowed=27.52080108617267, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
        [junit4] 2> 363942 T1068 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 363942 T1038 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-004/solrj_collection_shard2_replica1/data/index
        [junit4] 2> 363942 T1002 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 363943 T1038 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=53.876953125, floorSegmentMB=1.5595703125, forceMergeDeletesPctAllowed=27.52080108617267, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
        [junit4] 2> 363944 T1002 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 363944 T1002 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 363945 T1002 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 363946 T1002 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 363947 T1068 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 363947 T1002 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 363948 T1068 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 363948 T1002 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using configuration from collection solrj_collection
        [junit4] 2> 363948 T1068 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 363949 T1002 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 363949 T1068 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 363950 T1002 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-002/solrj_collection_shard1_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-002/solrj_collection_shard1_replica1/data/
        [junit4] 2> 363951 T1002 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3cc5a37d
        [junit4] 2> 363951 T1068 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 363951 T1068 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 363952 T1068 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using configuration from collection solrj_collection
        [junit4] 2> 363952 T1068 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 363953 T1068 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-006/solrj_collection_shard2_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-006/solrj_collection_shard2_replica2/data/
        [junit4] 2> 363953 T1068 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3cc5a37d
        [junit4] 2> 363953 T1002 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-002/solrj_collection_shard1_replica1/data
        [junit4] 2> 363955 T1002 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-002/solrj_collection_shard1_replica1/data/index/
        [junit4] 2> 363955 T1002 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-002/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
        [junit4] 2> 363956 T1068 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-006/solrj_collection_shard2_replica2/data
        [junit4] 2> 363957 T1068 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-006/solrj_collection_shard2_replica2/data/index/
        [junit4] 2> 363958 T1002 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-002/solrj_collection_shard1_replica1/data/index
        [junit4] 2> 363958 T1068 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-006/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
        [junit4] 2> 363958 T1002 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=53.876953125, floorSegmentMB=1.5595703125, forceMergeDeletesPctAllowed=27.52080108617267, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
        [junit4] 2> 363961 T1068 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/tempDir-006/solrj_collection_shard2_replica2/data/index
        [junit4] 2> 363961 T1068 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=53.876953125, floorSegmentMB=1.5595703125, forceMergeDeletesPctAllowed=27.52080108617267, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
        [junit4] 2> 363962 T1053 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(RateLimitedDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/index-MMapDirectory-001 lockFactory=NativeFSLockFactory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/index-MMapDirectory-001)),segFN=segments_1,generation=1}
        [junit4] 2> 363963 T1038 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(RateLimitedDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/index-MMapDirectory-002 lockFactory=NativeFSLockFactory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/index-MMapDirectory-002)),segFN=segments_1,generation=1}
        [junit4] 2> 363963 T1053 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 363963 T1038 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 363965 T1002 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(RateLimitedDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/index-MMapDirectory-003 lockFactory=NativeFSLockFactory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/index-MMapDirectory-003)),segFN=segments_1,generation=1}
        [junit4] 2> 363966 T1002 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 363968 T1068 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(RateLimitedDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/index-MMapDirectory-004 lockFactory=NativeFSLockFactory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-EA393A22F1FBC792-001/index-MMapDirectory-004)),segFN=segments_1,generation=1}
        [junit4] 2> 363969 T1068 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 363971 T1038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 363972 T1038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 363972 T1053 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 363973 T1002 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 363972 T1038 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 363973 T1002 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 363973 T1053 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 363974 T1002 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 363974 T1038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 363975 T1068 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 363975 T1002 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 363975 T1053 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 363976 T1002 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2>

    [...truncated too long message...]

    eeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
        [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
        [junit4] 2>
        [junit4] 2> 3046807 T23343 oaz.ClientCnxn$EventThread.processEvent ERROR Error while calling watcher java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@116b5967 rejected from java.util.concurrent.ThreadPoolExecutor@6c2936e[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 4]
        [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
        [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
        [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
        [junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:110)
        [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:263)
        [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
        [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
        [junit4] 2>
        [junit4] 2> 3046807 T23343 oaz.ClientCnxn$EventThread.processEvent ERROR Error while calling watcher java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@66b512a2 rejected from java.util.concurrent.ThreadPoolExecutor@6c2936e[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 4]
        [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
        [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
        [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
        [junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:110)
        [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:263)
        [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
        [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
        [junit4] 2>
        [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
        [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=DeleteLastCustomShardedReplicaTest -Dtests.method=testDistribSearch -Dtests.seed=EA393A22F1FBC792 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=es_VE -Dtests.timezone=Etc/UCT -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
        [junit4] ERROR 137s J1 | DeleteLastCustomShardedReplicaTest.testDistribSearch <<<
        [junit4] > Throwable #1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:20635/zmsxn, http://127.0.0.1:20640/zmsxn, http://127.0.0.1:20615/zmsxn]
        [junit4] > at __randomizedtesting.SeedInfo.seed([EA393A22F1FBC792:6BDFB43A86A4A7AE]:0)
        [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:333)
        [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrServer.sendRequest(CloudSolrServer.java:1015)
        [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrServer.requestWithRetryOnStaleState(CloudSolrServer.java:793)
        [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:736)
        [junit4] > at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.removeAndWaitForLastReplicaGone(DeleteLastCustomShardedReplicaTest.java:117)
        [junit4] > at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.doTest(DeleteLastCustomShardedReplicaTest.java:107)
        [junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
        [junit4] > at java.lang.Thread.run(Thread.java:745)
        [junit4] > Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Could not remove replica : customcollreplicadeletion/a/core_node2
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
        [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrServer.doRequest(LBHttpSolrServer.java:354)
        [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:313)
        [junit4] > ... 46 more
        [junit4] 2> 3046825 T23279 oas.SolrTestCaseJ4.deleteCore ###deleteCore
        [junit4] 2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-EA393A22F1FBC792-001
        [junit4] 2> 136611 T23278 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
        [junit4] 2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=es_VE, timezone=Etc/UCT
        [junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=110330360,total=489684992
        [junit4] 2> NOTE: All tests run in this JVM: [TestSystemIdResolver, TestManagedResource, TestComponentsName, TestFastOutputStream, StressHdfsTest, TestCollationField, TestZkChroot, TestCodecSupport, TestPartialUpdateDeduplication, PeerSyncTest, TestFunctionQuery, TestIntervalFaceting, TestRandomMergePolicy, TestClassNameShortening, ShowFileRequestHandlerTest, TestManagedSchemaFieldTypeResource, SortByFunctionTest, VMParamsZkACLAndCredentialsProvidersTest, SignatureUpdateProcessorFactoryTest, PreAnalyzedFieldTest, TestDocumentBuilder, SystemInfoHandlerTest, TestShardHandlerFactory, HttpPartitionTest, TestNoOpRegenerator, TestCollationFieldDocValues, DirectSolrSpellCheckerTest, ClusterStateTest, TestHighFrequencyDictionaryFactory, DistributedSuggestComponentTest, HighlighterMaxOffsetTest, ExternalFileFieldSortTest, TestSweetSpotSimilarityFactory, AddBlockUpdateTest, PrimitiveFieldTypeTest, OpenCloseCoreStressTest, JSONWriterTest, SharedFSAutoReplicaFailoverUtilsTest, SolrCoreCheckLockOnStartupTest, DebugComponentTest, ExternalCollectionsTest, SolrRequestParserTest, TestSchemaVersionResource, TestManagedSchemaFieldResource, TestFieldCollectionResource, TestAddFieldRealTimeGet, TestBM25SimilarityFactory, SuggesterFSTTest, CoreAdminRequestStatusTest, TestLFUCache, TestRecovery, MBeansHandlerTest, ClusterStateUpdateTest, LeaderElectionTest, ZkSolrClientTest, TestRandomDVFaceting, ZkCLITest, TestDistributedSearch, AutoCommitTest, TestDistributedGrouping, TermVectorComponentDistributedTest, HardAutoCommitTest, SpellCheckComponentTest, BadIndexSchemaTest, ConvertedLegacyTest, TestSort, TestFiltering, TestLazyCores, TestIndexSearcher, SoftAutoCommitTest, TestExtendedDismaxParser, DocValuesTest, SuggesterTSTTest, SuggesterTest, TestTrie, SuggesterWFSTTest, NoCacheHeaderTest, TestAtomicUpdateErrorCases, DirectUpdateHandlerOptimizeTest, StatelessScriptUpdateProcessorFactoryTest, DocValuesMultiTest, CacheHeaderTest, DisMaxRequestHandlerTest, TestWriterPerf, RequestHandlersTest, TestValueSourceCache, TermVectorComponentTest, RequiredFieldsTest, FastVectorHighlighterTest, LoggingHandlerTest, SolrPluginUtilsTest, ReturnFieldsTest, QueryParsingTest, CSVRequestHandlerTest, TestQuerySenderListener, TestQuerySenderNoQuery, SolrIndexConfigTest, TestBinaryField, DirectSolrConnectionTest, SpellPossibilityIteratorTest, TestXIncludeConfig, TestPerFieldSimilarity, TestLMJelinekMercerSimilarityFactory, TestDefaultSimilarityFactory, URLClassifyProcessorTest, SpellingQueryConverterTest, DOMUtilTest, ZkNodePropsTest, FileUtilsTest, TestDistributedMissingSort, CloudExitableDirectoryReaderTest, DeleteLastCustomShardedReplicaTest]
        [junit4] Completed on J1 in 136.89s, 1 test, 1 error <<< FAILURES!

    [...truncated 377 lines...]
    BUILD FAILED
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:531: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:472: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:189: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:496: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1359: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:966: There were test failures: 437 suites, 1798 tests, 3 errors, 54 ignored (9 assumptions)

    Total time: 191 minutes 21 seconds
    Build step 'Invoke Ant' marked build as failure
    Archiving artifacts
    Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #654
    Archived 5 artifacts
    Archive block size is 32768
    Received 0 blocks and 695028327 bytes
    Compression is 0.0%
    Took 1 min 57 sec
    Recording test results
    Email was triggered for: Failure
    Sending email for trigger: Failure
  • Apache Jenkins Server at Oct 24, 2014 at 6:12 am
    Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/657/

    1 tests failed.
    FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

    Error Message:
    Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1

    Stack Trace:
    org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1
      at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
      at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:583)
      at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:205)
      at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:606)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at java.lang.Thread.run(Thread.java:745)




    Build Log:
    [...truncated 11885 lines...]
        [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
        [junit4] 2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/init-core-data-001
        [junit4] 2> 561525 T948 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
        [junit4] 2> 561525 T948 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /hwaws/
        [junit4] 2> 561530 T948 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
        [junit4] 2> 561531 T948 oasc.ZkTestServer.run STARTING ZK TEST SERVER
        [junit4] 1> client port:0.0.0.0/0.0.0.0:0
        [junit4] 2> 561531 T949 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
        [junit4] 2> 561631 T948 oasc.ZkTestServer.run start zk server on port:58321
        [junit4] 2> 561632 T948 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 561633 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 561636 T955 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@57e19e66 name:ZooKeeperConnection Watcher:127.0.0.1:58321 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 561637 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 561637 T948 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 561637 T948 oascc.SolrZkClient.makePath makePath: /solr
        [junit4] 2> 561639 T948 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 561640 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 561641 T957 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c489ba2 name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 561641 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 561642 T948 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 561642 T948 oascc.SolrZkClient.makePath makePath: /collections/collection1
        [junit4] 2> 561644 T948 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
        [junit4] 2> 561645 T948 oascc.SolrZkClient.makePath makePath: /collections/control_collection
        [junit4] 2> 561647 T948 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
        [junit4] 2> 561649 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
        [junit4] 2> 561649 T948 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
        [junit4] 2> 561652 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
        [junit4] 2> 561652 T948 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
        [junit4] 2> 561754 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 561755 T948 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 561757 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
        [junit4] 2> 561757 T948 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
        [junit4] 2> 561759 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
        [junit4] 2> 561760 T948 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
        [junit4] 2> 561762 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
        [junit4] 2> 561762 T948 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
        [junit4] 2> 561764 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
        [junit4] 2> 561765 T948 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
        [junit4] 2> 561767 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
        [junit4] 2> 561767 T948 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
        [junit4] 2> 561769 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
        [junit4] 2> 561770 T948 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
        [junit4] 2> 561772 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
        [junit4] 2> 561772 T948 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
        [junit4] 2> 561774 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
        [junit4] 2> 561775 T948 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
        [junit4] 2> 561777 T948 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 561778 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 561779 T959 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69c3e028 name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 561779 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 561779 T948 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> Using new style solr.xml
        [junit4] 2> 561781 T948 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 561782 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 561783 T961 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@eceaafc name:ZooKeeperConnection Watcher:127.0.0.1:58321 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 561783 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 561783 T948 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 561783 T948 oascc.SolrZkClient.makePath makePath: /solr
        [junit4] 2> 561785 T948 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 561785 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 561786 T963 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a89dbcf name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 561787 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 561787 T948 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 561789 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf2/solrconfig.xml
        [junit4] 2> 561789 T948 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.xml
        [junit4] 2> 561791 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
        [junit4] 2> 561792 T948 oascc.SolrZkClient.makePath makePath: /configs/conf2/schema.xml
        [junit4] 2> 561894 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 561894 T948 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 561896 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
        [junit4] 2> 561896 T948 oascc.SolrZkClient.makePath makePath: /configs/conf2/stopwords.txt
        [junit4] 2> 561897 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
        [junit4] 2> 561898 T948 oascc.SolrZkClient.makePath makePath: /configs/conf2/protwords.txt
        [junit4] 2> 561899 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
        [junit4] 2> 561900 T948 oascc.SolrZkClient.makePath makePath: /configs/conf2/currency.xml
        [junit4] 2> 561901 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
        [junit4] 2> 561902 T948 oascc.SolrZkClient.makePath makePath: /configs/conf2/open-exchange-rates.json
        [junit4] 2> 561903 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
        [junit4] 2> 561903 T948 oascc.SolrZkClient.makePath makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
        [junit4] 2> 561905 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
        [junit4] 2> 561905 T948 oascc.SolrZkClient.makePath makePath: /configs/conf2/old_synonyms.txt
        [junit4] 2> 561907 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
        [junit4] 2> 561907 T948 oascc.SolrZkClient.makePath makePath: /configs/conf2/synonyms.txt
        [junit4] 2> 561909 T948 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
        [junit4] 2> 561909 T948 oascc.SolrZkClient.makePath makePath: /configs/conf2/elevate.xml
        [junit4] 2> 563055 T948 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 563126 T948 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
        [junit4] 2> 563130 T948 oejs.AbstractConnector.doStart Started sslsocketconnector@127.0.0.1:45232
        [junit4] 2> 563134 T948 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 563135 T948 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 563135 T948 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002
        [junit4] 2> 563135 T948 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002/'
        [junit4] 2> 563167 T948 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002/solr.xml
        [junit4] 2> 563211 T948 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002
        [junit4] 2> 563212 T948 oasc.CoreContainer.<init> New CoreContainer 1061830751
        [junit4] 2> 563212 T948 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002/]
        [junit4] 2> 563213 T948 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 563214 T948 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 563214 T948 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 563214 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 563214 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 563214 T948 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 563215 T948 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 563215 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 563215 T948 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 563215 T948 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 563218 T948 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 563220 T948 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 563221 T948 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 563221 T948 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 563221 T948 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58321/solr
        [junit4] 2> 563221 T948 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 563222 T948 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 563222 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 563224 T974 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@679fa099 name:ZooKeeperConnection Watcher:127.0.0.1:58321 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 563224 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 563225 T948 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 563226 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 563227 T976 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b64e155 name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 563227 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 563228 T948 oascc.SolrZkClient.makePath makePath: /overseer/queue
        [junit4] 2> 563230 T948 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
        [junit4] 2> 563232 T948 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
        [junit4] 2> 563233 T948 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
        [junit4] 2> 563235 T948 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
        [junit4] 2> 563237 T948 oascc.SolrZkClient.makePath makePath: /live_nodes
        [junit4] 2> 563238 T948 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45232_hwaws
        [junit4] 2> 563239 T948 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45232_hwaws
        [junit4] 2> 563240 T948 oascc.SolrZkClient.makePath makePath: /overseer_elect
        [junit4] 2> 563241 T948 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
        [junit4] 2> 563242 T948 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 563244 T948 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:45232_hwaws
        [junit4] 2> 563244 T948 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
        [junit4] 2> 563245 T948 oasc.Overseer.start Overseer (id=92676158186979334-127.0.0.1:45232_hwaws-n_0000000000) starting
        [junit4] 2> 563247 T948 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
        [junit4] 2> 563252 T948 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
        [junit4] 2> 563253 T978 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
        [junit4] 2> 563253 T948 oascc.SolrZkClient.makePath makePath: /clusterstate.json
        [junit4] 2> 563255 T948 oascc.SolrZkClient.makePath makePath: /aliases.json
        [junit4] 2> 563256 T948 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 563258 T977 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
        [junit4] 2> 563260 T948 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002
        [junit4] 2> 563877 T948 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002/conf/
        [junit4] 2> 563973 T948 oasc.CorePropertiesLocator.discover Found 1 core definitions
        [junit4] 2> 563974 T948 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 563975 T948 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 563980 T948 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 563980 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 563982 T982 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e58635d name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 563982 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 563982 T948 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 563983 T948 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 563987 T948 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
        [junit4] 2> 563987 T948 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
        [junit4] 2> 563987 T948 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 563988 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 563990 T984 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@683e3bb2 name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 563990 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 563990 T948 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 563992 T985 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 563994 T977 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: collection1
        [junit4] 2> 563994 T977 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
        [junit4] 2> 563995 T977 oasc.Overseer$ClusterStateUpdater.createCollection state version collection1 2
        [junit4] 2> 563996 T977 oasc.Overseer$ClusterStateUpdater.updateZkStates going to create_collection /collections/collection1/state.json
        [junit4] 2> 563997 T985 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 563998 T986 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 565157 T948 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
        [junit4] 2> 565157 T948 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 565164 T948 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
        [junit4] 2> 565166 T948 oejs.AbstractConnector.doStart Started sslsocketconnector@127.0.0.1:49224
        [junit4] 2> 565169 T948 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 565169 T948 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 565170 T948 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003
        [junit4] 2> 565170 T948 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003/'
        [junit4] 2> 565201 T948 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003/solr.xml
        [junit4] 2> 565245 T948 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003
        [junit4] 2> 565245 T948 oasc.CoreContainer.<init> New CoreContainer 1272352365
        [junit4] 2> 565245 T948 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003/]
        [junit4] 2> 565246 T948 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 565246 T948 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 565247 T948 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 565247 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 565247 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 565247 T948 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 565247 T948 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 565248 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 565248 T948 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 565248 T948 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 565251 T948 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 565253 T948 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 565253 T948 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 565254 T948 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 565254 T948 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58321/solr
        [junit4] 2> 565254 T948 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 565254 T948 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 565255 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 565257 T997 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4208aee0 name:ZooKeeperConnection Watcher:127.0.0.1:58321 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 565257 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 565258 T948 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 565260 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 565261 T999 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4bdc2efc name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 565261 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 565266 T948 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 566271 T948 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49224_hwaws
        [junit4] 2> 566272 T948 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49224_hwaws
        [junit4] 2> 566276 T948 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 566280 T948 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003
        [junit4] 2> 566453 T948 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003/conf/
        [junit4] 2> 567396 T948 oasc.CorePropertiesLocator.discover Found 1 core definitions
        [junit4] 2> 567397 T948 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 567397 T948 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 568582 T948 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
        [junit4] 2> 568583 T948 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 568589 T948 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
        [junit4] 2> 568592 T948 oejs.AbstractConnector.doStart Started sslsocketconnector@127.0.0.1:21602
        [junit4] 2> 568595 T948 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 568596 T948 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 568596 T948 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-004
        [junit4] 2> 568596 T948 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-004/'
        [junit4] 2> 568628 T948 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-004/solr.xml
        [junit4] 2> 568680 T948 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-004
        [junit4] 2> 568680 T948 oasc.CoreContainer.<init> New CoreContainer 1140943220
        [junit4] 2> 568681 T948 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-004/]
        [junit4] 2> 568682 T948 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 568682 T948 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 568682 T948 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 568682 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 568683 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 568683 T948 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 568683 T948 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 568683 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 568683 T948 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 568684 T948 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 568687 T948 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 568689 T948 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 568690 T948 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 568690 T948 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 568690 T948 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58321/solr
        [junit4] 2> 568691 T948 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 568691 T948 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 568692 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 568693 T1012 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e88a2f7 name:ZooKeeperConnection Watcher:127.0.0.1:58321 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 568694 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 568694 T948 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 568695 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 568697 T1014 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@dbf35b7 name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 568697 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 568702 T948 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 569706 T948 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:21602_hwaws
        [junit4] 2> 569707 T948 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:21602_hwaws
        [junit4] 2> 569710 T948 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 569715 T948 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-004
        [junit4] 2> 569722 T948 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-004/conf/
        [junit4] 2> 570802 T948 oasc.CorePropertiesLocator.discover Found 1 core definitions
        [junit4] 2> 570803 T948 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 570804 T948 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 572052 T948 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
        [junit4] 2> 572054 T948 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 572060 T948 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
        [junit4] 2> 572063 T948 oejs.AbstractConnector.doStart Started sslsocketconnector@127.0.0.1:55320
        [junit4] 2> 572066 T948 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 572066 T948 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 572066 T948 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-005
        [junit4] 2> 572066 T948 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-005/'
        [junit4] 2> 572097 T948 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-005/solr.xml
        [junit4] 2> 572140 T948 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-005
        [junit4] 2> 572141 T948 oasc.CoreContainer.<init> New CoreContainer 1866718818
        [junit4] 2> 572141 T948 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-005/]
        [junit4] 2> 572142 T948 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 572142 T948 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 572142 T948 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 572143 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 572143 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 572143 T948 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 572143 T948 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 572143 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 572144 T948 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 572144 T948 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 572146 T948 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 572148 T948 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 572149 T948 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 572149 T948 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 572149 T948 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58321/solr
        [junit4] 2> 572149 T948 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 572150 T948 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 572150 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 572152 T1027 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@58784167 name:ZooKeeperConnection Watcher:127.0.0.1:58321 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 572153 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 572153 T948 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 572154 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 572155 T1029 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2658e7dd name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 572155 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 572159 T948 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 573163 T948 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55320_hwaws
        [junit4] 2> 573164 T948 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55320_hwaws
        [junit4] 2> 573167 T948 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 573172 T948 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-005
        [junit4] 2> 573342 T948 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-005/conf/
        [junit4] 2> 574298 T948 oasc.CorePropertiesLocator.discover Found 1 core definitions
        [junit4] 2> 574300 T948 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 574300 T948 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 575987 T948 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
        [junit4] 2> 575988 T948 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 575996 T948 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
        [junit4] 2> 575999 T948 oejs.AbstractConnector.doStart Started sslsocketconnector@127.0.0.1:17772
        [junit4] 2> 576003 T948 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 576003 T948 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 576004 T948 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-006
        [junit4] 2> 576004 T948 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-006/'
        [junit4] 2> 576064 T948 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-006/solr.xml
        [junit4] 2> 576150 T948 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-006
        [junit4] 2> 576151 T948 oasc.CoreContainer.<init> New CoreContainer 1462154475
        [junit4] 2> 576151 T948 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-006/]
        [junit4] 2> 576153 T948 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 576153 T948 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 576154 T948 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 576154 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 576154 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 576155 T948 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 576155 T948 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 576155 T948 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 576156 T948 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 576156 T948 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 576160 T948 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 576163 T948 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 576163 T948 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 576163 T948 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 576164 T948 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58321/solr
        [junit4] 2> 576164 T948 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 576165 T948 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 576165 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 576167 T1042 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@28654636 name:ZooKeeperConnection Watcher:127.0.0.1:58321 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 576168 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 576168 T948 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 576170 T948 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 576171 T1044 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f18147c name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 576172 T948 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 576177 T948 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 577182 T948 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:17772_hwaws
        [junit4] 2> 577183 T948 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:17772_hwaws
        [junit4] 2> 577186 T948 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 577191 T948 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-006
        [junit4] 2> 578006 T948 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-006/conf/
        [junit4] 2> 578153 T948 oasc.CorePropertiesLocator.discover Found 1 core definitions
        [junit4] 2> 578154 T948 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 578155 T948 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 578476 T991 oasha.CollectionsHandler.handleCreateAction Creating Collection : wt=javabin&version=2&router.field=myOwnField&collection.configName=conf1&replicationFactor=2&action=CREATE&numShards=2&name=solrj_collection
        [junit4] 2> 578478 T985 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 578479 T978 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
        [junit4] 2> "operation":"create",
        [junit4] 2> "fromApi":"true",
        [junit4] 2> "name":"solrj_collection",
        [junit4] 2> "replicationFactor":"2",
        [junit4] 2> "collection.configName":"conf1",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "router.field":"myOwnField"}
        [junit4] 2> 578480 T1048 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : create , {
        [junit4] 2> "operation":"create",
        [junit4] 2> "fromApi":"true",
        [junit4] 2> "name":"solrj_collection",
        [junit4] 2> "replicationFactor":"2",
        [junit4] 2> "collection.configName":"conf1",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "router.field":"myOwnField"}
        [junit4] 2> 578481 T1048 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection
        [junit4] 2> 578482 T1048 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
        [junit4] 2> 578485 T985 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 578487 T977 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
        [junit4] 2> 578487 T977 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
        [junit4] 2> 578488 T977 oasc.Overseer$ClusterStateUpdater.createCollection state version solrj_collection 1
        [junit4] 2> 578490 T1045 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 578490 T1030 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 578490 T1000 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 578490 T985 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 578490 T1015 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 578490 T986 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 578585 T1048 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection solrj_collection, shardNames [shard1, shard2] , replicationFactor : 2
        [junit4] 2> 578586 T1048 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:55320_hwaws
        [junit4] 2> 578587 T1048 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:45232_hwaws
        [junit4] 2> 578589 T1048 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:49224_hwaws
        [junit4] 2> 578590 T1048 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:17772_hwaws
        [junit4] 2> 578744 T1019 oasha.CoreAdminHandler.handleCreateAction core create command wt=javabin&qt=/admin/cores&shard=shard1&version=2&collection.configName=conf1&collection=solrj_collection&action=CREATE&numShards=2&name=solrj_collection_shard1_replica1
        [junit4] 2> 578746 T967 oasha.CoreAdminHandler.handleCreateAction core create command wt=javabin&qt=/admin/cores&shard=shard1&version=2&collection.configName=conf1&collection=solrj_collection&action=CREATE&numShards=2&name=solrj_collection_shard1_replica2
        [junit4] 2> 578746 T989 oasha.CoreAdminHandler.handleCreateAction core create command wt=javabin&qt=/admin/cores&shard=shard2&version=2&collection.configName=conf1&collection=solrj_collection&action=CREATE&numShards=2&name=solrj_collection_shard2_replica1
        [junit4] 2> 578748 T1019 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down collection=solrj_collection
        [junit4] 2> 578749 T1034 oasha.CoreAdminHandler.handleCreateAction core create command wt=javabin&qt=/admin/cores&shard=shard2&version=2&collection.configName=conf1&collection=solrj_collection&action=CREATE&numShards=2&name=solrj_collection_shard2_replica2
        [junit4] 2> 578750 T985 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 578750 T1019 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 578751 T967 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down collection=solrj_collection
        [junit4] 2> 578751 T989 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down collection=solrj_collection
        [junit4] 2> 578752 T967 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 578752 T989 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 578753 T1034 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down collection=solrj_collection
        [junit4] 2> 578753 T977 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "operation":"state",
        [junit4] 2> "base_url":"https://127.0.0.1:55320/hwaws",
        [junit4] 2> "shard":"shard1",
        [junit4] 2> "core":"solrj_collection_shard1_replica1",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "node_name":"127.0.0.1:55320_hwaws",
        [junit4] 2> "state":"down",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "roles":null}
        [junit4] 2> 578754 T1034 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 578756 T977 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "operation":"state",
        [junit4] 2> "base_url":"https://127.0.0.1:45232/hwaws",
        [junit4] 2> "shard":"shard1",
        [junit4] 2> "core":"solrj_collection_shard1_replica2",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "node_name":"127.0.0.1:45232_hwaws",
        [junit4] 2> "state":"down",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "roles":null}
        [junit4] 2> 578758 T985 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 578759 T977 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "operation":"state",
        [junit4] 2> "base_url":"https://127.0.0.1:49224/hwaws",
        [junit4] 2> "shard":"shard2",
        [junit4] 2> "core":"solrj_collection_shard2_replica1",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "node_name":"127.0.0.1:49224_hwaws",
        [junit4] 2> "state":"down",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "roles":null}
        [junit4] 2> 578762 T977 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "operation":"state",
        [junit4] 2> "base_url":"https://127.0.0.1:17772/hwaws",
        [junit4] 2> "shard":"shard2",
        [junit4] 2> "core":"solrj_collection_shard2_replica2",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "node_name":"127.0.0.1:17772_hwaws",
        [junit4] 2> "state":"down",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "roles":null}
        [junit4] 2> 578764 T985 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 578765 T1015 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 578765 T1000 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 578765 T1030 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 578765 T985 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 578765 T986 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 578765 T1045 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 579750 T1019 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
        [junit4] 2> 579751 T1019 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 579752 T1019 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 579752 T1019 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 579753 T1019 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 579753 T1019 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-005/solrj_collection_shard1_replica1/'
        [junit4] 2> 579753 T989 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
        [junit4] 2> 579753 T967 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
        [junit4] 2> 579754 T989 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 579754 T967 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 579755 T989 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 579755 T1034 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
        [junit4] 2> 579755 T967 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 579756 T1034 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 579756 T989 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 579757 T1034 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 579756 T967 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 579757 T989 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 579757 T1034 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 579758 T967 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 579758 T989 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003/solrj_collection_shard2_replica1/'
        [junit4] 2> 579759 T1034 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 579759 T967 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002/solrj_collection_shard1_replica2/'
        [junit4] 2> 579760 T1034 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-006/solrj_collection_shard2_replica2/'
        [junit4] 2> 579936 T1034 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 579937 T989 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 579938 T967 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 579938 T1019 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 580278 T1019 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 580279 T1034 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 580279 T989 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 580280 T967 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 580281 T967 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 580294 T967 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
        [junit4] 2> 580379 T1019 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 580380 T1034 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 580380 T989 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 580392 T1019 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
        [junit4] 2> 580394 T1034 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
        [junit4] 2> 580394 T989 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
        [junit4] 2> 580820 T967 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 580836 T967 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 580840 T967 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 580855 T967 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 580867 T967 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 580874 T967 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 580875 T967 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 580877 T967 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 580877 T967 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 580878 T967 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 580879 T967 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 580879 T967 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using configuration from collection solrj_collection
        [junit4] 2> 580879 T967 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 580880 T967 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002/solrj_collection_shard1_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002/solrj_collection_shard1_replica2/data/
        [junit4] 2> 580880 T967 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@52be3108
        [junit4] 2> 580882 T967 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002/solrj_collection_shard1_replica2/data
        [junit4] 2> 580882 T967 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002/solrj_collection_shard1_replica2/data/index/
        [junit4] 2> 580883 T967 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
        [junit4] 2> 580883 T967 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-002/solrj_collection_shard1_replica2/data/index
        [junit4] 2> 580884 T967 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=24, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.23115559635087435]
        [junit4] 2> 580885 T967 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory(id=lucene-648b9be7); maxCacheMB=0.1875 maxMergeSizeMB=0.2431640625)),segFN=segments_1,generation=1}
        [junit4] 2> 580886 T967 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 580891 T967 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 580892 T967 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 580893 T967 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 580893 T967 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 580894 T967 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 580894 T967 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 580895 T967 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 580896 T967 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 580896 T967 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 580897 T967 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 580898 T967 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 580899 T967 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 580899 T967 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 580900 T967 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 580900 T967 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 580901 T967 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 580901 T967 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 580902 T967 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
        [junit4] 2> 580903 T967 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
        [junit4] 2> 580903 T967 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
        [junit4] 2> 580934 T967 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 580937 T967 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 580940 T967 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 580943 T967 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 580949 T1034 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 580950 T989 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 580951 T967 oasu.CommitTracker.<init> Hard AutoCommit: disabled
        [junit4] 2> 580951 T967 oasu.CommitTracker.<init> Soft AutoCommit: disabled
        [junit4] 2> 580952 T967 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=30.1533203125, floorSegmentMB=2.1123046875, forceMergeDeletesPctAllowed=16.79591377444389, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.536570311243519
        [junit4] 2> 580953 T967 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory(id=lucene-648b9be7); maxCacheMB=0.1875 maxMergeSizeMB=0.2431640625)),segFN=segments_1,generation=1}
        [junit4] 2> 580953 T967 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 580953 T1019 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 580954 T967 oass.SolrIndexSearcher.<init> Opening Searcher@6ae903b2[solrj_collection_shard1_replica2] main
        [junit4] 2> 580954 T967 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 580955 T967 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 580956 T967 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
        [junit4] 2> 580956 T967 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
        [junit4] 2> 580957 T967 oasr.RestManager.init Initializing RestManager with initArgs: {}
        [junit4] 2> 580957 T967 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
        [junit4] 2> 580958 T967 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
        [junit4] 2> 580959 T967 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
        [junit4] 2> 580959 T1034 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 580960 T989 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 580960 T967 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json
        [junit4] 2> 580963 T967 oasr.ManagedResourceStorage$ZooKeeperStorageIO$1.close Wrote 38 bytes to new znode /configs/conf1/_rest_managed.json
        [junit4] 2> 580963 T1019 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 580964 T1034 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 580964 T989 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 580963 T967 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
        [junit4] 2> 580965 T967 oasr.RestManager.init Initializing 0 registered ManagedResources
        [junit4] 2> 580967 T1019 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 580971 T1057 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica2] Registered new searcher Searcher@6ae903b2[solrj_collection_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
        [junit4] 2> 580976 T967 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica2
        [junit4] 2> 580977 T967 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica2 address:https://127.0.0.1:45232/hwaws collection:solrj_collection shard:shard1
        [junit4] 2> 580978 T967 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
        [junit4] 2> 580980 T989 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 580980 T1034 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 580984 T967 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
        [junit4] 2> 580985 T1019 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 580986 T985 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 580986 T967 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=7.8550313E15ms
        [junit4] 2> 580986 T989 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 580986 T1034 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 580991 T1019 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 580992 T1000 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 580992 T989 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 580992 T1045 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 580992 T1030 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 580992 T985 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 580992 T1015 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 580992 T986 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 580994 T989 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 580995 T989 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 580995 T989 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 580993 T1034 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 580996 T989 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 580997 T989 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 580998 T1034 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 580998 T989 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using configuration from collection solrj_collection
        [junit4] 2> 580998 T1019 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 580999 T989 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 580998 T1034 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 581000 T989 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003/solrj_collection_shard2_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003/solrj_collection_shard2_replica1/data/
        [junit4] 2> 581000 T1019 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 581000 T989 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@52be3108
        [junit4] 2> 581000 T1034 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 581001 T1019 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 581002 T1019 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 581002 T1034 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 581002 T989 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003/solrj_collection_shard2_replica1/data
        [junit4] 2> 581003 T1019 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 581003 T1034 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 581004 T1019 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 581003 T989 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003/solrj_collection_shard2_replica1/data/index/
        [junit4] 2> 581005 T1019 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using configuration from collection solrj_collection
        [junit4] 2> 581004 T1034 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using configuration from collection solrj_collection
        [junit4] 2> 581005 T1019 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 581005 T989 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003/solrj_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
        [junit4] 2> 581006 T1019 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-005/solrj_collection_shard1_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-005/solrj_collection_shard1_replica1/data/
        [junit4] 2> 581007 T989 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001/tempDir-003/solrj_collection_shard2_replica1/data/index
        [junit4] 2> 581006 T1034 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 581007 T989 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=24, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.23115559635087435]
        [junit4] 2> 581007 T1019 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@52be3108
        [junit4] 2> 581008 T1034 oasc.SolrCore.<init> [solrj_collection_shard2_replic

    [...truncated too long message...]

      org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1635)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
        [junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1037)Throwable #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1844, name=Thread-566, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
        [junit4] > Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Could not fully create collection: awholynewstresscollection_collection3_3
        [junit4] > at __randomizedtesting.SeedInfo.seed([3B60B290ACF4DF85]:0)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1635)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
        [junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1037)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1843, name=Thread-565, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
        [junit4] > Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Could not fully create collection: awholynewstresscollection_collection2_3
        [junit4] > at __randomizedtesting.SeedInfo.seed([3B60B290ACF4DF85]:0)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1635)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
        [junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1037)Throwable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1846, name=Thread-568, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
        [junit4] > Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Could not fully create collection: awholynewstresscollection_collection5_3
        [junit4] > at __randomizedtesting.SeedInfo.seed([3B60B290ACF4DF85]:0)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1635)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
        [junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1037)Throwable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1842, name=Thread-564, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
        [junit4] > Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Could not fully create collection: awholynewstresscollection_collection1_3
        [junit4] > at __randomizedtesting.SeedInfo.seed([3B60B290ACF4DF85]:0)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1635)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
        [junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1037)Throwable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1841, name=Thread-563, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
        [junit4] > Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Could not fully create collection: awholynewstresscollection_collection0_3
        [junit4] > at __randomizedtesting.SeedInfo.seed([3B60B290ACF4DF85]:0)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1635)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
        [junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1037)
        [junit4] 2> 1375559 T948 oas.SolrTestCaseJ4.deleteCore ###deleteCore
        [junit4] 2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-3B60B290ACF4DF85-001
        [junit4] 2> 814039 T947 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
        [junit4] 2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=FST41, a_t=FST41, a_i1=FST41, _version_=FST41, range_facet_l=FST41, timestamp=FST41, multiDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene41(blocksize=128))), text=Lucene41(blocksize=128), other_tl1=FST41, id=PostingsFormat(name=Direct)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=es, timezone=Atlantic/Faroe
        [junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=105790968,total=352321536
        [junit4] 2> NOTE: All tests run in this JVM: [PluginInfoTest, SpellPossibilityIteratorTest, TestDynamicFieldCollectionResource, TestSearchPerf, StatelessScriptUpdateProcessorFactoryTest, AsyncMigrateRouteKeyTest, TestConfigSets, TestLMJelinekMercerSimilarityFactory, TestSchemaNameResource, TestDocSet, ZkCLITest, TestIndexSearcher, FileBasedSpellCheckerTest, SuggesterTSTTest, DocumentAnalysisRequestHandlerTest, SolrIndexSplitterTest, TestBlendedInfixSuggestions, TermVectorComponentTest, TestChildDocTransformer, TestJoin, FieldMutatingUpdateProcessorTest, ResponseLogComponentTest, TestModifyConfFiles, DocExpirationUpdateProcessorFactoryTest, TestSort, TestImplicitCoreProperties, FieldAnalysisRequestHandlerTest, UUIDFieldTest, TestNonDefinedSimilarityFactory, TestFastLRUCache, LeaderFailoverAfterPartitionTest, CollectionsAPIDistributedZkTest]
        [junit4] Completed on J3 in 814.33s, 1 test, 1 error <<< FAILURES!

    [...truncated 1005 lines...]
    BUILD FAILED
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:531: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:472: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:189: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:496: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1359: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:966: There were test failures: 437 suites, 1798 tests, 1 error, 54 ignored (9 assumptions)

    Total time: 229 minutes 29 seconds
    Build step 'Invoke Ant' marked build as failure
    Archiving artifacts
    Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #654
    Archived 5 artifacts
    Archive block size is 32768
    Received 0 blocks and 698441186 bytes
    Compression is 0.0%
    Took 1 min 59 sec
    Recording test results
    Email was triggered for: Failure
    Sending email for trigger: Failure
  • Apache Jenkins Server at Oct 25, 2014 at 6:09 am
    Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/658/

    3 tests failed.
    REGRESSION: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.testDistribSearch

    Error Message:
    There were too many update fails - we expect it can happen, but shouldn't easily

    Stack Trace:
    java.lang.AssertionError: There were too many update fails - we expect it can happen, but shouldn't easily
      at __randomizedtesting.SeedInfo.seed([7623E1151B611AA4:F7C56F0D6C3E7A98]:0)
      at org.junit.Assert.fail(Assert.java:93)
      at org.junit.Assert.assertTrue(Assert.java:43)
      at org.junit.Assert.assertFalse(Assert.java:68)
      at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.doTest(ChaosMonkeyNothingIsSafeTest.java:223)
      at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:606)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at java.lang.Thread.run(Thread.java:745)


    REGRESSION: org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.testDistribSearch

    Error Message:
    No live SolrServers available to handle this request:[http://127.0.0.1:11571/blx, http://127.0.0.1:33031/blx, http://127.0.0.1:11248/blx]

    Stack Trace:
    org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:11571/blx, http://127.0.0.1:33031/blx, http://127.0.0.1:11248/blx]
      at __randomizedtesting.SeedInfo.seed([7623E1151B611AA4:F7C56F0D6C3E7A98]:0)
      at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:333)
      at org.apache.solr.client.solrj.impl.CloudSolrServer.sendRequest(CloudSolrServer.java:1015)
      at org.apache.solr.client.solrj.impl.CloudSolrServer.requestWithRetryOnStaleState(CloudSolrServer.java:793)
      at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:736)
      at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.removeAndWaitForLastReplicaGone(DeleteLastCustomShardedReplicaTest.java:117)
      at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.doTest(DeleteLastCustomShardedReplicaTest.java:107)
      at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:606)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at java.lang.Thread.run(Thread.java:745)
    Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Could not remove replica : customcollreplicadeletion/a/core_node2
      at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
      at org.apache.solr.client.solrj.impl.LBHttpSolrServer.doRequest(LBHttpSolrServer.java:354)
      at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:313)
      ... 46 more


    REGRESSION: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testDistribSearch

    Error Message:
    IOException occured when talking to server at: http://127.0.0.1:55246/collection1

    Stack Trace:
    org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:55246/collection1
      at __randomizedtesting.SeedInfo.seed([7623E1151B611AA4:F7C56F0D6C3E7A98]:0)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:583)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
      at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
      at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
      at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:223)
      at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:165)
      at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrServer(FullSolrCloudDistribCmdsTest.java:414)
      at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.doTest(FullSolrCloudDistribCmdsTest.java:144)
      at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:606)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at java.lang.Thread.run(Thread.java:745)
    Caused by: java.net.SocketException: Broken pipe
      at java.net.SocketOutputStream.socketWrite0(Native Method)
      at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113)
      at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
      at org.apache.http.impl.io.AbstractSessionOutputBuffer.flushBuffer(AbstractSessionOutputBuffer.java:159)
      at org.apache.http.impl.io.AbstractSessionOutputBuffer.writeLine(AbstractSessionOutputBuffer.java:258)
      at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:59)
      at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:46)
      at org.apache.http.impl.io.AbstractMessageWriter.write(AbstractMessageWriter.java:109)
      at org.apache.http.impl.AbstractHttpClientConnection.sendRequestHeader(AbstractHttpClientConnection.java:254)
      at org.apache.http.impl.conn.DefaultClientConnection.sendRequestHeader(DefaultClientConnection.java:270)
      at org.apache.http.impl.conn.ManagedClientConnectionImpl.sendRequestHeader(ManagedClientConnectionImpl.java:209)
      at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:203)
      at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:121)
      at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
      at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
      at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
      at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
      at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
      at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
      at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:465)
      ... 49 more




    Build Log:
    [...truncated 11911 lines...]
        [junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
        [junit4] 2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/init-core-data-001
        [junit4] 2> 248437 T608 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
        [junit4] 2> 248443 T608 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
        [junit4] 2> 248444 T608 oasc.ZkTestServer.run STARTING ZK TEST SERVER
        [junit4] 1> client port:0.0.0.0/0.0.0.0:0
        [junit4] 2> 248445 T609 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
        [junit4] 2> 248545 T608 oasc.ZkTestServer.run start zk server on port:55236
        [junit4] 2> 248546 T608 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 248547 T608 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 248551 T615 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a34c02c name:ZooKeeperConnection Watcher:127.0.0.1:55236 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 248551 T608 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 248552 T608 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 248552 T608 oascc.SolrZkClient.makePath makePath: /solr
        [junit4] 2> 248555 T608 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 248557 T608 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 248558 T617 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c091303 name:ZooKeeperConnection Watcher:127.0.0.1:55236/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 248559 T608 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 248559 T608 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 248559 T608 oascc.SolrZkClient.makePath makePath: /collections/collection1
        [junit4] 2> 248563 T608 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
        [junit4] 2> 248566 T608 oascc.SolrZkClient.makePath makePath: /collections/control_collection
        [junit4] 2> 248568 T608 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
        [junit4] 2> 248572 T608 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
        [junit4] 2> 248573 T608 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
        [junit4] 2> 248578 T608 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
        [junit4] 2> 248578 T608 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
        [junit4] 2> 248682 T608 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 248683 T608 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 248687 T608 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
        [junit4] 2> 248687 T608 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
        [junit4] 2> 248691 T608 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
        [junit4] 2> 248691 T608 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
        [junit4] 2> 248695 T608 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
        [junit4] 2> 248695 T608 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
        [junit4] 2> 248699 T608 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
        [junit4] 2> 248700 T608 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
        [junit4] 2> 248703 T608 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
        [junit4] 2> 248704 T608 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
        [junit4] 2> 248707 T608 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
        [junit4] 2> 248708 T608 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
        [junit4] 2> 248711 T608 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
        [junit4] 2> 248712 T608 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
        [junit4] 2> 248715 T608 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
        [junit4] 2> 248716 T608 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
        [junit4] 2> 250004 T608 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 250008 T608 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:55246
        [junit4] 2> 250008 T608 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 250009 T608 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 250009 T608 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-002
        [junit4] 2> 250009 T608 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-002/'
        [junit4] 2> 250043 T608 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-002/solr.xml
        [junit4] 2> 250127 T608 oasc.CoreContainer.<init> New CoreContainer 1814825197
        [junit4] 2> 250127 T608 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-002/]
        [junit4] 2> 250128 T608 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 250128 T608 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 250129 T608 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 250129 T608 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 250129 T608 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 250129 T608 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 250129 T608 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 250130 T608 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 250130 T608 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 250130 T608 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 250131 T608 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 250131 T608 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 250131 T608 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 250131 T608 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 250132 T608 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55236/solr
        [junit4] 2> 250132 T608 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 250132 T608 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 250133 T608 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 250135 T628 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3efe3716 name:ZooKeeperConnection Watcher:127.0.0.1:55236 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 250135 T608 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 250136 T608 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 250138 T608 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 250139 T630 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2d01f7d9 name:ZooKeeperConnection Watcher:127.0.0.1:55236/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 250140 T608 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 250141 T608 oascc.SolrZkClient.makePath makePath: /overseer/queue
        [junit4] 2> 250145 T608 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
        [junit4] 2> 250148 T608 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
        [junit4] 2> 250151 T608 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
        [junit4] 2> 250155 T608 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
        [junit4] 2> 250158 T608 oascc.SolrZkClient.makePath makePath: /live_nodes
        [junit4] 2> 250160 T608 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55246_
        [junit4] 2> 250161 T608 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55246_
        [junit4] 2> 250164 T608 oascc.SolrZkClient.makePath makePath: /overseer_elect
        [junit4] 2> 250167 T608 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
        [junit4] 2> 250169 T608 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 250171 T608 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:55246_
        [junit4] 2> 250172 T608 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
        [junit4] 2> 250174 T608 oasc.Overseer.start Overseer (id=92681822473289731-127.0.0.1:55246_-n_0000000000) starting
        [junit4] 2> 250177 T608 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
        [junit4] 2> 250188 T608 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
        [junit4] 2> 250189 T632 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
        [junit4] 2> 250191 T608 oascc.SolrZkClient.makePath makePath: /clusterstate.json
        [junit4] 2> 250193 T608 oascc.SolrZkClient.makePath makePath: /aliases.json
        [junit4] 2> 250195 T608 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 250200 T631 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
        [junit4] 2> 250207 T634 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
        [junit4] 2> 250207 T634 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
        [junit4] 2> 250208 T634 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 250208 T635 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 250212 T631 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
        [junit4] 2> "numShards":"1",
        [junit4] 2> "state":"down",
        [junit4] 2> "collection":"control_collection",
        [junit4] 2> "roles":null,
        [junit4] 2> "shard":null,
        [junit4] 2> "operation":"state",
        [junit4] 2> "base_url":"http://127.0.0.1:55246",
        [junit4] 2> "node_name":"127.0.0.1:55246_",
        [junit4] 2> "core":"collection1"}
        [junit4] 2> 250213 T631 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
        [junit4] 2> 250213 T631 oasc.Overseer$ClusterStateUpdater.createCollection state version control_collection 1
        [junit4] 2> 250213 T631 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
        [junit4] 2> 250216 T635 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 251209 T634 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
        [junit4] 2> 251210 T634 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
        [junit4] 2> 251211 T634 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 251211 T634 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
        [junit4] 2> 251212 T634 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 251212 T634 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-002/collection1/'
        [junit4] 2> 251215 T634 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-002/collection1/lib/.svn/' to classloader
        [junit4] 2> 251216 T634 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-002/collection1/lib/classes/' to classloader
        [junit4] 2> 251216 T634 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-002/collection1/lib/README' to classloader
        [junit4] 2> 251284 T634 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 251332 T634 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 251434 T634 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 251441 T634 oass.IndexSchema.readSchema [collection1] Schema name=test
        [junit4] 2> 251700 T634 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 251702 T634 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 251703 T634 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 251708 T634 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 251737 T634 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
        [junit4] 2> 251737 T634 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 251737 T634 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/control/data/
        [junit4] 2> 251738 T634 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@72b69ba3
        [junit4] 2> 251738 T634 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/control/data
        [junit4] 2> 251739 T634 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/control/data/index/
        [junit4] 2> 251739 T634 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
        [junit4] 2> 251739 T634 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/control/data/index
        [junit4] 2> 251740 T634 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=19, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4795777660978364]
        [junit4] 2> 251740 T634 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-3bc26253)),segFN=segments_1,generation=1}
        [junit4] 2> 251741 T634 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 251744 T634 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 251745 T634 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 251745 T634 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 251745 T634 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 251745 T634 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 251746 T634 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 251746 T634 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 251746 T634 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 251746 T634 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 251747 T634 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 251747 T634 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 251748 T634 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 251748 T634 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 251748 T634 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 251749 T634 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 251749 T634 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 251749 T634 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 251749 T634 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
        [junit4] 2> 251750 T634 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
        [junit4] 2> 251750 T634 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
        [junit4] 2> 251764 T634 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 251766 T634 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 251767 T634 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 251769 T634 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 251773 T634 oasu.CommitTracker.<init> Hard AutoCommit: disabled
        [junit4] 2> 251773 T634 oasu.CommitTracker.<init> Soft AutoCommit: disabled
        [junit4] 2> 251774 T634 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=1.1923828125, floorSegmentMB=1.828125, forceMergeDeletesPctAllowed=11.572364167646583, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8668722100270879
        [junit4] 2> 251774 T634 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-3bc26253)),segFN=segments_1,generation=1}
        [junit4] 2> 251775 T634 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 251775 T634 oass.SolrIndexSearcher.<init> Opening Searcher@4ee7d550[collection1] main
        [junit4] 2> 251775 T634 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
        [junit4] 2> 251776 T634 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 251776 T634 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
        [junit4] 2> 251777 T634 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
        [junit4] 2> 251777 T634 oasr.RestManager.init Initializing RestManager with initArgs: {}
        [junit4] 2> 251778 T634 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
        [junit4] 2> 251778 T634 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
        [junit4] 2> 251778 T634 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
        [junit4] 2> 251779 T634 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json
        [junit4] 2> 251782 T634 oasr.ManagedResourceStorage$ZooKeeperStorageIO$1.close Wrote 38 bytes to new znode /configs/conf1/_rest_managed.json
        [junit4] 2> 251782 T634 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
        [junit4] 2> 251782 T634 oasr.RestManager.init Initializing 0 registered ManagedResources
        [junit4] 2> 251785 T636 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4ee7d550[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
        [junit4] 2> 251787 T634 oasc.CoreContainer.registerCore registering core: collection1
        [junit4] 2> 251787 T639 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:55246 collection:control_collection shard:shard1
        [junit4] 2> 251788 T608 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
        [junit4] 2> 251788 T608 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 251789 T639 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
        [junit4] 2> 251789 T608 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 251790 T608 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 251791 T641 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@24b4b355 name:ZooKeeperConnection Watcher:127.0.0.1:55236/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 251791 T608 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 251792 T608 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 251793 T608 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 251796 T639 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
        [junit4] 2> 251798 T608 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
        [junit4] 2> 251798 T608 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
        [junit4] 2> 251798 T608 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 251798 T635 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 251799 T608 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 251799 T639 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
        [junit4] 2> 251799 T639 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
        [junit4] 2> ASYNC NEW_CORE C115 name=collection1 org.apache.solr.core.SolrCore@4c059601 url=http://127.0.0.1:55246/collection1 node=127.0.0.1:55246_ C115_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:55246, node_name=127.0.0.1:55246_, core=collection1}
        [junit4] 2> 251799 T639 C115 P55246 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:55246/collection1/
        [junit4] 2> 251800 T639 C115 P55246 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
        [junit4] 2> 251800 T639 C115 P55246 oasc.SyncStrategy.syncToMe http://127.0.0.1:55246/collection1/ has no replicas
        [junit4] 2> 251800 T639 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:55246/collection1/ shard1
        [junit4] 2> 251800 T639 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
        [junit4] 2> 251800 T643 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a10e529 name:ZooKeeperConnection Watcher:127.0.0.1:55236/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 251801 T608 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 251801 T608 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 251808 T644 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 251808 T635 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 251816 T631 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: collection1
        [junit4] 2> 251816 T631 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
        [junit4] 2> 251816 T631 oasc.Overseer$ClusterStateUpdater.createCollection state version collection1 2
        [junit4] 2> 251819 T635 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 251820 T631 oasc.Overseer$ClusterStateUpdater.updateZkStates going to create_collection /collections/collection1/state.json
        [junit4] 2> 251821 T635 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 251821 T644 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 251831 T635 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 251934 T635 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 251934 T644 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 251962 T639 oasc.ZkController.register We are http://127.0.0.1:55246/collection1/ and leader is http://127.0.0.1:55246/collection1/
        [junit4] 2> 251963 T639 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:55246
        [junit4] 2> 251963 T639 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
        [junit4] 2> 251963 T639 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
        [junit4] 2> 251963 T639 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
        [junit4] 2> 251965 T635 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 251971 T631 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
        [junit4] 2> "core_node_name":"core_node1",
        [junit4] 2> "numShards":"3",
        [junit4] 2> "state":"active",
        [junit4] 2> "collection":"control_collection",
        [junit4] 2> "roles":null,
        [junit4] 2> "shard":"shard1",
        [junit4] 2> "operation":"state",
        [junit4] 2> "base_url":"http://127.0.0.1:55246",
        [junit4] 2> "node_name":"127.0.0.1:55246_",
        [junit4] 2> "core":"collection1"}
        [junit4] 2> 252076 T635 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 252076 T644 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 253543 T608 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
        [junit4] 2> 253544 T608 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 253547 T608 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:55285
        [junit4] 2> 253548 T608 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 253548 T608 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 253548 T608 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-003
        [junit4] 2> 253549 T608 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-003/'
        [junit4] 2> 253609 T608 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-003/solr.xml
        [junit4] 2> 253716 T608 oasc.CoreContainer.<init> New CoreContainer 473449927
        [junit4] 2> 253716 T608 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-003/]
        [junit4] 2> 253718 T608 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 253718 T608 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 253718 T608 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 253719 T608 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 253719 T608 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 253720 T608 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 253720 T608 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 253720 T608 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 253721 T608 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 253721 T608 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 253722 T608 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 253722 T608 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 253723 T608 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 253723 T608 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 253723 T608 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55236/solr
        [junit4] 2> 253724 T608 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 253724 T608 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 253725 T608 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 253727 T655 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@326d1a4a name:ZooKeeperConnection Watcher:127.0.0.1:55236 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 253728 T608 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 253728 T608 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 253730 T608 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 253732 T657 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ac4ced4 name:ZooKeeperConnection Watcher:127.0.0.1:55236/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 253732 T608 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 253740 T608 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 254748 T608 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55285_
        [junit4] 2> 254749 T608 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55285_
        [junit4] 2> 254754 T608 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 254763 T659 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
        [junit4] 2> 254763 T659 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
        [junit4] 2> 254765 T635 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 254766 T659 oasc.ZkController.preRegister Registering watch for external collection collection1
        [junit4] 2> 254766 T659 oascc.ZkStateReader.addZkWatch addZkWatch collection1
        [junit4] 2> 254767 T659 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper...
        [junit4] 2> 254769 T659 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0
        [junit4] 2> 254769 T659 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 254769 T631 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
        [junit4] 2> "numShards":"3",
        [junit4] 2> "state":"down",
        [junit4] 2> "collection":"collection1",
        [junit4] 2> "roles":null,
        [junit4] 2> "shard":null,
        [junit4] 2> "operation":"state",
        [junit4] 2> "base_url":"http://127.0.0.1:55285",
        [junit4] 2> "node_name":"127.0.0.1:55285_",
        [junit4] 2> "core":"collection1"}
        [junit4] 2> 254773 T631 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
        [junit4] 2> 254774 T631 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
        [junit4] 2> 254781 T631 oasc.Overseer$ClusterStateUpdater.updateZkStates going to update_collection /collections/collection1/state.json
        [junit4] 2> 254782 T658 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating...
        [junit4] 2> 254783 T658 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1
        [junit4] 2> 255769 T659 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
        [junit4] 2> 255770 T659 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
        [junit4] 2> 255771 T659 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 255771 T659 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
        [junit4] 2> 255772 T659 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 255773 T659 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-003/collection1/'
        [junit4] 2> 255775 T659 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-003/collection1/lib/README' to classloader
        [junit4] 2> 255776 T659 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-003/collection1/lib/.svn/' to classloader
        [junit4] 2> 255776 T659 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-003/collection1/lib/classes/' to classloader
        [junit4] 2> 255842 T659 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 255889 T659 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 255990 T659 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 256000 T659 oass.IndexSchema.readSchema [collection1] Schema name=test
        [junit4] 2> 256336 T659 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 256338 T659 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 256341 T659 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 256350 T659 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 256385 T659 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
        [junit4] 2> 256386 T659 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 256386 T659 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-003/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/jetty1/
        [junit4] 2> 256387 T659 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@72b69ba3
        [junit4] 2> 256388 T659 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/jetty1
        [junit4] 2> 256388 T659 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/jetty1/index/
        [junit4] 2> 256388 T659 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
        [junit4] 2> 256389 T659 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/jetty1/index
        [junit4] 2> 256390 T659 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=19, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4795777660978364]
        [junit4] 2> 256391 T659 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-4399f154)),segFN=segments_1,generation=1}
        [junit4] 2> 256391 T659 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 256397 T659 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 256398 T659 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 256398 T659 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 256399 T659 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 256399 T659 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 256399 T659 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 256400 T659 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 256400 T659 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 256400 T659 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 256401 T659 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 256402 T659 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 256402 T659 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 256403 T659 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 256404 T659 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 256404 T659 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 256404 T659 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 256405 T659 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 256405 T659 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
        [junit4] 2> 256406 T659 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
        [junit4] 2> 256407 T659 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
        [junit4] 2> 256426 T659 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 256428 T659 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 256430 T659 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 256432 T659 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 256437 T659 oasu.CommitTracker.<init> Hard AutoCommit: disabled
        [junit4] 2> 256437 T659 oasu.CommitTracker.<init> Soft AutoCommit: disabled
        [junit4] 2> 256438 T659 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=1.1923828125, floorSegmentMB=1.828125, forceMergeDeletesPctAllowed=11.572364167646583, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8668722100270879
        [junit4] 2> 256439 T659 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-4399f154)),segFN=segments_1,generation=1}
        [junit4] 2> 256439 T659 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 256440 T659 oass.SolrIndexSearcher.<init> Opening Searcher@20ed4023[collection1] main
        [junit4] 2> 256440 T659 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
        [junit4] 2> 256441 T659 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 256442 T659 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
        [junit4] 2> 256442 T659 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
        [junit4] 2> 256442 T659 oasr.RestManager.init Initializing RestManager with initArgs: {}
        [junit4] 2> 256443 T659 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
        [junit4] 2> 256444 T659 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38 bytes from znode /configs/conf1/_rest_managed.json
        [junit4] 2> 256444 T659 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
        [junit4] 2> 256445 T659 oasr.ManagedResource.reloadFromStorage Loaded initArgs {} for /rest/managed
        [junit4] 2> 256445 T659 oasr.RestManager.init Initializing 0 registered ManagedResources
        [junit4] 2> 256450 T660 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@20ed4023[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
        [junit4] 2> 256454 T659 oasc.CoreContainer.registerCore registering core: collection1
        [junit4] 2> 256455 T663 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:55285 collection:collection1 shard:shard1
        [junit4] 2> 256456 T608 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
        [junit4] 2> 256456 T608 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 256456 T663 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
        [junit4] 2> 256463 T663 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
        [junit4] 2> 256466 T635 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 256466 T663 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
        [junit4] 2> 256467 T663 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
        [junit4] 2> ASYNC NEW_CORE C116 name=collection1 org.apache.solr.core.SolrCore@233730e0 url=http://127.0.0.1:55285/collection1 node=127.0.0.1:55285_ C116_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:55285, node_name=127.0.0.1:55285_, core=collection1}
        [junit4] 2> 256467 T663 C116 P55285 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:55285/collection1/
        [junit4] 2> 256467 T663 C116 P55285 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
        [junit4] 2> 256467 T663 C116 P55285 oasc.SyncStrategy.syncToMe http://127.0.0.1:55285/collection1/ has no replicas
        [junit4] 2> 256468 T663 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:55285/collection1/ shard1
        [junit4] 2> 256468 T663 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
        [junit4] 2> 256475 T631 oasc.Overseer$ClusterStateUpdater.updateZkStates going to update_collection /collections/collection1/state.json
        [junit4] 2> 256476 T658 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating...
        [junit4] 2> 256477 T658 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2
        [junit4] 2> 256486 T635 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 256487 T631 oasc.Overseer$ClusterStateUpdater.updateZkStates going to update_collection /collections/collection1/state.json
        [junit4] 2> 256487 T658 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating...
        [junit4] 2> 256488 T658 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3
        [junit4] 2> 256526 T663 oasc.ZkController.register We are http://127.0.0.1:55285/collection1/ and leader is http://127.0.0.1:55285/collection1/
        [junit4] 2> 256527 T663 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:55285
        [junit4] 2> 256527 T663 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
        [junit4] 2> 256527 T663 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
        [junit4] 2> 256527 T663 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
        [junit4] 2> 256529 T635 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 256533 T663 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3
        [junit4] 2> 256534 T631 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
        [junit4] 2> "core_node_name":"core_node1",
        [junit4] 2> "numShards":"3",
        [junit4] 2> "state":"active",
        [junit4] 2> "collection":"collection1",
        [junit4] 2> "roles":null,
        [junit4] 2> "shard":"shard1",
        [junit4] 2> "operation":"state",
        [junit4] 2> "base_url":"http://127.0.0.1:55285",
        [junit4] 2> "node_name":"127.0.0.1:55285_",
        [junit4] 2> "core":"collection1"}
        [junit4] 2> 256542 T631 oasc.Overseer$ClusterStateUpdater.updateZkStates going to update_collection /collections/collection1/state.json
        [junit4] 2> 256543 T658 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating...
        [junit4] 2> 256544 T658 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4
        [junit4] 2> 258395 T608 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
        [junit4] 2> 258397 T608 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 258400 T608 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:55303
        [junit4] 2> 258400 T608 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 258401 T608 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 258401 T608 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-004
        [junit4] 2> 258401 T608 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-004/'
        [junit4] 2> 258454 T608 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-004/solr.xml
        [junit4] 2> 258545 T608 oasc.CoreContainer.<init> New CoreContainer 1657780696
        [junit4] 2> 258545 T608 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-004/]
        [junit4] 2> 258547 T608 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 258547 T608 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 258547 T608 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 258548 T608 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 258548 T608 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 258548 T608 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 258549 T608 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 258549 T608 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 258550 T608 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 258550 T608 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 258551 T608 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 258551 T608 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 258552 T608 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 258552 T608 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 258552 T608 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55236/solr
        [junit4] 2> 258553 T608 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 258553 T608 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 258554 T608 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 258556 T674 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@33b24696 name:ZooKeeperConnection Watcher:127.0.0.1:55236 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 258557 T608 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 258558 T608 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 258561 T608 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 258563 T676 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@39e85f81 name:ZooKeeperConnection Watcher:127.0.0.1:55236/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 258563 T608 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 258573 T608 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 259582 T608 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55303_
        [junit4] 2> 259584 T608 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55303_
        [junit4] 2> 259589 T608 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 259599 T678 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
        [junit4] 2> 259599 T678 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
        [junit4] 2> 259601 T635 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 259603 T678 oasc.ZkController.preRegister Registering watch for external collection collection1
        [junit4] 2> 259603 T678 oascc.ZkStateReader.addZkWatch addZkWatch collection1
        [junit4] 2> 259603 T678 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper...
        [junit4] 2> 259605 T678 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4
        [junit4] 2> 259605 T631 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
        [junit4] 2> "numShards":"3",
        [junit4] 2> "state":"down",
        [junit4] 2> "collection":"collection1",
        [junit4] 2> "roles":null,
        [junit4] 2> "shard":null,
        [junit4] 2> "operation":"state",
        [junit4] 2> "base_url":"http://127.0.0.1:55303",
        [junit4] 2> "node_name":"127.0.0.1:55303_",
        [junit4] 2> "core":"collection1"}
        [junit4] 2> 259606 T678 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 259610 T631 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
        [junit4] 2> 259612 T631 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
        [junit4] 2> 259619 T631 oasc.Overseer$ClusterStateUpdater.updateZkStates going to update_collection /collections/collection1/state.json
        [junit4] 2> 259620 T658 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating...
        [junit4] 2> 259620 T677 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating...
        [junit4] 2> 259621 T658 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5
        [junit4] 2> 259622 T677 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5
        [junit4] 2> 260606 T678 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
        [junit4] 2> 260607 T678 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
        [junit4] 2> 260608 T678 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 260608 T678 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
        [junit4] 2> 260609 T678 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 260610 T678 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-004/collection1/'
        [junit4] 2> 260612 T678 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-004/collection1/lib/.svn/' to classloader
        [junit4] 2> 260612 T678 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-004/collection1/lib/README' to classloader
        [junit4] 2> 260613 T678 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-004/collection1/lib/classes/' to classloader
        [junit4] 2> 260705 T678 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 260772 T678 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 260874 T678 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 260892 T678 oass.IndexSchema.readSchema [collection1] Schema name=test
        [junit4] 2> 261304 T678 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 261307 T678 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 261309 T678 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 261315 T678 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 261347 T678 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
        [junit4] 2> 261347 T678 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 261347 T678 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-004/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/jetty2/
        [junit4] 2> 261348 T678 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@72b69ba3
        [junit4] 2> 261349 T678 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/jetty2
        [junit4] 2> 261349 T678 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/jetty2/index/
        [junit4] 2> 261349 T678 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
        [junit4] 2> 261350 T678 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest-7623E1151B611AA4-001/tempDir-001/jetty2/index
        [junit4] 2> 261350 T678 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=19, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4795777660978364]
        [junit4] 2> 261351 T678 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-5847ae1e)),segFN=segments_1,generation=1}
        [junit4] 2> 261352 T678 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 261358 T678 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 261359 T678 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 261359 T678 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 261359 T678 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 261359 T678 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 261360 T678 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 261360 T678 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 261361 T678 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 261361 T678 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 261362 T678 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 261362 T678 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 261363 T678 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 261364 T678 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 261364 T678 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 261364 T678 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 261365 T678 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHan

    [...truncated too long message...]

    dExecution(ThreadPoolExecutor.java:2048)
        [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
        [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
        [junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:110)
        [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:263)
        [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
        [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
        [junit4] 2>
        [junit4] 2> 2295778 T10906 oaz.ClientCnxn$EventThread.processEvent ERROR Error while calling watcher java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@301ea4f3 rejected from java.util.concurrent.ThreadPoolExecutor@d97dfbf[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 5]
        [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
        [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
        [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
        [junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:110)
        [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:263)
        [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
        [junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
        [junit4] 2>
        [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
        [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=DeleteLastCustomShardedReplicaTest -Dtests.method=testDistribSearch -Dtests.seed=7623E1151B611AA4 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=el_CY -Dtests.timezone=IET -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
        [junit4] ERROR 134s J0 | DeleteLastCustomShardedReplicaTest.testDistribSearch <<<
        [junit4] > Throwable #1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:11571/blx, http://127.0.0.1:33031/blx, http://127.0.0.1:11248/blx]
        [junit4] > at __randomizedtesting.SeedInfo.seed([7623E1151B611AA4:F7C56F0D6C3E7A98]:0)
        [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:333)
        [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrServer.sendRequest(CloudSolrServer.java:1015)
        [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrServer.requestWithRetryOnStaleState(CloudSolrServer.java:793)
        [junit4] > at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:736)
        [junit4] > at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.removeAndWaitForLastReplicaGone(DeleteLastCustomShardedReplicaTest.java:117)
        [junit4] > at org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest.doTest(DeleteLastCustomShardedReplicaTest.java:107)
        [junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
        [junit4] > at java.lang.Thread.run(Thread.java:745)
        [junit4] > Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Could not remove replica : customcollreplicadeletion/a/core_node2
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:569)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:215)
        [junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:211)
        [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrServer.doRequest(LBHttpSolrServer.java:354)
        [junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:313)
        [junit4] > ... 46 more
        [junit4] 2> 2295793 T10842 oas.SolrTestCaseJ4.deleteCore ###deleteCore
        [junit4] 2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.DeleteLastCustomShardedReplicaTest-7623E1151B611AA4-001
        [junit4] 2> 134090 T10841 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 2 leaked thread(s).
        [junit4] 2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=el_CY, timezone=IET
        [junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=46440504,total=184025088
        [junit4] 2> NOTE: All tests run in this JVM: [BlockCacheTest, CSVRequestHandlerTest, TestFileDictionaryLookup, TestBinaryResponseWriter, TestLeaderElectionZkExpiry, TestBulkSchemaConcurrent, RecoveryZkTest, HdfsDirectoryTest, TestPostingsSolrHighlighter, TestReplicationHandlerBackup, TestSolrDeletionPolicy1, TestCollectionAPI, ReturnFieldsTest, TestDefaultSearchFieldResource, HdfsSyncSliceTest, TestRecoveryHdfs, TestRequestStatusCollectionAPI, TestRestManager, TestReloadAndDeleteDocs, ConvertedLegacyTest, TestXIncludeConfig, TestFoldingMultitermQuery, TestFiltering, TestValueSourceCache, OverseerTest, FileUtilsTest, TestDefaultSimilarityFactory, TestStressReorder, TestFieldTypeCollectionResource, TestOrdValues, TestDistribDocBasedVersion, TestReRankQParserPlugin, MigrateRouteKeyTest, ChangedSchemaMergeTest, TestLazyCores, DirectSolrConnectionTest, BasicDistributedZkTest, SuggesterWFSTTest, TestSimpleQParserPlugin, TestCoreContainer, HdfsCollectionsAPIDistributedZkTest, SimplePostToolTest, TestSolrXml, TestArbitraryIndexDir, ParsingFieldUpdateProcessorsTest, CoreAdminCreateDiscoverTest, SynonymTokenizerTest, TestFaceting, TestWordDelimiterFilterFactory, DistanceFunctionTest, SoftAutoCommitTest, TestRangeQuery, PrimUtilsTest, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, BasicZkTest, ClusterStateUpdateTest, TestRandomFaceting, LeaderElectionTest, ZkSolrClientTest, TestZkChroot, TestRandomDVFaceting, ShardRoutingCustomTest, AutoCommitTest, TestDistributedGrouping, TestRecovery, DistributedSpellCheckComponentTest, TestJoin, TestMultiCoreConfBootstrap, TestReload, TestStressVersions, HardAutoCommitTest, DistributedTermsComponentTest, TestSolr4Spatial, SolrCmdDistributorTest, TestSort, OverseerCollectionProcessorTest, DistributedQueryElevationComponentTest, DocValuesTest, SuggesterTest, TestTrie, PolyFieldTest, SchemaVersionSpecificBehaviorTest, TestUpdate, TestAtomicUpdateErrorCases, DirectUpdateHandlerOptimizeTest, XsltUpdateRequestHandlerTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, StandardRequestHandlerTest, TestReversedWildcardFilterFactory, TestQueryTypes, TermsComponentTest, TermVectorComponentTest, RequiredFieldsTest, FastVectorHighlighterTest, LoggingHandlerTest, SolrPluginUtilsTest, TestJmxIntegration, TestAnalyzedSuggestions, UniqFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, PingRequestHandlerTest, TestLFUCache, TestQuerySenderListener, AlternateDirectoryTest, TestQuerySenderNoQuery, ResponseLogComponentTest, BadComponentTest, TestSolrDeletionPolicy2, TestDocSet, NumericFieldsTest, TestElisionMultitermQuery, TestFuzzyAnalyzedSuggestions, NotRequiredUniqueKeyTest, TestCharFilters, EchoParamsTest, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestIBSimilarityFactory, LegacyHTMLStripCharFilterTest, TestJmxMonitoredMap, ScriptEngineTest, OpenExchangeRatesOrgProviderTest, PreAnalyzedFieldTest, TestSuggestSpellingConverter, SpellingQueryConverterTest, TestSolrJ, TestUtils, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, UUIDFieldTest, TestRTGBase, CursorPagingTest, DistributedIntervalFacetingTest, TestCursorMarkWithoutUniqueKey, CloudExitableDirectoryReaderTest, DeleteLastCustomShardedReplicaTest]
        [junit4] Completed on J0 in 134.37s, 1 test, 1 error <<< FAILURES!

    [...truncated 523 lines...]
    BUILD FAILED
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:532: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:473: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:189: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:496: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1359: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:966: There were test failures: 437 suites, 1799 tests, 2 errors, 1 failure, 54 ignored (9 assumptions)

    Total time: 220 minutes 23 seconds
    Build step 'Invoke Ant' marked build as failure
    Archiving artifacts
    Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #654
    Archived 5 artifacts
    Archive block size is 32768
    Received 0 blocks and 694410032 bytes
    Compression is 0.0%
    Took 2 min 2 sec
    Recording test results
    Email was triggered for: Failure
    Sending email for trigger: Failure
  • Apache Jenkins Server at Oct 26, 2014 at 6:52 pm
    Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/659/

    2 tests failed.
    REGRESSION: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

    Error Message:
    Captured an uncaught exception in thread: Thread[id=1250, name=Thread-428, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]

    Stack Trace:
    com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1250, name=Thread-428, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
    Caused by: java.lang.NullPointerException
      at __randomizedtesting.SeedInfo.seed([44840241931F6D8A]:0)
      at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)


    FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler

    Error Message:
    Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10371, name=coreLoadExecutor-2180-thread-1, state=RUNNABLE, group=TGRP-TestReplicationHandler], registration stack trace below.

    Stack Trace:
    com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10371, name=coreLoadExecutor-2180-thread-1, state=RUNNABLE, group=TGRP-TestReplicationHandler], registration stack trace below.
      at __randomizedtesting.SeedInfo.seed([44840241931F6D8A]:0)
      at java.lang.Thread.getStackTrace(Thread.java:1589)
      at com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:166)
      at org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:728)
      at org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1314)
      at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1205)
      at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1197)
      at org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:47)
      at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:350)
      at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:276)
      at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:488)
      at org.apache.solr.core.SolrCore.(SolrCore.java:652)
      at org.apache.solr.core.CoreContainer.create(CoreContainer.java:509)
      at org.apache.solr.core.CoreContainer$1.call(CoreContainer.java:273)
      at org.apache.solr.core.CoreContainer$1.call(CoreContainer.java:267)
      at java.util.concurrent.FutureTask.run(FutureTask.java:262)
      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
      at java.lang.Thread.run(Thread.java:745)
    Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler-44840241931F6D8A-001/index-MMapDirectory-122 lockFactory=NativeFSLockFactory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler-44840241931F6D8A-001/index-MMapDirectory-122)
      at org.junit.Assert.fail(Assert.java:93)
      at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:47)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$2$1.apply(RandomizedRunner.java:699)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$2$1.apply(RandomizedRunner.java:696)
      at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:183)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$2.afterAlways(RandomizedRunner.java:712)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      ... 1 more




    Build Log:
    [...truncated 12046 lines...]
        [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
        [junit4] 2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/init-core-data-001
        [junit4] 2> 1992453 T1001 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
        [junit4] 2> 1992453 T1001 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_if/
        [junit4] 2> 1992461 T1001 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
        [junit4] 2> 1992463 T1001 oasc.ZkTestServer.run STARTING ZK TEST SERVER
        [junit4] 1> client port:0.0.0.0/0.0.0.0:0
        [junit4] 2> 1992464 T1002 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
        [junit4] 2> 1992563 T1001 oasc.ZkTestServer.run start zk server on port:53183
        [junit4] 2> 1992564 T1001 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1992565 T1001 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1992570 T1008 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@727956ea name:ZooKeeperConnection Watcher:127.0.0.1:53183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1992570 T1001 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1992571 T1001 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1992571 T1001 oascc.SolrZkClient.makePath makePath: /solr
        [junit4] 2> 1992575 T1001 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1992577 T1001 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1992579 T1010 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@262b8c37 name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1992579 T1001 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1992580 T1001 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1992580 T1001 oascc.SolrZkClient.makePath makePath: /collections/collection1
        [junit4] 2> 1992583 T1001 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
        [junit4] 2> 1992586 T1001 oascc.SolrZkClient.makePath makePath: /collections/control_collection
        [junit4] 2> 1992588 T1001 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
        [junit4] 2> 1992591 T1001 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
        [junit4] 2> 1992593 T1001 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
        [junit4] 2> 1992599 T1001 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
        [junit4] 2> 1992601 T1001 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
        [junit4] 2> 1992709 T1001 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 1992710 T1001 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 1992713 T1001 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
        [junit4] 2> 1992713 T1001 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
        [junit4] 2> 1992715 T1001 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
        [junit4] 2> 1992716 T1001 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
        [junit4] 2> 1992718 T1001 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
        [junit4] 2> 1992719 T1001 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
        [junit4] 2> 1992722 T1001 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
        [junit4] 2> 1992723 T1001 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
        [junit4] 2> 1992725 T1001 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
        [junit4] 2> 1992727 T1001 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
        [junit4] 2> 1992729 T1001 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
        [junit4] 2> 1992730 T1001 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
        [junit4] 2> 1992733 T1001 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
        [junit4] 2> 1992734 T1001 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
        [junit4] 2> 1992736 T1001 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
        [junit4] 2> 1992737 T1001 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
        [junit4] 2> Using new style solr.xml
        [junit4] 2> 1994439 T1001 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 1994444 T1001 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:55498
        [junit4] 2> 1994445 T1001 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 1994446 T1001 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 1994446 T1001 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-002
        [junit4] 2> 1994447 T1001 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-002/'
        [junit4] 2> 1994494 T1001 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-002/solr.xml
        [junit4] 2> 1994586 T1001 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-002
        [junit4] 2> 1994587 T1001 oasc.CoreContainer.<init> New CoreContainer 377137719
        [junit4] 2> 1994588 T1001 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-002/]
        [junit4] 2> 1994590 T1001 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 1994590 T1001 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 1994591 T1001 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 1994591 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 1994592 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 1994592 T1001 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 1994593 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 1994594 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 1994594 T1001 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 1994595 T1001 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 1994596 T1001 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 1994596 T1001 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 1994597 T1001 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 1994597 T1001 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 1994598 T1001 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53183/solr
        [junit4] 2> 1994599 T1001 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 1994599 T1001 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1994600 T1001 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1994602 T1021 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f3d0c68 name:ZooKeeperConnection Watcher:127.0.0.1:53183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1994603 T1001 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1994604 T1001 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1994607 T1001 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1994609 T1023 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c1f37d4 name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1994609 T1001 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1994612 T1001 oascc.SolrZkClient.makePath makePath: /overseer/queue
        [junit4] 2> 1994615 T1001 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
        [junit4] 2> 1994617 T1001 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
        [junit4] 2> 1994621 T1001 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
        [junit4] 2> 1994624 T1001 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
        [junit4] 2> 1994627 T1001 oascc.SolrZkClient.makePath makePath: /live_nodes
        [junit4] 2> 1994629 T1001 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55498__if
        [junit4] 2> 1994630 T1001 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55498__if
        [junit4] 2> 1994633 T1001 oascc.SolrZkClient.makePath makePath: /overseer_elect
        [junit4] 2> 1994635 T1001 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
        [junit4] 2> 1994638 T1001 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 1994644 T1001 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:55498__if
        [junit4] 2> 1994645 T1001 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
        [junit4] 2> 1994646 T1001 oasc.Overseer.start Overseer (id=92690615886675971-127.0.0.1:55498__if-n_0000000000) starting
        [junit4] 2> 1994649 T1001 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
        [junit4] 2> 1994657 T1001 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
        [junit4] 2> 1994670 T1025 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
        [junit4] 2> 1994671 T1001 oascc.SolrZkClient.makePath makePath: /clusterstate.json
        [junit4] 2> 1994675 T1001 oascc.SolrZkClient.makePath makePath: /aliases.json
        [junit4] 2> 1994676 T1001 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 1994680 T1024 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
        [junit4] 2> 1994683 T1001 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-002
        [junit4] 2> 1996026 T1001 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-002/conf/
        [junit4] 2> 1996264 T1001 oasc.CorePropertiesLocator.discover Found 1 core definitions
        [junit4] 2> 1996266 T1001 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 1996266 T1001 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 1996267 T1001 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1996268 T1001 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1996269 T1029 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@118b589b name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1996270 T1001 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1996270 T1001 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1996272 T1001 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 1996274 T1001 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
        [junit4] 2> 1998289 T1001 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
        [junit4] 2> 1998291 T1001 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 1998295 T1001 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:60017
        [junit4] 2> 1998296 T1001 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 1998297 T1001 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 1998297 T1001 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003
        [junit4] 2> 1998298 T1001 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003/'
        [junit4] 2> 1998368 T1001 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003/solr.xml
        [junit4] 2> 1998462 T1001 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003
        [junit4] 2> 1998463 T1001 oasc.CoreContainer.<init> New CoreContainer 939019324
        [junit4] 2> 1998464 T1001 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003/]
        [junit4] 2> 1998465 T1001 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 1998466 T1001 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 1998466 T1001 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 1998467 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 1998467 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 1998467 T1001 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 1998468 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 1998468 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 1998469 T1001 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 1998469 T1001 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 1998470 T1001 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 1998470 T1001 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 1998471 T1001 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 1998471 T1001 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 1998472 T1001 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53183/solr
        [junit4] 2> 1998472 T1001 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 1998472 T1001 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1998473 T1001 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1998475 T1040 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e5513d9 name:ZooKeeperConnection Watcher:127.0.0.1:53183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1998476 T1001 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1998477 T1001 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1998481 T1001 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1998483 T1042 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49743a75 name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1998483 T1001 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1998491 T1001 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 1999495 T1001 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60017__if
        [junit4] 2> 1999497 T1001 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60017__if
        [junit4] 2> 1999501 T1001 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 1999508 T1001 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003
        [junit4] 2> 2000903 T1001 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003/conf/
        [junit4] 2> 2000995 T1001 oasc.CorePropertiesLocator.discover Found 1 core definitions
        [junit4] 2> 2000997 T1001 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 2000998 T1001 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 2003070 T1001 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
        [junit4] 2> 2003072 T1001 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 2003077 T1001 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:10322
        [junit4] 2> 2003078 T1001 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 2003079 T1001 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 2003079 T1001 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004
        [junit4] 2> 2003080 T1001 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004/'
        [junit4] 2> 2003151 T1001 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004/solr.xml
        [junit4] 2> 2003246 T1001 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004
        [junit4] 2> 2003247 T1001 oasc.CoreContainer.<init> New CoreContainer 2134477858
        [junit4] 2> 2003247 T1001 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004/]
        [junit4] 2> 2003249 T1001 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 2003250 T1001 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 2003250 T1001 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 2003251 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 2003252 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 2003253 T1001 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 2003254 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 2003255 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 2003256 T1001 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 2003257 T1001 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 2003258 T1001 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 2003258 T1001 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 2003259 T1001 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 2003259 T1001 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 2003259 T1001 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53183/solr
        [junit4] 2> 2003260 T1001 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 2003260 T1001 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 2003261 T1001 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 2003264 T1057 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b1f4d6f name:ZooKeeperConnection Watcher:127.0.0.1:53183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 2003265 T1001 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 2003266 T1001 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 2003270 T1001 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 2003272 T1059 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6af43e4 name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 2003272 T1001 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 2003280 T1001 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 2004285 T1001 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:10322__if
        [junit4] 2> 2004287 T1001 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:10322__if
        [junit4] 2> 2004292 T1001 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 2004299 T1001 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004
        [junit4] 2> 2005786 T1001 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004/conf/
        [junit4] 2> 2005789 T1001 oasc.CorePropertiesLocator.discover Found 1 core definitions
        [junit4] 2> 2005791 T1001 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 2005791 T1001 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 2007728 T1001 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
        [junit4] 2> 2007730 T1001 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 2007734 T1001 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:15701
        [junit4] 2> 2007735 T1001 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 2007736 T1001 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 2007737 T1001 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005
        [junit4] 2> 2007737 T1001 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005/'
        [junit4] 2> 2007808 T1001 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005/solr.xml
        [junit4] 2> 2007920 T1001 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005
        [junit4] 2> 2007921 T1001 oasc.CoreContainer.<init> New CoreContainer 936002112
        [junit4] 2> 2007922 T1001 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005/]
        [junit4] 2> 2007924 T1001 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 2007925 T1001 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 2007926 T1001 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 2007926 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 2007927 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 2007928 T1001 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 2007928 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 2007929 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 2007930 T1001 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 2007930 T1001 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 2007931 T1001 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 2007932 T1001 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 2007933 T1001 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 2007934 T1001 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 2007934 T1001 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53183/solr
        [junit4] 2> 2007935 T1001 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 2007936 T1001 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 2007937 T1001 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 2007939 T1072 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3fc3945d name:ZooKeeperConnection Watcher:127.0.0.1:53183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 2007940 T1001 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 2007941 T1001 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 2007944 T1001 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 2007946 T1074 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2804e04b name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 2007947 T1001 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 2007954 T1001 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 2008959 T1001 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:15701__if
        [junit4] 2> 2008961 T1001 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:15701__if
        [junit4] 2> 2008965 T1001 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 2008972 T1001 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005
        [junit4] 2> 2008997 T1001 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005/conf/
        [junit4] 2> 2010602 T1001 oasc.CorePropertiesLocator.discover Found 1 core definitions
        [junit4] 2> 2010603 T1001 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 2010604 T1001 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 2012645 T1001 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
        [junit4] 2> 2012647 T1001 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 2012653 T1001 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:21594
        [junit4] 2> 2012655 T1001 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 2012656 T1001 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 2012656 T1001 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006
        [junit4] 2> 2012657 T1001 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006/'
        [junit4] 2> 2012758 T1001 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006/solr.xml
        [junit4] 2> 2012852 T1001 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006
        [junit4] 2> 2012853 T1001 oasc.CoreContainer.<init> New CoreContainer 596592926
        [junit4] 2> 2012854 T1001 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006/]
        [junit4] 2> 2012856 T1001 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 2012856 T1001 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 2012857 T1001 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 2012858 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 2012858 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 2012858 T1001 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 2012859 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 2012860 T1001 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 2012860 T1001 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 2012861 T1001 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 2012862 T1001 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 2012862 T1001 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 2012863 T1001 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 2012864 T1001 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 2012864 T1001 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53183/solr
        [junit4] 2> 2012865 T1001 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 2012865 T1001 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 2012866 T1001 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 2012868 T1087 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@140962cf name:ZooKeeperConnection Watcher:127.0.0.1:53183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 2012869 T1001 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 2012870 T1001 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 2012873 T1001 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 2012875 T1089 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49edeec0 name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 2012876 T1001 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 2012888 T1001 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 2013892 T1001 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:21594__if
        [junit4] 2> 2013893 T1001 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:21594__if
        [junit4] 2> 2013898 T1001 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 2013904 T1001 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006
        [junit4] 2> 2013912 T1001 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006/conf/
        [junit4] 2> 2016051 T1001 oasc.CorePropertiesLocator.discover Found 1 core definitions
        [junit4] 2> 2016052 T1001 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 2016053 T1001 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 2016071 T1033 oasha.CollectionsHandler.handleCreateAction Creating Collection : wt=javabin&action=CREATE&collection.configName=conf1&version=2&replicationFactor=2&router.field=myOwnField&name=solrj_collection&numShards=2
        [junit4] 2> 2016077 T1045 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 2016079 T1025 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
        [junit4] 2> "operation":"create",
        [junit4] 2> "fromApi":"true",
        [junit4] 2> "name":"solrj_collection",
        [junit4] 2> "replicationFactor":"2",
        [junit4] 2> "collection.configName":"conf1",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "router.field":"myOwnField"}
        [junit4] 2> 2016460 T1092 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : create , {
        [junit4] 2> "operation":"create",
        [junit4] 2> "fromApi":"true",
        [junit4] 2> "name":"solrj_collection",
        [junit4] 2> "replicationFactor":"2",
        [junit4] 2> "collection.configName":"conf1",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "router.field":"myOwnField"}
        [junit4] 2> 2016461 T1092 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection
        [junit4] 2> 2016461 T1092 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
        [junit4] 2> 2016465 T1045 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 2016468 T1024 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
        [junit4] 2> 2016468 T1024 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
        [junit4] 2> 2016468 T1024 oasc.Overseer$ClusterStateUpdater.createCollection state version solrj_collection 1
        [junit4] 2> 2016471 T1075 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 2016471 T1043 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 2016471 T1044 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 2016471 T1060 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 2016471 T1045 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 2016471 T1090 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 2016565 T1092 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection solrj_collection, shardNames [shard1, shard2] , replicationFactor : 2
        [junit4] 2> 2016566 T1092 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:15701__if
        [junit4] 2> 2016568 T1092 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:21594__if
        [junit4] 2> 2016570 T1092 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:60017__if
        [junit4] 2> 2016572 T1092 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:10322__if
        [junit4] 2> 2016573 T1065 oasha.CoreAdminHandler.handleCreateAction core create command wt=javabin&shard=shard1&qt=/admin/cores&action=CREATE&collection.configName=conf1&version=2&name=solrj_collection_shard1_replica1&collection=solrj_collection&numShards=2
        [junit4] 2> 2016573 T1079 oasha.CoreAdminHandler.handleCreateAction core create command wt=javabin&shard=shard1&qt=/admin/cores&action=CREATE&collection.configName=conf1&version=2&name=solrj_collection_shard1_replica2&collection=solrj_collection&numShards=2
        [junit4] 2> 2016575 T1032 oasha.CoreAdminHandler.handleCreateAction core create command wt=javabin&shard=shard2&qt=/admin/cores&action=CREATE&collection.configName=conf1&version=2&name=solrj_collection_shard2_replica1&collection=solrj_collection&numShards=2
        [junit4] 2> 2016576 T1065 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down collection=solrj_collection
        [junit4] 2> 2016576 T1049 oasha.CoreAdminHandler.handleCreateAction core create command wt=javabin&shard=shard2&qt=/admin/cores&action=CREATE&collection.configName=conf1&version=2&name=solrj_collection_shard2_replica2&collection=solrj_collection&numShards=2
        [junit4] 2> 2016578 T1045 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 2016578 T1065 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 2016580 T1032 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down collection=solrj_collection
        [junit4] 2> 2016581 T1032 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 2016581 T1049 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down collection=solrj_collection
        [junit4] 2> 2016582 T1079 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down collection=solrj_collection
        [junit4] 2> 2016583 T1049 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 2016584 T1079 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 2016584 T1024 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "core":"solrj_collection_shard1_replica1",
        [junit4] 2> "shard":"shard1",
        [junit4] 2> "node_name":"127.0.0.1:15701__if",
        [junit4] 2> "operation":"state",
        [junit4] 2> "roles":null,
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "base_url":"http://127.0.0.1:15701/_if",
        [junit4] 2> "state":"down",
        [junit4] 2> "numShards":"2"}
        [junit4] 2> 2016588 T1024 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "core":"solrj_collection_shard2_replica1",
        [junit4] 2> "shard":"shard2",
        [junit4] 2> "node_name":"127.0.0.1:60017__if",
        [junit4] 2> "operation":"state",
        [junit4] 2> "roles":null,
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "base_url":"http://127.0.0.1:60017/_if",
        [junit4] 2> "state":"down",
        [junit4] 2> "numShards":"2"}
        [junit4] 2> 2016591 T1045 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 2016592 T1024 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "core":"solrj_collection_shard2_replica2",
        [junit4] 2> "shard":"shard2",
        [junit4] 2> "node_name":"127.0.0.1:10322__if",
        [junit4] 2> "operation":"state",
        [junit4] 2> "roles":null,
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "base_url":"http://127.0.0.1:10322/_if",
        [junit4] 2> "state":"down",
        [junit4] 2> "numShards":"2"}
        [junit4] 2> 2016596 T1024 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "core":"solrj_collection_shard1_replica2",
        [junit4] 2> "shard":"shard1",
        [junit4] 2> "node_name":"127.0.0.1:21594__if",
        [junit4] 2> "operation":"state",
        [junit4] 2> "roles":null,
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "base_url":"http://127.0.0.1:21594/_if",
        [junit4] 2> "state":"down",
        [junit4] 2> "numShards":"2"}
        [junit4] 2> 2016599 T1045 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 2016600 T1043 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 2016600 T1060 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 2016600 T1075 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 2016600 T1045 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 2016600 T1044 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 2016601 T1090 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 2017579 T1065 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
        [junit4] 2> 2017580 T1065 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 2017581 T1065 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 2017581 T1065 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 2017582 T1032 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
        [junit4] 2> 2017583 T1032 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 2017584 T1065 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 2017585 T1032 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 2017586 T1049 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
        [junit4] 2> 2017585 T1065 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005/solrj_collection_shard1_replica1/'
        [junit4] 2> 2017587 T1049 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 2017587 T1032 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 2017591 T1079 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
        [junit4] 2> 2017591 T1049 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 2017591 T1032 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 2017591 T1079 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 2017592 T1032 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003/solrj_collection_shard2_replica1/'
        [junit4] 2> 2017592 T1049 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 2017593 T1079 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 2017594 T1079 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 2017594 T1049 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 2017595 T1049 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004/solrj_collection_shard2_replica2/'
        [junit4] 2> 2017595 T1079 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 2017596 T1079 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006/solrj_collection_shard1_replica2/'
        [junit4] 2> 2017783 T1049 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 2017784 T1079 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 2017784 T1032 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 2017789 T1065 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 2017854 T1049 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 2017855 T1079 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 2017857 T1032 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 2017864 T1065 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 2017955 T1049 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 2017956 T1079 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 2017958 T1032 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 2017965 T1065 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 2017972 T1079 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
        [junit4] 2> 2017973 T1049 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
        [junit4] 2> 2017973 T1032 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
        [junit4] 2> 2017982 T1065 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
        [junit4] 2> 2018621 T1032 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 2018637 T1065 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 2018646 T1032 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 2018646 T1049 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 2018651 T1032 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 2018653 T1065 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 2018657 T1065 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 2018658 T1049 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 2018662 T1049 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 2018671 T1032 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 2018675 T1065 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 2018679 T1032 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 2018684 T1065 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 2018685 T1049 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 2018688 T1032 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 2018690 T1032 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 2018693 T1032 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 2018693 T1065 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 2018694 T1032 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 2018695 T1049 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 2018695 T1065 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 2018696 T1032 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 2018696 T1032 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 2018696 T1065 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 2018697 T1032 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using configuration from collection solrj_collection
        [junit4] 2> 2018698 T1065 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 2018698 T1032 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 2018700 T1032 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003/solrj_collection_shard2_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003/solrj_collection_shard2_replica1/data/
        [junit4] 2> 2018700 T1032 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2998a407
        [junit4] 2> 2018701 T1065 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 2018702 T1065 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 2018702 T1032 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003/solrj_collection_shard2_replica1/data
        [junit4] 2> 2018703 T1049 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 2018702 T1065 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using configuration from collection solrj_collection
        [junit4] 2> 2018703 T1032 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003/solrj_collection_shard2_replica1/data/index/
        [junit4] 2> 2018705 T1049 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 2018704 T1065 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 2018705 T1049 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 2018705 T1032 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003/solrj_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
        [junit4] 2> 2018707 T1049 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 2018708 T1032 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-003/solrj_collection_shard2_replica1/data/index
        [junit4] 2> 2018708 T1049 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 2018709 T1049 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 2018709 T1079 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 2018706 T1065 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005/solrj_collection_shard1_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005/solrj_collection_shard1_replica1/data/
        [junit4] 2> 2018709 T1049 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using configuration from collection solrj_collection
        [junit4] 2> 2018709 T1032 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=568370488, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
        [junit4] 2> 2018711 T1049 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 2018711 T1065 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2998a407
        [junit4] 2> 2018713 T1032 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-7878966d)),segFN=segments_1,generation=1}
        [junit4] 2> 2018713 T1049 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004/solrj_collection_shard2_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004/solrj_collection_shard2_replica2/data/
        [junit4] 2> 2018714 T1065 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005/solrj_collection_shard1_replica1/data
        [junit4] 2> 2018714 T1049 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2998a407
        [junit4] 2> 2018714 T1032 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 2018716 T1065 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005/solrj_collection_shard1_replica1/data/index/
        [junit4] 2> 2018716 T1049 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004/solrj_collection_shard2_replica2/data
        [junit4] 2> 2018717 T1065 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
        [junit4] 2> 2018717 T1049 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004/solrj_collection_shard2_replica2/data/index/
        [junit4] 2> 2018718 T1065 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-005/solrj_collection_shard1_replica1/data/index
        [junit4] 2> 2018718 T1049 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
        [junit4] 2> 2018720 T1065 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=568370488, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
        [junit4] 2> 2018721 T1049 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-004/solrj_collection_shard2_replica2/data/index
        [junit4] 2> 2018722 T1065 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-197b76a6)),segFN=segments_1,generation=1}
        [junit4] 2> 2018722 T1049 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=568370488, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
        [junit4] 2> 2018723 T1079 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 2018723 T1065 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 2018724 T1049 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-ddc6720)),segFN=segments_1,generation=1}
        [junit4] 2> 2018725 T1032 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 2018726 T1032 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 2018727 T1032 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 2018727 T1032 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 2018728 T1032 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 2018729 T1032 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 2018729 T1079 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 2018730 T1049 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 2018730 T1032 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 2018731 T1032 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 2018732 T1032 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 2018732 T1065 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 2018733 T1032 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 2018733 T1065 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 2018734 T1032 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 2018734 T1065 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 2018735 T1032 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 2018735 T1065 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 2018736 T1032 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 2018736 T1065 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 2018737 T1032 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 2018737 T1065 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 2018738 T1032 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 2018739 T1065 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 2018739 T1032 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 2018738 T1049 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 2018740 T1032 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 2018740 T1065 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 2018742 T1032 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
        [junit4] 2> 2018741 T1049 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 2018743 T1032 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
        [junit4] 2> 2018743 T1049 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 2018742 T1065 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 2018744 T1049 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 2018744 T1032 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
        [junit4] 2> 2018746 T1065 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 2018746 T1049 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 2018747 T1065 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 2018747 T1049 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 2018748 T1065 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 2018749 T1049 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 2018749 T1065 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 2018750 T1049 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 2018750 T1049 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 2018751 T1065 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 2018751 T1065 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 2018752 T1049 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 2018752 T1065 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 2018753 T1049 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 2018754 T1065 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 2018755 T1049 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 2018755 T1065 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
        [junit4] 2> 2018756 T1049 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 2018756 T1065 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
        [junit4] 2> 2018757 T1049 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 2018757 T1079 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 2018758 T1065 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
        [junit4] 2> 2018757 T1049 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 2018760 T1049 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 2018761 T1049 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 2018762 T1049 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
        [junit4] 2> 2018763 T1049 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
        [junit4] 2> 2018764 T1049 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
        [junit4] 2> 2018774 T1079 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 2018781 T1065 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 2018781 T1032 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 2018781 T1049 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 2018782 T1079 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 2018784 T1079 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 2018784 T1065 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 2018784 T1079 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 2018785 T1032 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 2018785 T1079 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 2018785 T1049 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 2018787 T1079 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 2018788 T1079 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 2018787 T1065 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 2018789 T1032 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 2018789 T1079 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using configuration from collection solrj_collection
        [junit4] 2> 2018790 T1049 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 2018791 T1079 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 2018792 T1079 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006/solrj_collection_shard1_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006/solrj_collection_shard1_replica2/data/
        [junit4] 2> 2018792 T1065 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 2018793 T1032 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 2018793 T1079 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2998a407
        [junit4] 2> 2018801 T1049 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 2018804 T1079 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006/solrj_collection_shard1_replica2/data
        [junit4] 2> 2018805 T1079 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006/solrj_collection_shard1_replica2/data/index/
        [junit4] 2> 2018806 T1079 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
        [junit4] 2> 2018807 T1079 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-44840241931F6D8A-001/tempDir-006/solrj_collection_shard1_replica2/data/index
        [junit4] 2> 2018808 T1079 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=568370488, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
        [junit4] 2> 2018810 T1079 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-5b09b45d)),segFN=segments_1,generation=1}
        [junit4] 2> 2018810 T1079 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 2018811 T1065 oasu.CommitTracker.<init> Hard AutoCommit: disabled
        [junit4] 2> 2018812 T1065 oasu.CommitTracker.<init> Soft AutoCommit: disabled
        [junit4] 2> 2018812 T1032 oasu.CommitTracker.<init> Hard AutoCommit: disabled
        [junit4] 2> 2018813 T1065 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=909042512, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
        [junit4] 2> 2018813 T1049 oasu.CommitTracker.<init> Hard AutoCommit: disabled
        [junit4] 2> 2018814 T1065 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-197b76a6)),segFN=segments_1,generation=1}
        [junit4] 2> 2018814 T1049 oasu.CommitTracker.<init> Soft AutoCommit: disabled
        [junit4] 2> 2018813 T1032 oasu.CommitTracker.<init> Soft AutoCommit: disabled
        [junit4] 2> 2018815 T1065 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 2018816 T1049 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=909042512, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
        [junit4] 2> 2018818 T1049 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-ddc6720)),segFN=segments_1,generation=1}


    [...truncated too long message...]

    -5.x/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler-44840241931F6D8A-001/solr-instance-001/collection1/data
        [junit4] 2> 3075404 T9703 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
        [junit4] 2> 3080399 T9703 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=906770506
        [junit4] 2> 3080401 T9703 oasc.SolrCore.close [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@545a125c
        [junit4] 2> 3080401 T9703 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=3,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0}
        [junit4] 2> 3080402 T9703 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
        [junit4] 2> 3080402 T9703 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
        [junit4] 2> 3080402 T9703 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
        [junit4] 2> 3080406 T9703 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
        [junit4] 2> 3080406 T9703 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
        [junit4] 2> 3080407 T9703 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler-44840241931F6D8A-001/solr-instance-002/collection1/data/index.20141026212038163 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler-44840241931F6D8A-001/solr-instance-002/collection1/data/index.20141026212038163;done=false>>]
        [junit4] 2> 3080407 T9703 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler-44840241931F6D8A-001/solr-instance-002/collection1/data/index.20141026212038163
        [junit4] 2> 3080407 T9703 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler-44840241931F6D8A-001/solr-instance-002/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler-44840241931F6D8A-001/solr-instance-002/collection1/data;done=false>>]
        [junit4] 2> 3080408 T9703 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler-44840241931F6D8A-001/solr-instance-002/collection1/data
        [junit4] 2> 3080409 T9703 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
        [junit4] 2> 3085419 T9703 oas.SolrTestCaseJ4.deleteCore ###deleteCore
        [junit4] 2> NOTE: test params are: codec=Asserting(Lucene50), sim=DefaultSimilarity, locale=fr_CA, timezone=Asia/Amman
        [junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=170517960,total=522715136
        [junit4] 2> NOTE: All tests run in this JVM: [TestComponentsName, TestManagedSchemaFieldResource, TestNRTOpen, SystemInfoHandlerTest, TestLFUCache, CurrencyFieldOpenExchangeTest, TriLevelCompositeIdRoutingTest, SpatialFilterTest, SolrIndexConfigTest, TestIntervalFaceting, TestBulkSchemaConcurrent, TestCustomSort, AnalysisErrorHandlingTest, ZkControllerTest, TestRecovery, TestExpandComponent, TestPivotHelperCode, HttpPartitionTest, TestCollationFieldDocValues, TestFieldTypeCollectionResource, TestBulkSchemaAPI, TestAtomicUpdateErrorCases, DirectUpdateHandlerOptimizeTest, TestQuerySenderNoQuery, TestGroupingSearch, TestBM25SimilarityFactory, DocValuesMissingTest, FullSolrCloudDistribCmdsTest, ShowFileRequestHandlerTest, TestSimpleQParserPlugin, UUIDUpdateProcessorFallbackTest, PathHierarchyTokenizerFactoryTest, MBeansHandlerTest, TestSystemIdResolver, PreAnalyzedFieldTest, RequiredFieldsTest, TestShardHandlerFactory, DocumentBuilderTest, SampleTest, TestRestManager, ChaosMonkeySafeLeaderTest, PrimitiveFieldTypeTest, JSONWriterTest, TestCodecSupport, DisMaxRequestHandlerTest, TestSweetSpotSimilarityFactory, NoCacheHeaderTest, SolrCoreTest, TestConfig, MultiThreadedOCPTest, TestDistribDocBasedVersion, ConnectionManagerTest, TestReRankQParserPlugin, LeaderElectionIntegrationTest, ShardRoutingTest, BasicZkTest, TestReplicationHandler]
        [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
        [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestReplicationHandler -Dtests.seed=44840241931F6D8A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=fr_CA -Dtests.timezone=Asia/Amman -Dtests.asserts=true -Dtests.file.encoding=UTF-8
        [junit4] ERROR 0.00s J1 | TestReplicationHandler (suite) <<<
        [junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10371, name=coreLoadExecutor-2180-thread-1, state=RUNNABLE, group=TGRP-TestReplicationHandler], registration stack trace below.
        [junit4] > at __randomizedtesting.SeedInfo.seed([44840241931F6D8A]:0)
        [junit4] > at java.lang.Thread.getStackTrace(Thread.java:1589)
        [junit4] > at com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:166)
        [junit4] > at org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:728)
        [junit4] > at org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1314)
        [junit4] > at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1205)
        [junit4] > at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1197)
        [junit4] > at org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:47)
        [junit4] > at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:350)
        [junit4] > at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:276)
        [junit4] > at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:488)
        [junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:796)
        [junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:652)
        [junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:509)
        [junit4] > at org.apache.solr.core.CoreContainer$1.call(CoreContainer.java:273)
        [junit4] > at org.apache.solr.core.CoreContainer$1.call(CoreContainer.java:267)
        [junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:262)
        [junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        [junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        [junit4] > at java.lang.Thread.run(Thread.java:745)
        [junit4] > Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler-44840241931F6D8A-001/index-MMapDirectory-122 lockFactory=NativeFSLockFactory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler-44840241931F6D8A-001/index-MMapDirectory-122)
        [junit4] > at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:47)
        [junit4] > at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:183)
        [junit4] > ... 1 more
        [junit4] Completed on J1 in 1043.72s, 14 tests, 1 error <<< FAILURES!

    [...truncated 246 lines...]
    BUILD FAILED
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:532: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:473: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:189: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:496: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1359: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:966: There were test failures: 438 suites, 1800 tests, 1 suite-level error, 1 error, 54 ignored (9 assumptions)

    Total time: 225 minutes 44 seconds
    Build step 'Invoke Ant' marked build as failure
    Archiving artifacts
    Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #654
    Archived 5 artifacts
    Archive block size is 32768
    Received 0 blocks and 478958103 bytes
    Compression is 0.0%
    Took 1 min 29 sec
    Recording test results
    Email was triggered for: Failure
    Sending email for trigger: Failure
  • Apache Jenkins Server at Oct 28, 2014 at 12:18 am
    Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/660/

    2 tests failed.
    REGRESSION: org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest.testDistribSearch

    Error Message:
    There are still nodes recoverying - waited for 30 seconds

    Stack Trace:
    java.lang.AssertionError: There are still nodes recoverying - waited for 30 seconds
      at __randomizedtesting.SeedInfo.seed([B5A13DAD736935A:8ABC9DC2A069F366]:0)
      at org.junit.Assert.fail(Assert.java:93)
      at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:178)
      at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:840)
      at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1459)
      at org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest.doTest(DistribDocExpirationUpdateProcessorTest.java:79)
      at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:606)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
      at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
      at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
      at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
      at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
      at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
      at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
      at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      at java.lang.Thread.run(Thread.java:745)


    FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

    Error Message:
    Captured an uncaught exception in thread: Thread[id=399, name=Thread-149, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]

    Stack Trace:
    com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=399, name=Thread-149, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
    Caused by: java.lang.NullPointerException
      at __randomizedtesting.SeedInfo.seed([B5A13DAD736935A]:0)
      at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)




    Build Log:
    [...truncated 11573 lines...]
        [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
        [junit4] 2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/init-core-data-001
        [junit4] 2> 95802 T170 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
        [junit4] 2> 95803 T170 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /iv/dh
        [junit4] 2> 95810 T170 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
        [junit4] 2> 95821 T170 oasc.ZkTestServer.run STARTING ZK TEST SERVER
        [junit4] 1> client port:0.0.0.0/0.0.0.0:0
        [junit4] 2> 95830 T171 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
        [junit4] 2> 96030 T170 oasc.ZkTestServer.run start zk server on port:48179
        [junit4] 2> 96040 T170 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 96283 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 96341 T172 oazs.NIOServerCnxn.doIO WARN Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
        [junit4] 2> 98043 T176 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@53e0e375 name:ZooKeeperConnection Watcher:127.0.0.1:48179 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 98044 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 98044 T170 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 98055 T170 oascc.SolrZkClient.makePath makePath: /solr
        [junit4] 2> 98177 T170 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 98178 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 98180 T179 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@26f32b74 name:ZooKeeperConnection Watcher:127.0.0.1:48179/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 98181 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 98181 T170 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 98189 T170 oascc.SolrZkClient.makePath makePath: /collections/collection1
        [junit4] 2> 98194 T170 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
        [junit4] 2> 98198 T170 oascc.SolrZkClient.makePath makePath: /collections/control_collection
        [junit4] 2> 98201 T170 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
        [junit4] 2> 98205 T170 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
        [junit4] 2> 98206 T170 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
        [junit4] 2> 98212 T170 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
        [junit4] 2> 98213 T170 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
        [junit4] 2> 98218 T170 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 98219 T170 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 98223 T170 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
        [junit4] 2> 98224 T170 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
        [junit4] 2> 98228 T170 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
        [junit4] 2> 98228 T170 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
        [junit4] 2> 98233 T170 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
        [junit4] 2> 98233 T170 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
        [junit4] 2> 98238 T170 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
        [junit4] 2> 98238 T170 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
        [junit4] 2> 98242 T170 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
        [junit4] 2> 98243 T170 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
        [junit4] 2> 98247 T170 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
        [junit4] 2> 98248 T170 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
        [junit4] 2> 98252 T170 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
        [junit4] 2> 98253 T170 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
        [junit4] 2> 98257 T170 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
        [junit4] 2> 98258 T170 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
        [junit4] 2> Using old style solr.xml
        [junit4] 2> 99465 T170 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 99469 T170 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:48183
        [junit4] 2> 99470 T170 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 99470 T170 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 99470 T170 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-002
        [junit4] 2> 99471 T170 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-002/'
        [junit4] 2> 99501 T170 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-002/solr.xml
        [junit4] 2> 99574 T170 oasc.CoreContainer.<init> New CoreContainer 1463550078
        [junit4] 2> 99574 T170 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-002/]
        [junit4] 2> 99576 T170 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 99576 T170 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 99576 T170 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 99577 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 99577 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 99577 T170 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 99577 T170 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 99578 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 99578 T170 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 99578 T170 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 99579 T170 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 99579 T170 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 99579 T170 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 99580 T170 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 99580 T170 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48179/solr
        [junit4] 2> 99605 T170 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 99605 T170 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 99606 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 99609 T190 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2ac1105f name:ZooKeeperConnection Watcher:127.0.0.1:48179 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 99609 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 99610 T170 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 99624 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 99626 T192 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@688b97a9 name:ZooKeeperConnection Watcher:127.0.0.1:48179/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 99626 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 99647 T170 oascc.SolrZkClient.makePath makePath: /overseer/queue
        [junit4] 2> 99659 T170 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
        [junit4] 2> 99669 T170 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
        [junit4] 2> 99674 T170 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
        [junit4] 2> 99678 T170 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
        [junit4] 2> 99702 T170 oascc.SolrZkClient.makePath makePath: /live_nodes
        [junit4] 2> 99704 T170 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48183_iv%2Fdh
        [junit4] 2> 99722 T170 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48183_iv%2Fdh
        [junit4] 2> 99730 T170 oascc.SolrZkClient.makePath makePath: /overseer_elect
        [junit4] 2> 99733 T170 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
        [junit4] 2> 99736 T170 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 99760 T170 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:48183_iv%2Fdh
        [junit4] 2> 99760 T170 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
        [junit4] 2> 99764 T170 oasc.Overseer.start Overseer (id=92697411967516675-127.0.0.1:48183_iv%2Fdh-n_0000000000) starting
        [junit4] 2> 99784 T170 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
        [junit4] 2> 99824 T170 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
        [junit4] 2> 99847 T194 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
        [junit4] 2> 99848 T170 oascc.SolrZkClient.makePath makePath: /clusterstate.json
        [junit4] 2> 99851 T170 oascc.SolrZkClient.makePath makePath: /aliases.json
        [junit4] 2> 99853 T170 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 99890 T193 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
        [junit4] 2> 99900 T170 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 99901 T170 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 99944 T170 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 99945 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 99947 T198 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71d1ff7f name:ZooKeeperConnection Watcher:127.0.0.1:48179/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 99947 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 99948 T170 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 99950 T170 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 99958 T170 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
        [junit4] 2> 99958 T170 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
        [junit4] 2> 99959 T170 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 99959 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 99961 T200 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@638b6eec name:ZooKeeperConnection Watcher:127.0.0.1:48179/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 99962 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 99962 T170 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 99980 T201 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 100004 T193 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: collection1
        [junit4] 2> 100005 T193 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
        [junit4] 2> 100019 T193 oasc.Overseer$ClusterStateUpdater.createCollection state version collection1 2
        [junit4] 2> 100031 T193 oasc.Overseer$ClusterStateUpdater.updateZkStates going to create_collection /collections/collection1/state.json
        [junit4] 2> 100049 T201 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 100049 T202 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
        [junit4] 2> 101097 T170 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
        [junit4] 2> 101103 T170 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 101105 T170 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:48188
        [junit4] 2> 101106 T170 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 101106 T170 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 101106 T170 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-003
        [junit4] 2> 101107 T170 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-003/'
        [junit4] 2> 101137 T170 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-003/solr.xml
        [junit4] 2> 101195 T170 oasc.CoreContainer.<init> New CoreContainer 354244941
        [junit4] 2> 101196 T170 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-003/]
        [junit4] 2> 101197 T170 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 101197 T170 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 101197 T170 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 101197 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 101198 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 101198 T170 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 101198 T170 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 101198 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 101199 T170 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 101199 T170 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 101200 T170 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 101200 T170 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 101200 T170 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 101200 T170 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 101201 T170 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48179/solr
        [junit4] 2> 101201 T170 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 101201 T170 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 101202 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 101204 T213 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@39d1d90f name:ZooKeeperConnection Watcher:127.0.0.1:48179 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 101205 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 101205 T170 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 101207 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 101208 T215 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d8e872 name:ZooKeeperConnection Watcher:127.0.0.1:48179/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 101209 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 101217 T170 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 102225 T170 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48188_iv%2Fdh
        [junit4] 2> 102227 T170 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48188_iv%2Fdh
        [junit4] 2> 102233 T170 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 102246 T170 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 102246 T170 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 103459 T170 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
        [junit4] 2> 103460 T170 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 103462 T170 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:48191
        [junit4] 2> 103463 T170 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 103463 T170 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 103463 T170 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-004
        [junit4] 2> 103464 T170 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-004/'
        [junit4] 2> 103494 T170 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-004/solr.xml
        [junit4] 2> 103550 T170 oasc.CoreContainer.<init> New CoreContainer 2044751973
        [junit4] 2> 103551 T170 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-004/]
        [junit4] 2> 103552 T170 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 103552 T170 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 103552 T170 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 103552 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 103553 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 103553 T170 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 103553 T170 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 103554 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 103554 T170 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 103554 T170 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 103555 T170 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 103555 T170 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 103555 T170 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 103555 T170 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 103556 T170 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48179/solr
        [junit4] 2> 103556 T170 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 103556 T170 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 103557 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 103559 T228 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7df33565 name:ZooKeeperConnection Watcher:127.0.0.1:48179 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 103560 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 103560 T170 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 103562 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 103564 T230 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a9c0d0a name:ZooKeeperConnection Watcher:127.0.0.1:48179/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 103564 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 103573 T170 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 104581 T170 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48191_iv%2Fdh
        [junit4] 2> 104583 T170 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48191_iv%2Fdh
        [junit4] 2> 104589 T170 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 104596 T170 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 104596 T170 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 105850 T170 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
        [junit4] 2> 105850 T170 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 105853 T170 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:48194
        [junit4] 2> 105853 T170 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 105854 T170 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 105854 T170 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-005
        [junit4] 2> 105854 T170 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-005/'
        [junit4] 2> 105884 T170 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-005/solr.xml
        [junit4] 2> 105941 T170 oasc.CoreContainer.<init> New CoreContainer 46096765
        [junit4] 2> 105941 T170 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-005/]
        [junit4] 2> 105942 T170 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 105942 T170 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 105943 T170 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 105943 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 105943 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 105943 T170 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 105944 T170 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 105944 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 105944 T170 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 105944 T170 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 105945 T170 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 105945 T170 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 105946 T170 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 105946 T170 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 105946 T170 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48179/solr
        [junit4] 2> 105946 T170 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 105947 T170 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 105947 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 105950 T243 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@629956f3 name:ZooKeeperConnection Watcher:127.0.0.1:48179 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 105950 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 105950 T170 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 105953 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 105955 T245 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2435968a name:ZooKeeperConnection Watcher:127.0.0.1:48179/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 105955 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 105965 T170 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 106974 T170 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48194_iv%2Fdh
        [junit4] 2> 106976 T170 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48194_iv%2Fdh
        [junit4] 2> 106982 T170 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 106990 T170 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 106990 T170 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 108242 T170 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
        [junit4] 2> 108243 T170 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 108245 T170 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:48197
        [junit4] 2> 108246 T170 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 108246 T170 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 108246 T170 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-006
        [junit4] 2> 108247 T170 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-006/'
        [junit4] 2> 108276 T170 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-006/solr.xml
        [junit4] 2> 108384 T170 oasc.CoreContainer.<init> New CoreContainer 1031298825
        [junit4] 2> 108385 T170 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-006/]
        [junit4] 2> 108386 T170 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 108387 T170 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 108387 T170 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 108388 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 108388 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 108388 T170 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 108389 T170 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 108389 T170 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 108390 T170 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 108390 T170 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 108391 T170 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 108391 T170 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 108392 T170 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 108392 T170 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 108393 T170 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48179/solr
        [junit4] 2> 108393 T170 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 108393 T170 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 108395 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 108397 T258 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29a76766 name:ZooKeeperConnection Watcher:127.0.0.1:48179 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 108398 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 108398 T170 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 108401 T170 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 108403 T260 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b7ca312 name:ZooKeeperConnection Watcher:127.0.0.1:48179/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 108403 T170 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 108412 T170 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 109421 T170 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48197_iv%2Fdh
        [junit4] 2> 109423 T170 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48197_iv%2Fdh
        [junit4] 2> 109429 T170 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 109437 T170 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 109437 T170 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 109452 T205 oasha.CollectionsHandler.handleCreateAction Creating Collection : name=solrj_collection&wt=javabin&replicationFactor=2&version=2&collection.configName=conf1&action=CREATE&numShards=2&router.field=myOwnField
        [junit4] 2> 109454 T201 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 109456 T194 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
        [junit4] 2> "operation":"create",
        [junit4] 2> "fromApi":"true",
        [junit4] 2> "name":"solrj_collection",
        [junit4] 2> "replicationFactor":"2",
        [junit4] 2> "collection.configName":"conf1",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "router.field":"myOwnField"}
        [junit4] 2> 109463 T263 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : create , {
        [junit4] 2> "operation":"create",
        [junit4] 2> "fromApi":"true",
        [junit4] 2> "name":"solrj_collection",
        [junit4] 2> "replicationFactor":"2",
        [junit4] 2> "collection.configName":"conf1",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "router.field":"myOwnField"}
        [junit4] 2> 109470 T263 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection
        [junit4] 2> 109471 T263 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
        [junit4] 2> 109477 T201 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 109483 T193 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
        [junit4] 2> 109484 T193 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
        [junit4] 2> 109484 T193 oasc.Overseer$ClusterStateUpdater.createCollection state version solrj_collection 1
        [junit4] 2> 109489 T216 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 109490 T231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 109490 T246 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 109489 T261 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 109489 T202 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 109489 T201 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 109578 T263 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection solrj_collection, shardNames [shard1, shard2] , replicationFactor : 2
        [junit4] 2> 109579 T263 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:48197_iv%2Fdh
        [junit4] 2> 109580 T263 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:48188_iv%2Fdh
        [junit4] 2> 109582 T263 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:48183_iv%2Fdh
        [junit4] 2> 109583 T263 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:48194_iv%2Fdh
        [junit4] 2> 109585 T207 oasha.CoreAdminHandler.handleCreateAction core create command name=solrj_collection_shard1_replica2&wt=javabin&shard=shard1&collection=solrj_collection&version=2&collection.configName=conf1&action=CREATE&qt=/admin/cores&numShards=2
        [junit4] 2> 109585 T250 oasha.CoreAdminHandler.handleCreateAction core create command name=solrj_collection_shard1_replica1&wt=javabin&shard=shard1&collection=solrj_collection&version=2&collection.configName=conf1&action=CREATE&qt=/admin/cores&numShards=2
        [junit4] 2> 109587 T182 oasha.CoreAdminHandler.handleCreateAction core create command name=solrj_collection_shard2_replica1&wt=javabin&shard=shard2&collection=solrj_collection&version=2&collection.configName=conf1&action=CREATE&qt=/admin/cores&numShards=2
        [junit4] 2> 109588 T235 oasha.CoreAdminHandler.handleCreateAction core create command name=solrj_collection_shard2_replica2&wt=javabin&shard=shard2&collection=solrj_collection&version=2&collection.configName=conf1&action=CREATE&qt=/admin/cores&numShards=2
        [junit4] 2> 109604 T250 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down collection=solrj_collection
        [junit4] 2> 109604 T182 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down collection=solrj_collection
        [junit4] 2> 109604 T235 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down collection=solrj_collection
        [junit4] 2> 109604 T207 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down collection=solrj_collection
        [junit4] 2> 109613 T201 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 109613 T250 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 109613 T235 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 109613 T207 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 109614 T182 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 109625 T193 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "node_name":"127.0.0.1:48197_iv%2Fdh",
        [junit4] 2> "roles":null,
        [junit4] 2> "shard":"shard1",
        [junit4] 2> "core":"solrj_collection_shard1_replica1",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "state":"down",
        [junit4] 2> "base_url":"http://127.0.0.1:48197/iv/dh",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "operation":"state"}
        [junit4] 2> 109637 T193 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "node_name":"127.0.0.1:48188_iv%2Fdh",
        [junit4] 2> "roles":null,
        [junit4] 2> "shard":"shard1",
        [junit4] 2> "core":"solrj_collection_shard1_replica2",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "state":"down",
        [junit4] 2> "base_url":"http://127.0.0.1:48188/iv/dh",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "operation":"state"}
        [junit4] 2> 109641 T201 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 109643 T193 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "node_name":"127.0.0.1:48194_iv%2Fdh",
        [junit4] 2> "roles":null,
        [junit4] 2> "shard":"shard2",
        [junit4] 2> "core":"solrj_collection_shard2_replica2",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "state":"down",
        [junit4] 2> "base_url":"http://127.0.0.1:48194/iv/dh",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "operation":"state"}
        [junit4] 2> 109648 T193 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "node_name":"127.0.0.1:48183_iv%2Fdh",
        [junit4] 2> "roles":null,
        [junit4] 2> "shard":"shard2",
        [junit4] 2> "core":"solrj_collection_shard2_replica1",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "state":"down",
        [junit4] 2> "base_url":"http://127.0.0.1:48183/iv/dh",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "operation":"state"}
        [junit4] 2> 109652 T201 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 109655 T261 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 109655 T231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 109655 T201 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 109655 T202 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 109655 T246 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 109655 T216 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 110615 T207 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
        [junit4] 2> 110615 T250 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
        [junit4] 2> 110615 T235 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
        [junit4] 2> 110616 T250 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 110615 T182 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
        [junit4] 2> 110615 T207 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 110617 T250 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 110617 T182 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 110616 T235 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 110618 T207 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 110619 T207 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 110618 T250 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 110619 T235 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 110619 T182 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 110620 T207 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 110620 T235 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 110620 T250 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 110620 T182 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 110622 T250 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-006/solrj_collection_shard1_replica1/'
        [junit4] 2> 110621 T207 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-003/solrj_collection_shard1_replica2/'
        [junit4] 2> 110623 T182 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 110622 T235 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 110623 T182 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-002/solrj_collection_shard2_replica1/'
        [junit4] 2> 110624 T235 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-005/solrj_collection_shard2_replica2/'
        [junit4] 2> 110689 T250 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 110695 T235 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 110699 T182 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 110714 T207 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 110766 T250 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 110768 T250 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 110775 T182 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 110776 T182 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 110777 T235 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 110779 T235 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 110785 T250 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
        [junit4] 2> 110786 T207 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 110788 T207 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 110792 T182 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
        [junit4] 2> 110796 T235 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
        [junit4] 2> 110802 T207 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
        [junit4] 2> 111275 T182 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 111279 T235 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 111283 T207 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 111289 T182 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 111290 T235 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 111292 T250 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 111293 T182 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 111293 T235 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 111295 T207 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 111298 T207 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 111302 T250 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 111306 T250 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 111324 T235 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 111325 T182 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 111329 T207 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 111330 T235 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 111333 T250 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 111334 T182 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 111335 T207 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 111336 T235 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 111337 T235 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 111340 T235 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 111340 T250 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 111340 T235 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 111342 T235 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 111342 T235 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 111342 T182 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 111343 T235 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using configuration from collection solrj_collection
        [junit4] 2> 111343 T207 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 111344 T235 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 111344 T182 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 111345 T235 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-005/solrj_collection_shard2_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-005/solrj_collection_shard2_replica2/data/
        [junit4] 2> 111346 T207 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 111346 T182 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 111347 T250 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 111347 T207 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 111347 T235 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@d7b3d24
        [junit4] 2> 111348 T207 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 111347 T182 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 111349 T250 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 111350 T250 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 111350 T207 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 111350 T250 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 111350 T182 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 111351 T207 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 111352 T250 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 111352 T182 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 111353 T250 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 111352 T207 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using configuration from collection solrj_collection
        [junit4] 2> 111354 T250 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using configuration from collection solrj_collection
        [junit4] 2> 111353 T182 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using configuration from collection solrj_collection
        [junit4] 2> 111354 T250 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 111354 T207 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 111355 T250 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-006/solrj_collection_shard1_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-006/solrj_collection_shard1_replica1/data/
        [junit4] 2> 111355 T182 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 111357 T207 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-003/solrj_collection_shard1_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-003/solrj_collection_shard1_replica2/data/
        [junit4] 2> 111357 T250 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@d7b3d24
        [junit4] 2> 111358 T207 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@d7b3d24
        [junit4] 2> 111358 T235 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-005/solrj_collection_shard2_replica2/data
        [junit4] 2> 111357 T182 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-002/solrj_collection_shard2_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-002/solrj_collection_shard2_replica1/data/
        [junit4] 2> 111360 T235 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-005/solrj_collection_shard2_replica2/data/index/
        [junit4] 2> 111359 T250 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-006/solrj_collection_shard1_replica1/data
        [junit4] 2> 111360 T235 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-005/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
        [junit4] 2> 111360 T207 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-003/solrj_collection_shard1_replica2/data
        [junit4] 2> 111360 T182 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@d7b3d24
        [junit4] 2> 111362 T207 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-003/solrj_collection_shard1_replica2/data/index/
        [junit4] 2> 111362 T235 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-005/solrj_collection_shard2_replica2/data/index
        [junit4] 2> 111365 T182 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-002/solrj_collection_shard2_replica1/data
        [junit4] 2> 111361 T250 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-006/solrj_collection_shard1_replica1/data/index/
        [junit4] 2> 111365 T182 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-002/solrj_collection_shard2_replica1/data/index/
        [junit4] 2> 111365 T235 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=93.4365234375, floorSegmentMB=1.7080078125, forceMergeDeletesPctAllowed=0.895545927665925, segmentsPerTier=7.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19072077976595292
        [junit4] 2> 111363 T207 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-003/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
        [junit4] 2> 111366 T182 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-002/solrj_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
        [junit4] 2> 111366 T250 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-006/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
        [junit4] 2> 111368 T182 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-002/solrj_collection_shard2_replica1/data/index
        [junit4] 2> 111367 T207 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-003/solrj_collection_shard1_replica2/data/index
        [junit4] 2> 111367 T235 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-67dc87d9)),segFN=segments_1,generation=1}
        [junit4] 2> 111369 T207 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=93.4365234375, floorSegmentMB=1.7080078125, forceMergeDeletesPctAllowed=0.895545927665925, segmentsPerTier=7.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19072077976595292
        [junit4] 2> 111369 T182 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=93.4365234375, floorSegmentMB=1.7080078125, forceMergeDeletesPctAllowed=0.895545927665925, segmentsPerTier=7.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19072077976595292
        [junit4] 2> 111371 T207 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-378cd5fb)),segFN=segments_1,generation=1}
        [junit4] 2> 111368 T250 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-B5A13DAD736935A-001/tempDir-006/solrj_collection_shard1_replica1/data/index
        [junit4] 2> 111371 T182 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-768c5c9c)),segFN=segments_1,generation=1}
        [junit4] 2> 111371 T207 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 111370 T235 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 111372 T182 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 111372 T250 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=93.4365234375, floorSegmentMB=1.7080078125, forceMergeDeletesPctAllowed=0.895545927665925, segmentsPerTier=7.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19072077976595292
        [junit4] 2> 111376 T250 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-584ab273)),segFN=segments_1,generation=1}
        [junit4] 2> 111376 T250 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 111395 T235 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 111396 T182 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 111395 T207 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 111398 T250 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 111470 T235 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 111470 T182 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 111470 T207 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 111470 T250 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 111472 T207 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 111471 T182 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 111471 T235 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 111473 T182 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 111472 T207 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 111472 T250 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 111474 T207 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 111474 T182 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 111473 T235 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 111475 T182 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 111475 T207 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 111476 T182 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 111477 T207 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 111474 T250 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 111477 T207 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 111477 T182 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 111475 T235 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 111478 T182 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 111478 T207 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 111478 T250 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 111479 T235 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 111481 T182 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 111480 T250 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 111481 T235 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 111481 T207 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 111482 T250 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 111482 T235 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 111482 T182 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 111483 T207 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 111483 T235 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 111483 T250 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 111485 T182 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 111485 T207 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 111485 T250 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 111485 T235 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 111486 T182 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 111487 T207 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 111487 T250 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 111487 T235 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 111488 T207 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 111488 T207 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 111488 T182 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 111489 T207 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 111489 T235 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 111489 T250 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 111489 T182 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 111490 T207 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 111491 T182 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 111491 T235 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 111492 T207 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
        [junit4] 2> 111491 T250 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 111492 T182 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 111493 T207 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
        [junit4] 2> 111494 T250 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 111494 T182 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
        [junit4] 2> 111494 T235 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 111495 T207 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
        [junit4] 2> 111495 T250 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 111496 T250 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 111495 T235 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 111497 T250 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 111496 T182 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
        [junit4] 2> 111497 T235 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 111498 T250 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 111498 T182 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
        [junit4] 2> 111499 T235 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 111499 T250 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
        [junit4] 2> 111500 T235 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
        [junit4] 2> 111501 T250 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
        [junit4] 2> 111501 T235 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
        [junit4] 2> 111503 T250 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
        [junit4] 2> 111503 T235 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
        [junit4] 2> 111528 T207 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 111528 T235 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 111528 T250 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 111528 T182 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 111531 T207 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 111531 T235 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 111532 T182 oashl.XMLLoader.init x

    [...truncated too long message...]

    sr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-B5A13DAD736935A-001/tempDir-001/jetty4/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-B5A13DAD736935A-001/tempDir-001/jetty4/index;done=true>>]
        [junit4] 2> 3145634 T4250 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating...
        [junit4] 2> 3145634 T3323 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-B5A13DAD736935A-001/tempDir-001/jetty4/index
        [junit4] 2> 3145634 T3179 oasc.Overseer.close Overseer (id=92697604223991822-127.0.0.1:46372_u%2Fp-n_0000000004) closing
        [junit4] 2> 3145635 T3323 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 1 directories currently being tracked
        [junit4] 2> 3145637 T4250 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 53
        [junit4] 2> 3145637 T3323 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-B5A13DAD736935A-001/tempDir-001/jetty4 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-B5A13DAD736935A-001/tempDir-001/jetty4;done=false>>]
        [junit4] 2> 3145635 T4257 oasc.Overseer$ClusterStateUpdater.amILeader According to ZK I (id=92697604223991822-127.0.0.1:46372_u%2Fp-n_0000000004) am no longer a leader.
        [junit4] 2> 3145638 T3323 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-B5A13DAD736935A-001/tempDir-001/jetty4
        [junit4] 2> 3145638 T4250 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
        [junit4] 2> 3145639 T3323 oasc.LeaderElector$ElectionWatcher.process WARN org.apache.solr.common.SolrException: Fatal Error, SolrCore not found:collection1 in []
        [junit4] 2> at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:222)
        [junit4] 2> at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:162)
        [junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:124)
        [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:313)
        [junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:220)
        [junit4] 2> at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:466)
        [junit4] 2> at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:229)
        [junit4] 2> at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:162)
        [junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:124)
        [junit4] 2> at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
        [junit4] 2> at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:357)
        [junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:266)
        [junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
        [junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
        [junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        [junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        [junit4] 2> at java.lang.Thread.run(Thread.java:745)
        [junit4] 2>
        [junit4] 2> 3145638 T4257 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:46372_u%2Fp
        [junit4] 2> 3145640 T3179 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/u/p,null}
        [junit4] 2> 3150395 T3179 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
        [junit4] 2> 3150398 T3179 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:20444 20444
        [junit4] 2> 3150691 T3180 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:20444 20444
        [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
        [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=DistribDocExpirationUpdateProcessorTest -Dtests.method=testDistribSearch -Dtests.seed=B5A13DAD736935A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=it -Dtests.timezone=Atlantic/Faeroe -Dtests.asserts=true -Dtests.file.encoding=UTF-8
        [junit4] FAILURE 121s J3 | DistribDocExpirationUpdateProcessorTest.testDistribSearch <<<
        [junit4] > Throwable #1: java.lang.AssertionError: There are still nodes recoverying - waited for 30 seconds
        [junit4] > at __randomizedtesting.SeedInfo.seed([B5A13DAD736935A:8ABC9DC2A069F366]:0)
        [junit4] > at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:178)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:840)
        [junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1459)
        [junit4] > at org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest.doTest(DistribDocExpirationUpdateProcessorTest.java:79)
        [junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
        [junit4] > at java.lang.Thread.run(Thread.java:745)
        [junit4] 2> 3150701 T3179 oas.SolrTestCaseJ4.deleteCore ###deleteCore
        [junit4] 2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-B5A13DAD736935A-001
        [junit4] 2> 120968 T3178 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
        [junit4] 2> NOTE: test params are: codec=Asserting(Lucene50): {id=PostingsFormat(name=LuceneVarGapFixedInterval), _version_=FST50}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=it, timezone=Atlantic/Faeroe
        [junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=131202928,total=337641472
        [junit4] 2> NOTE: All tests run in this JVM: [EchoParamsTest, TestOverriddenPrefixQueryForCustomFieldType, SolrIndexSplitterTest, TestSearchPerf, TestInfoStreamLogging, ModifyConfFileTest, DistributedFacetPivotLongTailTest, SolrInfoMBeanTest, TestNonNRTOpen, TermVectorComponentTest, TimeZoneUtilsTest, StandardRequestHandlerTest, CollectionsAPIDistributedZkTest, SuggesterTest, TestHighlightDedupGrouping, ShardSplitTest, FieldMutatingUpdateProcessorTest, TestFieldTypeResource, TestMergePolicyConfig, CollectionsAPIAsyncDistributedZkTest, SOLR749Test, OverseerRolesTest, ReplicationFactorTest, DocExpirationUpdateProcessorFactoryTest, FileBasedSpellCheckerTest, TestAnalyzeInfixSuggestions, DocValuesMultiTest, ResponseLogComponentTest, RemoteQueryErrorTest, SimpleFacetsTest, TestConfigSets, HdfsWriteToMultipleCollectionsTest, BadIndexSchemaTest, TestImplicitCoreProperties, TestShortCircuitedRequests, TestJoin, TestReplicaProperties, HdfsUnloadDistributedZkTest, OutputWriterTest, TestMiniSolrCloudCluster, UUIDFieldTest, TestDistributedMissingSort, SolrPluginUtilsTest, DistribDocExpirationUpdateProcessorTest]
        [junit4] Completed on J3 in 121.78s, 1 test, 1 failure <<< FAILURES!

    [...truncated 671 lines...]
    BUILD FAILED
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:532: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:473: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:189: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:496: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1359: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:966: There were test failures: 438 suites, 1800 tests, 1 error, 1 failure, 54 ignored (9 assumptions)

    Total time: 243 minutes 54 seconds
    Build step 'Invoke Ant' marked build as failure
    Archiving artifacts
    Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #654
    Archived 5 artifacts
    Archive block size is 32768
    Received 0 blocks and 491344509 bytes
    Compression is 0.0%
    Took 1 min 24 sec
    Recording test results
    Email was triggered for: Failure
    Sending email for trigger: Failure
  • Apache Jenkins Server at Oct 28, 2014 at 9:15 pm
    Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/661/

    1 tests failed.
    FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

    Error Message:
    Captured an uncaught exception in thread: Thread[id=28050, name=Thread-5840, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]

    Stack Trace:
    com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=28050, name=Thread-5840, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
    Caused by: java.lang.NullPointerException
      at __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
      at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)




    Build Log:
    [...truncated 12106 lines...]
        [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
        [junit4] 2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/init-core-data-001
        [junit4] 2> 1204771 T27685 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
        [junit4] 2> 1204771 T27685 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
        [junit4] 2> 1204776 T27685 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
        [junit4] 2> 1204776 T27685 oasc.ZkTestServer.run STARTING ZK TEST SERVER
        [junit4] 1> client port:0.0.0.0/0.0.0.0:0
        [junit4] 2> 1204777 T27686 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
        [junit4] 2> 1204877 T27685 oasc.ZkTestServer.run start zk server on port:63966
        [junit4] 2> 1204878 T27685 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1204879 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1204882 T27692 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b69ea46 name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1204882 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1204882 T27685 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1204882 T27685 oascc.SolrZkClient.makePath makePath: /solr
        [junit4] 2> 1204884 T27685 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1204885 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1204885 T27694 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3661390e name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1204886 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1204886 T27685 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1204886 T27685 oascc.SolrZkClient.makePath makePath: /collections/collection1
        [junit4] 2> 1204887 T27685 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
        [junit4] 2> 1204888 T27685 oascc.SolrZkClient.makePath makePath: /collections/control_collection
        [junit4] 2> 1204889 T27685 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
        [junit4] 2> 1204890 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
        [junit4] 2> 1204891 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
        [junit4] 2> 1204893 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
        [junit4] 2> 1204893 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
        [junit4] 2> 1204995 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 1204996 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 1204998 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
        [junit4] 2> 1204998 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
        [junit4] 2> 1205000 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
        [junit4] 2> 1205001 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
        [junit4] 2> 1205003 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
        [junit4] 2> 1205003 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
        [junit4] 2> 1205005 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
        [junit4] 2> 1205006 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
        [junit4] 2> 1205007 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
        [junit4] 2> 1205008 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
        [junit4] 2> 1205010 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
        [junit4] 2> 1205010 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
        [junit4] 2> 1205012 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
        [junit4] 2> 1205013 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
        [junit4] 2> 1205014 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
        [junit4] 2> 1205015 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
        [junit4] 2> 1205017 T27685 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1205017 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1205019 T27696 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@41e8ba68 name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1205019 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1205019 T27685 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> Using old style solr.xml
        [junit4] 2> 1205021 T27685 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1205021 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1205022 T27698 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6f3ce7f0 name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1205023 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1205023 T27685 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1205023 T27685 oascc.SolrZkClient.makePath makePath: /solr
        [junit4] 2> 1205024 T27685 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1205025 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1205026 T27700 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@67c9879b name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1205026 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1205026 T27685 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1205028 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf2/solrconfig.xml
        [junit4] 2> 1205029 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.xml
        [junit4] 2> 1205031 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
        [junit4] 2> 1205032 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf2/schema.xml
        [junit4] 2> 1205134 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 1205134 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
        [junit4] 2> 1205136 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
        [junit4] 2> 1205137 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf2/stopwords.txt
        [junit4] 2> 1205138 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
        [junit4] 2> 1205139 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf2/protwords.txt
        [junit4] 2> 1205140 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
        [junit4] 2> 1205141 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf2/currency.xml
        [junit4] 2> 1205142 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
        [junit4] 2> 1205143 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf2/open-exchange-rates.json
        [junit4] 2> 1205144 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
        [junit4] 2> 1205145 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
        [junit4] 2> 1205146 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
        [junit4] 2> 1205147 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf2/old_synonyms.txt
        [junit4] 2> 1205148 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
        [junit4] 2> 1205149 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf2/synonyms.txt
        [junit4] 2> 1205150 T27685 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
        [junit4] 2> 1205151 T27685 oascc.SolrZkClient.makePath makePath: /configs/conf2/elevate.xml
        [junit4] 2> 1206445 T27685 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 1206449 T27685 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
        [junit4] 2> 1206454 T27685 oejs.AbstractConnector.doStart Started sslsocketconnector@127.0.0.1:63972
        [junit4] 2> 1206456 T27685 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 1206456 T27685 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 1206457 T27685 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-002
        [junit4] 2> 1206457 T27685 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-002/'
        [junit4] 2> 1206488 T27685 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-002/solr.xml
        [junit4] 2> 1206573 T27685 oasc.CoreContainer.<init> New CoreContainer 480716668
        [junit4] 2> 1206573 T27685 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-002/]
        [junit4] 2> 1206575 T27685 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 1206575 T27685 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 1206575 T27685 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 1206576 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 1206576 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 1206576 T27685 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 1206577 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 1206577 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 1206578 T27685 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 1206578 T27685 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 1206581 T27685 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 1206583 T27685 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 1206583 T27685 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 1206584 T27685 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 1206584 T27685 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63966/solr
        [junit4] 2> 1206584 T27685 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 1206585 T27685 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1206586 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1206587 T27711 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@181b9f03 name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1206588 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1206589 T27685 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1206590 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1206591 T27713 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@63a586f5 name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1206592 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1206593 T27685 oascc.SolrZkClient.makePath makePath: /overseer/queue
        [junit4] 2> 1206595 T27685 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
        [junit4] 2> 1206597 T27685 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
        [junit4] 2> 1206599 T27685 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
        [junit4] 2> 1206601 T27685 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
        [junit4] 2> 1206603 T27685 oascc.SolrZkClient.makePath makePath: /live_nodes
        [junit4] 2> 1206604 T27685 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63972_
        [junit4] 2> 1206605 T27685 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63972_
        [junit4] 2> 1206606 T27685 oascc.SolrZkClient.makePath makePath: /overseer_elect
        [junit4] 2> 1206607 T27685 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
        [junit4] 2> 1206609 T27685 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 1206610 T27685 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:63972_
        [junit4] 2> 1206611 T27685 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
        [junit4] 2> 1206612 T27685 oasc.Overseer.start Overseer (id=92702442030432262-127.0.0.1:63972_-n_0000000000) starting
        [junit4] 2> 1206614 T27685 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
        [junit4] 2> 1206619 T27685 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
        [junit4] 2> 1206620 T27715 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
        [junit4] 2> 1206621 T27685 oascc.SolrZkClient.makePath makePath: /clusterstate.json
        [junit4] 2> 1206623 T27685 oascc.SolrZkClient.makePath makePath: /aliases.json
        [junit4] 2> 1206624 T27685 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 1206626 T27714 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
        [junit4] 2> 1206630 T27685 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 1206630 T27685 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 1206635 T27685 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1206635 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1206637 T27719 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@41205c8b name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1206637 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1206637 T27685 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1206638 T27685 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 1206640 T27685 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
        [junit4] 2> 1207706 T27685 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
        [junit4] 2> 1207707 T27685 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 1207710 T27685 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
        [junit4] 2> 1207712 T27685 oejs.AbstractConnector.doStart Started sslsocketconnector@127.0.0.1:63976
        [junit4] 2> 1207714 T27685 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 1207714 T27685 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 1207715 T27685 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003
        [junit4] 2> 1207715 T27685 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/'
        [junit4] 2> 1207746 T27685 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solr.xml
        [junit4] 2> 1207804 T27685 oasc.CoreContainer.<init> New CoreContainer 1821520330
        [junit4] 2> 1207804 T27685 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/]
        [junit4] 2> 1207805 T27685 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 1207805 T27685 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 1207805 T27685 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 1207805 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 1207806 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 1207806 T27685 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 1207806 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 1207806 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 1207807 T27685 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 1207807 T27685 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 1207809 T27685 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 1207810 T27685 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 1207810 T27685 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 1207811 T27685 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 1207811 T27685 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63966/solr
        [junit4] 2> 1207811 T27685 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 1207811 T27685 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1207812 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1207813 T27730 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d5684a6 name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1207814 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1207815 T27685 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1207816 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1207817 T27732 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16a0a9be name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1207817 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1207820 T27685 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 1208822 T27685 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63976_
        [junit4] 2> 1208823 T27685 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63976_
        [junit4] 2> 1208827 T27685 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 1208832 T27685 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 1208832 T27685 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 1210182 T27685 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
        [junit4] 2> 1210184 T27685 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 1210188 T27685 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
        [junit4] 2> 1210189 T27685 oejs.AbstractConnector.doStart Started sslsocketconnector@127.0.0.1:63979
        [junit4] 2> 1210191 T27685 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 1210192 T27685 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 1210192 T27685 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004
        [junit4] 2> 1210192 T27685 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/'
        [junit4] 2> 1210222 T27685 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solr.xml
        [junit4] 2> 1210279 T27685 oasc.CoreContainer.<init> New CoreContainer 1318011298
        [junit4] 2> 1210280 T27685 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/]
        [junit4] 2> 1210281 T27685 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 1210281 T27685 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 1210281 T27685 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 1210281 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 1210281 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 1210282 T27685 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 1210282 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 1210282 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 1210282 T27685 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 1210283 T27685 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 1210284 T27685 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 1210286 T27685 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 1210286 T27685 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 1210286 T27685 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 1210287 T27685 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63966/solr
        [junit4] 2> 1210287 T27685 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 1210287 T27685 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1210288 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1210290 T27747 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e86c09a name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1210291 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1210292 T27685 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1210294 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1210295 T27749 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22acd0e4 name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1210295 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1210299 T27685 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 1211302 T27685 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63979_
        [junit4] 2> 1211303 T27685 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63979_
        [junit4] 2> 1211306 T27685 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 1211311 T27685 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 1211311 T27685 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 1212478 T27685 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
        [junit4] 2> 1212478 T27685 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 1212482 T27685 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
        [junit4] 2> 1212483 T27685 oejs.AbstractConnector.doStart Started sslsocketconnector@127.0.0.1:63982
        [junit4] 2> 1212485 T27685 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 1212486 T27685 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 1212486 T27685 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005
        [junit4] 2> 1212528 T27685 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/'
        [junit4] 2> 1212559 T27685 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solr.xml
        [junit4] 2> 1212615 T27685 oasc.CoreContainer.<init> New CoreContainer 1213264595
        [junit4] 2> 1212616 T27685 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/]
        [junit4] 2> 1212617 T27685 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 1212617 T27685 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 1212617 T27685 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 1212617 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 1212617 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 1212618 T27685 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 1212618 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 1212618 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 1212618 T27685 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 1212619 T27685 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 1212620 T27685 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 1212622 T27685 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 1212622 T27685 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 1212622 T27685 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 1212623 T27685 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63966/solr
        [junit4] 2> 1212623 T27685 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 1212623 T27685 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1212624 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1212625 T27762 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@439a3bd0 name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1212626 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1212626 T27685 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1212628 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1212628 T27764 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7ec3410d name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1212629 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1212632 T27685 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 1213635 T27685 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63982_
        [junit4] 2> 1213636 T27685 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63982_
        [junit4] 2> 1213639 T27685 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 1213643 T27685 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 1213643 T27685 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 1214950 T27685 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
        [junit4] 2> 1214950 T27685 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 1214954 T27685 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
        [junit4] 2> 1214955 T27685 oejs.AbstractConnector.doStart Started sslsocketconnector@127.0.0.1:63988
        [junit4] 2> 1214958 T27685 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 1214958 T27685 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 1214958 T27685 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006
        [junit4] 2> 1214958 T27685 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/'
        [junit4] 2> 1214991 T27685 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solr.xml
        [junit4] 2> 1215051 T27685 oasc.CoreContainer.<init> New CoreContainer 1019796720
        [junit4] 2> 1215052 T27685 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/]
        [junit4] 2> 1215053 T27685 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
        [junit4] 2> 1215053 T27685 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
        [junit4] 2> 1215053 T27685 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 1215053 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 1215054 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 1215054 T27685 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 1215054 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 1215054 T27685 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 1215055 T27685 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 1215055 T27685 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 1215057 T27685 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
        [junit4] 2> 1215058 T27685 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 1215059 T27685 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 1215059 T27685 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 1215059 T27685 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63966/solr
        [junit4] 2> 1215059 T27685 oasc.ZkController.checkChrootPath zkHost includes chroot
        [junit4] 2> 1215060 T27685 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
        [junit4] 2> 1215060 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1215061 T27777 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e421349 name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1215062 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1215063 T27685 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
        [junit4] 2> 1215064 T27685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
        [junit4] 2> 1215065 T27779 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75ae54ac name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
        [junit4] 2> 1215065 T27685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
        [junit4] 2> 1215069 T27685 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
        [junit4] 2> 1216071 T27685 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63988_
        [junit4] 2> 1216072 T27685 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63988_
        [junit4] 2> 1216075 T27685 oasc.Overseer.close Overseer (id=null) closing
        [junit4] 2> 1216079 T27685 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
        [junit4] 2> 1216080 T27685 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 1216200 T27723 oasha.CollectionsHandler.handleCreateAction Creating Collection : action=CREATE&wt=javabin&numShards=2&collection.configName=conf1&version=2&name=solrj_collection&router.field=myOwnField&replicationFactor=2
        [junit4] 2> 1216202 T27735 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 1216202 T27715 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
        [junit4] 2> "operation":"create",
        [junit4] 2> "fromApi":"true",
        [junit4] 2> "name":"solrj_collection",
        [junit4] 2> "replicationFactor":"2",
        [junit4] 2> "collection.configName":"conf1",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "router.field":"myOwnField"}
        [junit4] 2> 1216204 T27783 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : create , {
        [junit4] 2> "operation":"create",
        [junit4] 2> "fromApi":"true",
        [junit4] 2> "name":"solrj_collection",
        [junit4] 2> "replicationFactor":"2",
        [junit4] 2> "collection.configName":"conf1",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "router.field":"myOwnField"}
        [junit4] 2> 1216205 T27783 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection
        [junit4] 2> 1216205 T27783 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
        [junit4] 2> 1216207 T27735 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 1216209 T27714 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
        [junit4] 2> 1216209 T27714 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
        [junit4] 2> 1216209 T27714 oasc.Overseer$ClusterStateUpdater.createCollection state version solrj_collection 1
        [junit4] 2> 1216211 T27734 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 1216211 T27735 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 1216211 T27765 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 1216211 T27750 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 1216211 T27733 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 1216211 T27780 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 1216307 T27783 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection solrj_collection, shardNames [shard1, shard2] , replicationFactor : 2
        [junit4] 2> 1216308 T27783 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:63988_
        [junit4] 2> 1216309 T27783 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:63976_
        [junit4] 2> 1216311 T27783 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:63979_
        [junit4] 2> 1216314 T27783 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:63982_
        [junit4] 2> 1216432 T27743 oasha.CoreAdminHandler.handleCreateAction core create command action=CREATE&wt=javabin&numShards=2&collection.configName=conf1&version=2&name=solrj_collection_shard2_replica1&collection=solrj_collection&qt=/admin/cores&shard=shard2
        [junit4] 2> 1216434 T27743 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down collection=solrj_collection
        [junit4] 2> 1216435 T27743 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 1216435 T27735 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 1216438 T27724 oasha.CoreAdminHandler.handleCreateAction core create command action=CREATE&wt=javabin&numShards=2&collection.configName=conf1&version=2&name=solrj_collection_shard1_replica2&collection=solrj_collection&qt=/admin/cores&shard=shard1
        [junit4] 2> 1216438 T27769 oasha.CoreAdminHandler.handleCreateAction core create command action=CREATE&wt=javabin&numShards=2&collection.configName=conf1&version=2&name=solrj_collection_shard1_replica1&collection=solrj_collection&qt=/admin/cores&shard=shard1
        [junit4] 2> 1216438 T27714 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "base_url":"https://127.0.0.1:63979",
        [junit4] 2> "operation":"state",
        [junit4] 2> "core":"solrj_collection_shard2_replica1",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "state":"down",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "roles":null,
        [junit4] 2> "shard":"shard2",
        [junit4] 2> "node_name":"127.0.0.1:63979_"}
        [junit4] 2> 1216441 T27724 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down collection=solrj_collection
        [junit4] 2> 1216442 T27724 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 1216442 T27754 oasha.CoreAdminHandler.handleCreateAction core create command action=CREATE&wt=javabin&numShards=2&collection.configName=conf1&version=2&name=solrj_collection_shard2_replica2&collection=solrj_collection&qt=/admin/cores&shard=shard2
        [junit4] 2> 1216442 T27769 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down collection=solrj_collection
        [junit4] 2> 1216442 T27735 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 1216443 T27769 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 1216444 T27714 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "base_url":"https://127.0.0.1:63976",
        [junit4] 2> "operation":"state",
        [junit4] 2> "core":"solrj_collection_shard1_replica2",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "state":"down",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "roles":null,
        [junit4] 2> "shard":"shard1",
        [junit4] 2> "node_name":"127.0.0.1:63976_"}
        [junit4] 2> 1216445 T27754 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down collection=solrj_collection
        [junit4] 2> 1216446 T27735 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 1216446 T27714 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "base_url":"https://127.0.0.1:63988",
        [junit4] 2> "operation":"state",
        [junit4] 2> "core":"solrj_collection_shard1_replica1",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "state":"down",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "roles":null,
        [junit4] 2> "shard":"shard1",
        [junit4] 2> "node_name":"127.0.0.1:63988_"}
        [junit4] 2> 1216446 T27754 oasc.ZkController.waitForCoreNodeName look for our core node name
        [junit4] 2> 1216447 T27714 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
        [junit4] 2> "base_url":"https://127.0.0.1:63982",
        [junit4] 2> "operation":"state",
        [junit4] 2> "core":"solrj_collection_shard2_replica2",
        [junit4] 2> "numShards":"2",
        [junit4] 2> "state":"down",
        [junit4] 2> "collection":"solrj_collection",
        [junit4] 2> "roles":null,
        [junit4] 2> "shard":"shard2",
        [junit4] 2> "node_name":"127.0.0.1:63982_"}
        [junit4] 2> 1216448 T27735 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
        [junit4] 2> 1216449 T27735 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 1216449 T27733 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 1216449 T27734 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 1216449 T27750 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 1216449 T27765 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 1216449 T27780 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
        [junit4] 2> 1217436 T27743 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
        [junit4] 2> 1217437 T27743 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 1217437 T27743 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 1217438 T27743 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 1217438 T27743 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 1217438 T27743 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/'
        [junit4] 2> 1217444 T27724 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
        [junit4] 2> 1217444 T27724 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 1217444 T27769 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
        [junit4] 2> 1217445 T27769 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 1217445 T27724 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 1217445 T27724 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 1217445 T27769 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 1217445 T27769 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 1217446 T27724 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 1217446 T27724 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/'
        [junit4] 2> 1217446 T27769 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 1217446 T27769 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/'
        [junit4] 2> 1217446 T27754 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
        [junit4] 2> 1217447 T27754 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
        [junit4] 2> 1217448 T27754 oasc.ZkController.createCollectionZkNode Collection zkNode exists
        [junit4] 2> 1217448 T27754 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
        [junit4] 2> 1217449 T27754 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
        [junit4] 2> 1217450 T27754 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/'
        [junit4] 2> 1217538 T27769 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 1217539 T27743 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 1217557 T27724 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 1217613 T27754 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 1217627 T27743 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 1217633 T27769 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 1217645 T27724 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 1217653 T27754 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 1217728 T27743 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 1217734 T27769 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 1217742 T27743 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
        [junit4] 2> 1217746 T27724 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 1217747 T27769 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
        [junit4] 2> 1217754 T27754 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
        [junit4] 2> 1217756 T27724 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
        [junit4] 2> 1217807 T27754 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
        [junit4] 2> 1218411 T27724 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 1218426 T27724 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 1218438 T27724 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 1218440 T27743 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 1218444 T27754 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 1218448 T27743 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 1218450 T27724 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 1218451 T27754 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 1218451 T27743 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 1218455 T27754 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 1218455 T27724 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 1218460 T27724 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 1218461 T27724 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 1218464 T27724 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 1218464 T27724 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 1218465 T27743 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 1218465 T27724 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 1218466 T27724 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 1218466 T27724 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using configuration from collection solrj_collection
        [junit4] 2> 1218466 T27724 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 1218467 T27724 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/data/
        [junit4] 2> 1218467 T27724 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3eb6814a
        [junit4] 2> 1218469 T27754 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 1218470 T27743 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 1218470 T27724 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/data
        [junit4] 2> 1218472 T27724 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/data/index/
        [junit4] 2> 1218472 T27724 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
        [junit4] 2> 1218473 T27769 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
        [junit4] 2> 1218473 T27724 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/data/index
        [junit4] 2> 1218474 T27724 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=97.9267578125, floorSegmentMB=1.2958984375, forceMergeDeletesPctAllowed=11.900176490131441, segmentsPerTier=49.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
        [junit4] 2> 1218474 T27754 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 1218475 T27724 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-7c885a91)),segFN=segments_1,generation=1}
        [junit4] 2> 1218475 T27743 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 1218475 T27724 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 1218476 T27743 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 1218477 T27743 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 1218477 T27743 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 1218478 T27743 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 1218478 T27743 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 1218479 T27743 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using configuration from collection solrj_collection
        [junit4] 2> 1218479 T27743 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 1218479 T27743 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/data/
        [junit4] 2> 1218480 T27769 oass.IndexSchema.readSchema default search field in schema is text
        [junit4] 2> 1218480 T27754 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 1218480 T27743 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3eb6814a
        [junit4] 2> 1218481 T27724 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 1218481 T27754 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 1218482 T27724 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 1218482 T27754 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 1218483 T27724 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 1218483 T27769 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 1218483 T27754 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 1218484 T27743 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/data
        [junit4] 2> 1218483 T27724 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 1218485 T27754 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 1218485 T27743 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/data/index/
        [junit4] 2> 1218485 T27724 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 1218486 T27743 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
        [junit4] 2> 1218486 T27754 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 1218487 T27743 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/data/index
        [junit4] 2> 1218486 T27724 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 1218488 T27743 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=97.9267578125, floorSegmentMB=1.2958984375, forceMergeDeletesPctAllowed=11.900176490131441, segmentsPerTier=49.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
        [junit4] 2> 1218487 T27754 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using configuration from collection solrj_collection
        [junit4] 2> 1218488 T27724 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 1218489 T27743 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-68b86762)),segFN=segments_1,generation=1}
        [junit4] 2> 1218489 T27754 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 1218490 T27743 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 1218489 T27724 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 1218490 T27754 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/data/
        [junit4] 2> 1218491 T27724 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 1218491 T27754 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3eb6814a
        [junit4] 2> 1218492 T27724 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1218493 T27724 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1218494 T27724 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1218494 T27724 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 1218495 T27754 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/data
        [junit4] 2> 1218495 T27769 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 1218495 T27743 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 1218496 T27754 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/data/index/
        [junit4] 2> 1218495 T27724 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 1218496 T27754 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
        [junit4] 2> 1218496 T27743 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 1218497 T27724 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 1218498 T27743 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 1218498 T27754 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/data/index
        [junit4] 2> 1218498 T27743 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 1218499 T27754 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=97.9267578125, floorSegmentMB=1.2958984375, forceMergeDeletesPctAllowed=11.900176490131441, segmentsPerTier=49.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
        [junit4] 2> 1218498 T27724 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 1218499 T27743 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 1218500 T27724 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 1218501 T27754 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-5c32ed3a)),segFN=segments_1,generation=1}
        [junit4] 2> 1218500 T27743 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 1218501 T27769 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
        [junit4] 2> 1218502 T27743 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 1218501 T27754 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 1218501 T27724 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
        [junit4] 2> 1218502 T27743 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 1218503 T27724 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
        [junit4] 2> 1218503 T27743 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 1218504 T27724 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
        [junit4] 2> 1218504 T27743 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1218505 T27743 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1218506 T27743 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1218506 T27743 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 1218507 T27743 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 1218507 T27743 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 1218507 T27769 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 1218508 T27743 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 1218508 T27754 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 1218509 T27769 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 1218509 T27754 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 1218509 T27743 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 1218510 T27754 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 1218509 T27769 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 1218510 T27754 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 1218510 T27743 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
        [junit4] 2> 1218511 T27754 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 1218511 T27743 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
        [junit4] 2> 1218510 T27769 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
        [junit4] 2> 1218512 T27743 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
        [junit4] 2> 1218512 T27754 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 1218513 T27769 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
        [junit4] 2> 1218513 T27754 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 1218513 T27769 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
        [junit4] 2> 1218514 T27754 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 1218514 T27769 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using configuration from collection solrj_collection
        [junit4] 2> 1218514 T27754 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 1218515 T27769 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 1218515 T27769 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/data/
        [junit4] 2> 1218515 T27754 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1218516 T27769 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3eb6814a
        [junit4] 2> 1218516 T27754 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1218517 T27754 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1218518 T27754 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 1218518 T27754 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
        [junit4] 2> 1218519 T27754 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
        [junit4] 2> 1218519 T27754 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 1218520 T27754 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 1218520 T27754 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
        [junit4] 2> 1218521 T27769 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/data
        [junit4] 2> 1218521 T27754 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
        [junit4] 2> 1218522 T27754 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
        [junit4] 2> 1218523 T27769 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/data/index/
        [junit4] 2> 1218523 T27769 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
        [junit4] 2> 1218524 T27769 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/data/index
        [junit4] 2> 1218525 T27769 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=97.9267578125, floorSegmentMB=1.2958984375, forceMergeDeletesPctAllowed=11.900176490131441, segmentsPerTier=49.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
        [junit4] 2> 1218527 T27769 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-21a88cd6)),segFN=segments_1,generation=1}
        [junit4] 2> 1218527 T27769 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 1218533 T27769 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
        [junit4] 2> 1218533 T27724 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1218533 T27754 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1218533 T27769 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
        [junit4] 2> 1218533 T27743 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1218534 T27769 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
        [junit4] 2> 1218535 T27769 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
        [junit4] 2> 1218535 T27769 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
        [junit4] 2> 1218536 T27769 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
        [junit4] 2> 1218536 T27754 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1218536 T27724 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1218537 T27743 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1218537 T27769 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
        [junit4] 2> 1218538 T27769 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-t

    [...truncated too long message...]

    home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/addReplicaColl_shard2_replica3/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/addReplicaColl_shard2_replica3/data;done=false>>]
        [junit4] 2> 1369735 T27685 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/addReplicaColl_shard2_replica3/data
        [junit4] 2> 1369735 T27685 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/addReplicaColl_shard2_replica3/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/addReplicaColl_shard2_replica3/data/index;done=false>>]
        [junit4] 2> 1369736 T27685 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/addReplicaColl_shard2_replica3/data/index
        [junit4] 2> 1369737 T27685 oasc.Overseer.close Overseer (id=92702442030432271-127.0.0.1:63988_-n_0000000004) closing
        [junit4] 2> 1369738 T29287 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:63988_
        [junit4] 2> 1369751 T27976 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
        [junit4] 2> 1369756 T27685 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
        [junit4] 2> 1370474 T27685 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 63972
        [junit4] 2> 1370475 T27685 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
        [junit4] 2> 1370477 T27685 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:63966 63966
        [junit4] 2> 1370659 T27969 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:63966 63966
        [junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
        [junit4] 2> NOTE: reproduce with: ant test -Dtestcase=CollectionsAPIDistributedZkTest -Dtests.method=testDistribSearch -Dtests.seed=901F7D855C61C262 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=pt_BR -Dtests.timezone=Pacific/Norfolk -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
        [junit4] ERROR 166s J3 | CollectionsAPIDistributedZkTest.testDistribSearch <<<
        [junit4] > Throwable #1: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=28050, name=Thread-5840, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
        [junit4] > Caused by: java.lang.NullPointerException
        [junit4] > at __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
        [junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=28048, name=Thread-5838, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
        [junit4] > Caused by: java.lang.NullPointerException
        [junit4] > at __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
        [junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)Throwable #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=28049, name=Thread-5839, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
        [junit4] > Caused by: java.lang.NullPointerException
        [junit4] > at __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
        [junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=28053, name=Thread-5843, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
        [junit4] > Caused by: java.lang.NullPointerException
        [junit4] > at __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
        [junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)Throwable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=28051, name=Thread-5841, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
        [junit4] > Caused by: java.lang.NullPointerException
        [junit4] > at __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
        [junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)Throwable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=28052, name=Thread-5842, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
        [junit4] > Caused by: java.lang.NullPointerException
        [junit4] > at __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
        [junit4] > at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)
        [junit4] 2> 1370705 T27685 oas.SolrTestCaseJ4.deleteCore ###deleteCore
        [junit4] 2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval), _version_=BlockTreeOrds(blocksize=128), intDefault=BlockTreeOrds(blocksize=128), range_facet_l=PostingsFormat(name=LuceneVarGapFixedInterval), other_tl1=BlockTreeOrds(blocksize=128), a_i1=PostingsFormat(name=LuceneVarGapFixedInterval), timestamp=PostingsFormat(name=LuceneVarGapFixedInterval), text=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=LuceneVarGapFixedInterval), id=FST50}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=DefaultSimilarity, locale=pt_BR, timezone=Pacific/Norfolk
        [junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=211097640,total=522190848
        [junit4] 2> NOTE: All tests run in this JVM: [StatelessScriptUpdateProcessorFactoryTest, IndexBasedSpellCheckerTest, TestSearcherReuse, TestMergePolicyConfig, DocumentAnalysisRequestHandlerTest, SpellPossibilityIteratorTest, TestFastLRUCache, AliasIntegrationTest, SolrInfoMBeanTest, TestSearchPerf, HdfsWriteToMultipleCollectionsTest, TestLMJelinekMercerSimilarityFactory, LoggingHandlerTest, CollectionsAPIAsyncDistributedZkTest, TimeZoneUtilsTest, TestPhraseSuggestions, TestPHPSerializedResponseWriter, DOMUtilTest, TestCSVLoader, TestNonNRTOpen, SOLR749Test, TestIndexSearcher, TestOverriddenPrefixQueryForCustomFieldType, DistributedDebugComponentTest, UniqFieldsUpdateProcessorFactoryTest, TestMiniSolrCloudCluster, TestDistributedMissingSort, TestReplicaProperties, CoreContainerCoreInitFailuresTest, TestInfoStreamLogging, FileBasedSpellCheckerTest, TestDistributedGrouping, TestLRUCache, DistributedFacetPivotLongTailTest, ShardRoutingTest, TestAnalyzeInfixSuggestions, TestConfigSets, RequestHandlersTest, CursorPagingTest, ResponseLogComponentTest, TermVectorComponentDistributedTest, EchoParamsTest, TestModifyConfFiles, OverseerStatusTest, FieldMutatingUpdateProcessorTest, UnloadDistributedZkTest, ZkCLITest, TestDocSet, DocExpirationUpdateProcessorFactoryTest, AtomicUpdatesTest, OutputWriterTest, TestSort, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest]
        [junit4] Completed on J3 in 170.41s, 1 test, 1 error <<< FAILURES!

    [...truncated 850 lines...]
    BUILD FAILED
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:532: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:473: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:189: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:496: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1359: The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:966: There were test failures: 438 suites, 1800 tests, 1 error, 54 ignored (9 assumptions)

    Total time: 224 minutes 20 seconds
    Build step 'Invoke Ant' marked build as failure
    Archiving artifacts
    Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #654
    Archived 5 artifacts
    Archive block size is 32768
    Received 0 blocks and 419876900 bytes
    Compression is 0.0%
    Took 1 min 15 sec
    Recording test results
    Email was triggered for: Failure
    Sending email for trigger: Failure
  • Shalin Shekhar Mangar at Oct 29, 2014 at 8:39 am
    My bad. I introduced a failure which happens only on nightly runs so I
    didn't catch it in my local test run. I have committed a fix.
    On Wed, Oct 29, 2014 at 2:44 AM, Apache Jenkins Server wrote:

    Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/661/

    1 tests failed.
    FAILED:
    org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

    Error Message:
    Captured an uncaught exception in thread: Thread[id=28050,
    name=Thread-5840, state=RUNNABLE,
    group=TGRP-CollectionsAPIDistributedZkTest]

    Stack Trace:
    com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
    uncaught exception in thread: Thread[id=28050, name=Thread-5840,
    state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
    Caused by: java.lang.NullPointerException
    at __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
    at
    org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)




    Build Log:
    [...truncated 12106 lines...]
    [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
    [junit4] 2> Creating dataDir:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/init-core-data-001
    [junit4] 2> 1204771 T27685 oas.SolrTestCaseJ4.buildSSLConfig
    Randomized ssl (true) and clientAuth (false)
    [junit4] 2> 1204771 T27685
    oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext
    system property: /
    [junit4] 2> 1204776 T27685 oas.SolrTestCaseJ4.setUp ###Starting
    testDistribSearch
    [junit4] 2> 1204776 T27685 oasc.ZkTestServer.run STARTING ZK TEST
    SERVER
    [junit4] 1> client port:0.0.0.0/0.0.0.0:0
    [junit4] 2> 1204777 T27686
    oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
    [junit4] 2> 1204877 T27685 oasc.ZkTestServer.run start zk server on
    port:63966
    [junit4] 2> 1204878 T27685
    oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default
    ZkCredentialsProvider
    [junit4] 2> 1204879 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1204882 T27692 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@b69ea46
    name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent
    state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1204882 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1204882 T27685 oascc.SolrZkClient.createZkACLProvider
    Using default ZkACLProvider
    [junit4] 2> 1204882 T27685 oascc.SolrZkClient.makePath makePath: /solr
    [junit4] 2> 1204884 T27685
    oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default
    ZkCredentialsProvider
    [junit4] 2> 1204885 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1204885 T27694 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@3661390e
    name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event
    WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1204886 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1204886 T27685 oascc.SolrZkClient.createZkACLProvider
    Using default ZkACLProvider
    [junit4] 2> 1204886 T27685 oascc.SolrZkClient.makePath makePath:
    /collections/collection1
    [junit4] 2> 1204887 T27685 oascc.SolrZkClient.makePath makePath:
    /collections/collection1/shards
    [junit4] 2> 1204888 T27685 oascc.SolrZkClient.makePath makePath:
    /collections/control_collection
    [junit4] 2> 1204889 T27685 oascc.SolrZkClient.makePath makePath:
    /collections/control_collection/shards
    [junit4] 2> 1204890 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
    to /configs/conf1/solrconfig.xml
    [junit4] 2> 1204891 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf1/solrconfig.xml
    [junit4] 2> 1204893 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
    to /configs/conf1/schema.xml
    [junit4] 2> 1204893 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf1/schema.xml
    [junit4] 2> 1204995 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
    to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
    [junit4] 2> 1204996 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf1/solrconfig.snippet.randomindexconfig.xml
    [junit4] 2> 1204998 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
    to /configs/conf1/stopwords.txt
    [junit4] 2> 1204998 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf1/stopwords.txt
    [junit4] 2> 1205000 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt
    to /configs/conf1/protwords.txt
    [junit4] 2> 1205001 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf1/protwords.txt
    [junit4] 2> 1205003 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml
    to /configs/conf1/currency.xml
    [junit4] 2> 1205003 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf1/currency.xml
    [junit4] 2> 1205005 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
    to /configs/conf1/enumsConfig.xml
    [junit4] 2> 1205006 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf1/enumsConfig.xml
    [junit4] 2> 1205007 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
    to /configs/conf1/open-exchange-rates.json
    [junit4] 2> 1205008 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf1/open-exchange-rates.json
    [junit4] 2> 1205010 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
    to /configs/conf1/mapping-ISOLatin1Accent.txt
    [junit4] 2> 1205010 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf1/mapping-ISOLatin1Accent.txt
    [junit4] 2> 1205012 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
    to /configs/conf1/old_synonyms.txt
    [junit4] 2> 1205013 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf1/old_synonyms.txt
    [junit4] 2> 1205014 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
    to /configs/conf1/synonyms.txt
    [junit4] 2> 1205015 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf1/synonyms.txt
    [junit4] 2> 1205017 T27685
    oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default
    ZkCredentialsProvider
    [junit4] 2> 1205017 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1205019 T27696 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@41e8ba68
    name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event
    WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1205019 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1205019 T27685 oascc.SolrZkClient.createZkACLProvider
    Using default ZkACLProvider
    [junit4] 2> Using old style solr.xml
    [junit4] 2> 1205021 T27685
    oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default
    ZkCredentialsProvider
    [junit4] 2> 1205021 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1205022 T27698 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@6f3ce7f0
    name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent
    state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1205023 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1205023 T27685 oascc.SolrZkClient.createZkACLProvider
    Using default ZkACLProvider
    [junit4] 2> 1205023 T27685 oascc.SolrZkClient.makePath makePath: /solr
    [junit4] 2> 1205024 T27685
    oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default
    ZkCredentialsProvider
    [junit4] 2> 1205025 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1205026 T27700 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@67c9879b
    name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event
    WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1205026 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1205026 T27685 oascc.SolrZkClient.createZkACLProvider
    Using default ZkACLProvider
    [junit4] 2> 1205028 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
    to /configs/conf2/solrconfig.xml
    [junit4] 2> 1205029 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf2/solrconfig.xml
    [junit4] 2> 1205031 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
    to /configs/conf2/schema.xml
    [junit4] 2> 1205032 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf2/schema.xml
    [junit4] 2> 1205134 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
    to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
    [junit4] 2> 1205134 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf2/solrconfig.snippet.randomindexconfig.xml
    [junit4] 2> 1205136 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
    to /configs/conf2/stopwords.txt
    [junit4] 2> 1205137 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf2/stopwords.txt
    [junit4] 2> 1205138 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt
    to /configs/conf2/protwords.txt
    [junit4] 2> 1205139 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf2/protwords.txt
    [junit4] 2> 1205140 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml
    to /configs/conf2/currency.xml
    [junit4] 2> 1205141 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf2/currency.xml
    [junit4] 2> 1205142 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
    to /configs/conf2/open-exchange-rates.json
    [junit4] 2> 1205143 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf2/open-exchange-rates.json
    [junit4] 2> 1205144 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
    to /configs/conf2/mapping-ISOLatin1Accent.txt
    [junit4] 2> 1205145 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf2/mapping-ISOLatin1Accent.txt
    [junit4] 2> 1205146 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
    to /configs/conf2/old_synonyms.txt
    [junit4] 2> 1205147 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf2/old_synonyms.txt
    [junit4] 2> 1205148 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
    to /configs/conf2/synonyms.txt
    [junit4] 2> 1205149 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf2/synonyms.txt
    [junit4] 2> 1205150 T27685 oasc.AbstractZkTestCase.putConfig put
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/elevate.xml
    to /configs/conf2/elevate.xml
    [junit4] 2> 1205151 T27685 oascc.SolrZkClient.makePath makePath:
    /configs/conf2/elevate.xml
    [junit4] 2> 1206445 T27685 oejs.Server.doStart jetty-8.1.10.v20130312
    [junit4] 2> 1206449 T27685 oejus.SslContextFactory.doStart Enabled
    Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello,
    SSLv3, TLSv1, TLSv1.1, TLSv1.2]
    [junit4] 2> 1206454 T27685 oejs.AbstractConnector.doStart Started
    sslsocketconnector@127.0.0.1:63972
    [junit4] 2> 1206456 T27685 oass.SolrDispatchFilter.init
    SolrDispatchFilter.init()
    [junit4] 2> 1206456 T27685 oasc.SolrResourceLoader.locateSolrHome
    JNDI not configured for solr (NoInitialContextEx)
    [junit4] 2> 1206457 T27685 oasc.SolrResourceLoader.locateSolrHome
    using system property solr.solr.home:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-002
    [junit4] 2> 1206457 T27685 oasc.SolrResourceLoader.<init> new
    SolrResourceLoader for directory:
    '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-002/'
    [junit4] 2> 1206488 T27685 oasc.ConfigSolr.fromFile Loading container
    configuration from
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-002/solr.xml
    [junit4] 2> 1206573 T27685 oasc.CoreContainer.<init> New
    CoreContainer 480716668
    [junit4] 2> 1206573 T27685 oasc.CoreContainer.load Loading cores into
    CoreContainer
    [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-002/]
    [junit4] 2> 1206575 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting socketTimeout to: 120000
    [junit4] 2> 1206575 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting urlScheme to: null
    [junit4] 2> 1206575 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting connTimeout to: 15000
    [junit4] 2> 1206576 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxConnectionsPerHost to: 20
    [junit4] 2> 1206576 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxConnections to: 10000
    [junit4] 2> 1206576 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting corePoolSize to: 0
    [junit4] 2> 1206577 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maximumPoolSize to: 2147483647
    [junit4] 2> 1206577 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxThreadIdleTime to: 5
    [junit4] 2> 1206578 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting sizeOfQueue to: -1
    [junit4] 2> 1206578 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting fairnessPolicy to: false
    [junit4] 2> 1206581 T27685 oasu.UpdateShardHandler.<init> Creating
    UpdateShardHandler HTTP client with params:
    socketTimeout=3400000&connTimeout=45000&retry=false
    [junit4] 2> 1206583 T27685 oasl.LogWatcher.createWatcher SLF4J impl
    is org.slf4j.impl.Log4jLoggerFactory
    [junit4] 2> 1206583 T27685 oasl.LogWatcher.newRegisteredLogWatcher
    Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
    [junit4] 2> 1206584 T27685 oasc.CoreContainer.load Host Name:
    127.0.0.1
    [junit4] 2> 1206584 T27685 oasc.ZkContainer.initZooKeeper Zookeeper
    client=127.0.0.1:63966/solr
    [junit4] 2> 1206584 T27685 oasc.ZkController.checkChrootPath zkHost
    includes chroot
    [junit4] 2> 1206585 T27685
    oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default
    ZkCredentialsProvider
    [junit4] 2> 1206586 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1206587 T27711 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@181b9f03
    name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent
    state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1206588 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1206589 T27685 oascc.SolrZkClient.createZkACLProvider
    Using default ZkACLProvider
    [junit4] 2> 1206590 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1206591 T27713 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@63a586f5
    name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event
    WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1206592 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1206593 T27685 oascc.SolrZkClient.makePath makePath:
    /overseer/queue
    [junit4] 2> 1206595 T27685 oascc.SolrZkClient.makePath makePath:
    /overseer/collection-queue-work
    [junit4] 2> 1206597 T27685 oascc.SolrZkClient.makePath makePath:
    /overseer/collection-map-running
    [junit4] 2> 1206599 T27685 oascc.SolrZkClient.makePath makePath:
    /overseer/collection-map-completed
    [junit4] 2> 1206601 T27685 oascc.SolrZkClient.makePath makePath:
    /overseer/collection-map-failure
    [junit4] 2> 1206603 T27685 oascc.SolrZkClient.makePath makePath:
    /live_nodes
    [junit4] 2> 1206604 T27685 oasc.ZkController.createEphemeralLiveNode
    Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63972_
    [junit4] 2> 1206605 T27685 oascc.SolrZkClient.makePath makePath:
    /live_nodes/127.0.0.1:63972_
    [junit4] 2> 1206606 T27685 oascc.SolrZkClient.makePath makePath:
    /overseer_elect
    [junit4] 2> 1206607 T27685 oascc.SolrZkClient.makePath makePath:
    /overseer_elect/election
    [junit4] 2> 1206609 T27685 oasc.Overseer.close Overseer (id=null)
    closing
    [junit4] 2> 1206610 T27685
    oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader
    127.0.0.1:63972_
    [junit4] 2> 1206611 T27685 oascc.SolrZkClient.makePath makePath:
    /overseer_elect/leader
    [junit4] 2> 1206612 T27685 oasc.Overseer.start Overseer
    (id=92702442030432262-127.0.0.1:63972_-n_0000000000) starting
    [junit4] 2> 1206614 T27685 oascc.SolrZkClient.makePath makePath:
    /overseer/queue-work
    [junit4] 2> 1206619 T27685
    oasc.OverseerAutoReplicaFailoverThread.<init> Starting
    OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=1000
    autoReplicaFailoverWaitAfterExpiration=30000
    autoReplicaFailoverBadNodeExpiration=60000
    [junit4] 2> 1206620 T27715 oasc.OverseerCollectionProcessor.run
    Process current queue of collection creations
    [junit4] 2> 1206621 T27685 oascc.SolrZkClient.makePath makePath:
    /clusterstate.json
    [junit4] 2> 1206623 T27685 oascc.SolrZkClient.makePath makePath:
    /aliases.json
    [junit4] 2> 1206624 T27685
    oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
    state from ZooKeeper...
    [junit4] 2> 1206626 T27714 oasc.Overseer$ClusterStateUpdater.run
    Starting to work on the main queue
    [junit4] 2> 1206630 T27685 oass.SolrDispatchFilter.init
    user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
    [junit4] 2> 1206630 T27685 oass.SolrDispatchFilter.init
    SolrDispatchFilter.init() done
    [junit4] 2> 1206635 T27685
    oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default
    ZkCredentialsProvider
    [junit4] 2> 1206635 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1206637 T27719 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@41205c8b
    name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event
    WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1206637 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1206637 T27685 oascc.SolrZkClient.createZkACLProvider
    Using default ZkACLProvider
    [junit4] 2> 1206638 T27685
    oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
    state from ZooKeeper...
    [junit4] 2> 1206640 T27685 oasc.ChaosMonkey.monkeyLog monkey: init -
    expire sessions:false cause connection loss:false
    [junit4] 2> 1207706 T27685
    oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
    [junit4] 2> 1207707 T27685 oejs.Server.doStart jetty-8.1.10.v20130312
    [junit4] 2> 1207710 T27685 oejus.SslContextFactory.doStart Enabled
    Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello,
    SSLv3, TLSv1, TLSv1.1, TLSv1.2]
    [junit4] 2> 1207712 T27685 oejs.AbstractConnector.doStart Started
    sslsocketconnector@127.0.0.1:63976
    [junit4] 2> 1207714 T27685 oass.SolrDispatchFilter.init
    SolrDispatchFilter.init()
    [junit4] 2> 1207714 T27685 oasc.SolrResourceLoader.locateSolrHome
    JNDI not configured for solr (NoInitialContextEx)
    [junit4] 2> 1207715 T27685 oasc.SolrResourceLoader.locateSolrHome
    using system property solr.solr.home:
    ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003
    [junit4] 2> 1207715 T27685 oasc.SolrResourceLoader.<init> new
    SolrResourceLoader for directory:
    '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/'
    [junit4] 2> 1207746 T27685 oasc.ConfigSolr.fromFile Loading container
    configuration from
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solr.xml
    [junit4] 2> 1207804 T27685 oasc.CoreContainer.<init> New
    CoreContainer 1821520330
    [junit4] 2> 1207804 T27685 oasc.CoreContainer.load Loading cores into
    CoreContainer
    [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/]
    [junit4] 2> 1207805 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting socketTimeout to: 120000
    [junit4] 2> 1207805 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting urlScheme to: null
    [junit4] 2> 1207805 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting connTimeout to: 15000
    [junit4] 2> 1207805 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxConnectionsPerHost to: 20
    [junit4] 2> 1207806 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxConnections to: 10000
    [junit4] 2> 1207806 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting corePoolSize to: 0
    [junit4] 2> 1207806 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maximumPoolSize to: 2147483647
    [junit4] 2> 1207806 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxThreadIdleTime to: 5
    [junit4] 2> 1207807 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting sizeOfQueue to: -1
    [junit4] 2> 1207807 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting fairnessPolicy to: false
    [junit4] 2> 1207809 T27685 oasu.UpdateShardHandler.<init> Creating
    UpdateShardHandler HTTP client with params:
    socketTimeout=3400000&connTimeout=45000&retry=false
    [junit4] 2> 1207810 T27685 oasl.LogWatcher.createWatcher SLF4J impl
    is org.slf4j.impl.Log4jLoggerFactory
    [junit4] 2> 1207810 T27685 oasl.LogWatcher.newRegisteredLogWatcher
    Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
    [junit4] 2> 1207811 T27685 oasc.CoreContainer.load Host Name:
    127.0.0.1
    [junit4] 2> 1207811 T27685 oasc.ZkContainer.initZooKeeper Zookeeper
    client=127.0.0.1:63966/solr
    [junit4] 2> 1207811 T27685 oasc.ZkController.checkChrootPath zkHost
    includes chroot
    [junit4] 2> 1207811 T27685
    oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default
    ZkCredentialsProvider
    [junit4] 2> 1207812 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1207813 T27730 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@d5684a6
    name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent
    state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1207814 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1207815 T27685 oascc.SolrZkClient.createZkACLProvider
    Using default ZkACLProvider
    [junit4] 2> 1207816 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1207817 T27732 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@16a0a9be
    name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event
    WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1207817 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1207820 T27685
    oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
    state from ZooKeeper...
    [junit4] 2> 1208822 T27685 oasc.ZkController.createEphemeralLiveNode
    Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63976_
    [junit4] 2> 1208823 T27685 oascc.SolrZkClient.makePath makePath:
    /live_nodes/127.0.0.1:63976_
    [junit4] 2> 1208827 T27685 oasc.Overseer.close Overseer (id=null)
    closing
    [junit4] 2> 1208832 T27685 oass.SolrDispatchFilter.init
    user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
    [junit4] 2> 1208832 T27685 oass.SolrDispatchFilter.init
    SolrDispatchFilter.init() done
    [junit4] 2> 1210182 T27685
    oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
    [junit4] 2> 1210184 T27685 oejs.Server.doStart jetty-8.1.10.v20130312
    [junit4] 2> 1210188 T27685 oejus.SslContextFactory.doStart Enabled
    Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello,
    SSLv3, TLSv1, TLSv1.1, TLSv1.2]
    [junit4] 2> 1210189 T27685 oejs.AbstractConnector.doStart Started
    sslsocketconnector@127.0.0.1:63979
    [junit4] 2> 1210191 T27685 oass.SolrDispatchFilter.init
    SolrDispatchFilter.init()
    [junit4] 2> 1210192 T27685 oasc.SolrResourceLoader.locateSolrHome
    JNDI not configured for solr (NoInitialContextEx)
    [junit4] 2> 1210192 T27685 oasc.SolrResourceLoader.locateSolrHome
    using system property solr.solr.home:
    ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004
    [junit4] 2> 1210192 T27685 oasc.SolrResourceLoader.<init> new
    SolrResourceLoader for directory:
    '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/'
    [junit4] 2> 1210222 T27685 oasc.ConfigSolr.fromFile Loading container
    configuration from
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solr.xml
    [junit4] 2> 1210279 T27685 oasc.CoreContainer.<init> New
    CoreContainer 1318011298
    [junit4] 2> 1210280 T27685 oasc.CoreContainer.load Loading cores into
    CoreContainer
    [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/]
    [junit4] 2> 1210281 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting socketTimeout to: 120000
    [junit4] 2> 1210281 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting urlScheme to: null
    [junit4] 2> 1210281 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting connTimeout to: 15000
    [junit4] 2> 1210281 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxConnectionsPerHost to: 20
    [junit4] 2> 1210281 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxConnections to: 10000
    [junit4] 2> 1210282 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting corePoolSize to: 0
    [junit4] 2> 1210282 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maximumPoolSize to: 2147483647
    [junit4] 2> 1210282 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxThreadIdleTime to: 5
    [junit4] 2> 1210282 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting sizeOfQueue to: -1
    [junit4] 2> 1210283 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting fairnessPolicy to: false
    [junit4] 2> 1210284 T27685 oasu.UpdateShardHandler.<init> Creating
    UpdateShardHandler HTTP client with params:
    socketTimeout=3400000&connTimeout=45000&retry=false
    [junit4] 2> 1210286 T27685 oasl.LogWatcher.createWatcher SLF4J impl
    is org.slf4j.impl.Log4jLoggerFactory
    [junit4] 2> 1210286 T27685 oasl.LogWatcher.newRegisteredLogWatcher
    Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
    [junit4] 2> 1210286 T27685 oasc.CoreContainer.load Host Name:
    127.0.0.1
    [junit4] 2> 1210287 T27685 oasc.ZkContainer.initZooKeeper Zookeeper
    client=127.0.0.1:63966/solr
    [junit4] 2> 1210287 T27685 oasc.ZkController.checkChrootPath zkHost
    includes chroot
    [junit4] 2> 1210287 T27685
    oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default
    ZkCredentialsProvider
    [junit4] 2> 1210288 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1210290 T27747 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@e86c09a
    name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent
    state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1210291 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1210292 T27685 oascc.SolrZkClient.createZkACLProvider
    Using default ZkACLProvider
    [junit4] 2> 1210294 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1210295 T27749 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@22acd0e4
    name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event
    WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1210295 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1210299 T27685
    oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
    state from ZooKeeper...
    [junit4] 2> 1211302 T27685 oasc.ZkController.createEphemeralLiveNode
    Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63979_
    [junit4] 2> 1211303 T27685 oascc.SolrZkClient.makePath makePath:
    /live_nodes/127.0.0.1:63979_
    [junit4] 2> 1211306 T27685 oasc.Overseer.close Overseer (id=null)
    closing
    [junit4] 2> 1211311 T27685 oass.SolrDispatchFilter.init
    user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
    [junit4] 2> 1211311 T27685 oass.SolrDispatchFilter.init
    SolrDispatchFilter.init() done
    [junit4] 2> 1212478 T27685
    oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
    [junit4] 2> 1212478 T27685 oejs.Server.doStart jetty-8.1.10.v20130312
    [junit4] 2> 1212482 T27685 oejus.SslContextFactory.doStart Enabled
    Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello,
    SSLv3, TLSv1, TLSv1.1, TLSv1.2]
    [junit4] 2> 1212483 T27685 oejs.AbstractConnector.doStart Started
    sslsocketconnector@127.0.0.1:63982
    [junit4] 2> 1212485 T27685 oass.SolrDispatchFilter.init
    SolrDispatchFilter.init()
    [junit4] 2> 1212486 T27685 oasc.SolrResourceLoader.locateSolrHome
    JNDI not configured for solr (NoInitialContextEx)
    [junit4] 2> 1212486 T27685 oasc.SolrResourceLoader.locateSolrHome
    using system property solr.solr.home:
    ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005
    [junit4] 2> 1212528 T27685 oasc.SolrResourceLoader.<init> new
    SolrResourceLoader for directory:
    '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/'
    [junit4] 2> 1212559 T27685 oasc.ConfigSolr.fromFile Loading container
    configuration from
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solr.xml
    [junit4] 2> 1212615 T27685 oasc.CoreContainer.<init> New
    CoreContainer 1213264595
    [junit4] 2> 1212616 T27685 oasc.CoreContainer.load Loading cores into
    CoreContainer
    [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/]
    [junit4] 2> 1212617 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting socketTimeout to: 120000
    [junit4] 2> 1212617 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting urlScheme to: null
    [junit4] 2> 1212617 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting connTimeout to: 15000
    [junit4] 2> 1212617 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxConnectionsPerHost to: 20
    [junit4] 2> 1212617 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxConnections to: 10000
    [junit4] 2> 1212618 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting corePoolSize to: 0
    [junit4] 2> 1212618 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maximumPoolSize to: 2147483647
    [junit4] 2> 1212618 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxThreadIdleTime to: 5
    [junit4] 2> 1212618 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting sizeOfQueue to: -1
    [junit4] 2> 1212619 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting fairnessPolicy to: false
    [junit4] 2> 1212620 T27685 oasu.UpdateShardHandler.<init> Creating
    UpdateShardHandler HTTP client with params:
    socketTimeout=3400000&connTimeout=45000&retry=false
    [junit4] 2> 1212622 T27685 oasl.LogWatcher.createWatcher SLF4J impl
    is org.slf4j.impl.Log4jLoggerFactory
    [junit4] 2> 1212622 T27685 oasl.LogWatcher.newRegisteredLogWatcher
    Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
    [junit4] 2> 1212622 T27685 oasc.CoreContainer.load Host Name:
    127.0.0.1
    [junit4] 2> 1212623 T27685 oasc.ZkContainer.initZooKeeper Zookeeper
    client=127.0.0.1:63966/solr
    [junit4] 2> 1212623 T27685 oasc.ZkController.checkChrootPath zkHost
    includes chroot
    [junit4] 2> 1212623 T27685
    oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default
    ZkCredentialsProvider
    [junit4] 2> 1212624 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1212625 T27762 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@439a3bd0
    name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent
    state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1212626 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1212626 T27685 oascc.SolrZkClient.createZkACLProvider
    Using default ZkACLProvider
    [junit4] 2> 1212628 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1212628 T27764 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@7ec3410d
    name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event
    WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1212629 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1212632 T27685
    oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
    state from ZooKeeper...
    [junit4] 2> 1213635 T27685 oasc.ZkController.createEphemeralLiveNode
    Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63982_
    [junit4] 2> 1213636 T27685 oascc.SolrZkClient.makePath makePath:
    /live_nodes/127.0.0.1:63982_
    [junit4] 2> 1213639 T27685 oasc.Overseer.close Overseer (id=null)
    closing
    [junit4] 2> 1213643 T27685 oass.SolrDispatchFilter.init
    user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
    [junit4] 2> 1213643 T27685 oass.SolrDispatchFilter.init
    SolrDispatchFilter.init() done
    [junit4] 2> 1214950 T27685
    oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
    [junit4] 2> 1214950 T27685 oejs.Server.doStart jetty-8.1.10.v20130312
    [junit4] 2> 1214954 T27685 oejus.SslContextFactory.doStart Enabled
    Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello,
    SSLv3, TLSv1, TLSv1.1, TLSv1.2]
    [junit4] 2> 1214955 T27685 oejs.AbstractConnector.doStart Started
    sslsocketconnector@127.0.0.1:63988
    [junit4] 2> 1214958 T27685 oass.SolrDispatchFilter.init
    SolrDispatchFilter.init()
    [junit4] 2> 1214958 T27685 oasc.SolrResourceLoader.locateSolrHome
    JNDI not configured for solr (NoInitialContextEx)
    [junit4] 2> 1214958 T27685 oasc.SolrResourceLoader.locateSolrHome
    using system property solr.solr.home:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006
    [junit4] 2> 1214958 T27685 oasc.SolrResourceLoader.<init> new
    SolrResourceLoader for directory:
    '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/'
    [junit4] 2> 1214991 T27685 oasc.ConfigSolr.fromFile Loading container
    configuration from
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solr.xml
    [junit4] 2> 1215051 T27685 oasc.CoreContainer.<init> New
    CoreContainer 1019796720
    [junit4] 2> 1215052 T27685 oasc.CoreContainer.load Loading cores into
    CoreContainer
    [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/]
    [junit4] 2> 1215053 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting socketTimeout to: 120000
    [junit4] 2> 1215053 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting urlScheme to: null
    [junit4] 2> 1215053 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting connTimeout to: 15000
    [junit4] 2> 1215053 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxConnectionsPerHost to: 20
    [junit4] 2> 1215054 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxConnections to: 10000
    [junit4] 2> 1215054 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting corePoolSize to: 0
    [junit4] 2> 1215054 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maximumPoolSize to: 2147483647
    [junit4] 2> 1215054 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting maxThreadIdleTime to: 5
    [junit4] 2> 1215055 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting sizeOfQueue to: -1
    [junit4] 2> 1215055 T27685 oashc.HttpShardHandlerFactory.getParameter
    Setting fairnessPolicy to: false
    [junit4] 2> 1215057 T27685 oasu.UpdateShardHandler.<init> Creating
    UpdateShardHandler HTTP client with params:
    socketTimeout=3400000&connTimeout=45000&retry=false
    [junit4] 2> 1215058 T27685 oasl.LogWatcher.createWatcher SLF4J impl
    is org.slf4j.impl.Log4jLoggerFactory
    [junit4] 2> 1215059 T27685 oasl.LogWatcher.newRegisteredLogWatcher
    Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
    [junit4] 2> 1215059 T27685 oasc.CoreContainer.load Host Name:
    127.0.0.1
    [junit4] 2> 1215059 T27685 oasc.ZkContainer.initZooKeeper Zookeeper
    client=127.0.0.1:63966/solr
    [junit4] 2> 1215059 T27685 oasc.ZkController.checkChrootPath zkHost
    includes chroot
    [junit4] 2> 1215060 T27685
    oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default
    ZkCredentialsProvider
    [junit4] 2> 1215060 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1215061 T27777 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@1e421349
    name:ZooKeeperConnection Watcher:127.0.0.1:63966 got event WatchedEvent
    state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1215062 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1215063 T27685 oascc.SolrZkClient.createZkACLProvider
    Using default ZkACLProvider
    [junit4] 2> 1215064 T27685 oascc.ConnectionManager.waitForConnected
    Waiting for client to connect to ZooKeeper
    [junit4] 2> 1215065 T27779 oascc.ConnectionManager.process Watcher
    org.apache.solr.common.cloud.ConnectionManager@75ae54ac
    name:ZooKeeperConnection Watcher:127.0.0.1:63966/solr got event
    WatchedEvent state:SyncConnected type:None path:null path:null type:None
    [junit4] 2> 1215065 T27685 oascc.ConnectionManager.waitForConnected
    Client is connected to ZooKeeper
    [junit4] 2> 1215069 T27685
    oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
    state from ZooKeeper...
    [junit4] 2> 1216071 T27685 oasc.ZkController.createEphemeralLiveNode
    Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63988_
    [junit4] 2> 1216072 T27685 oascc.SolrZkClient.makePath makePath:
    /live_nodes/127.0.0.1:63988_
    [junit4] 2> 1216075 T27685 oasc.Overseer.close Overseer (id=null)
    closing
    [junit4] 2> 1216079 T27685 oass.SolrDispatchFilter.init
    user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
    [junit4] 2> 1216080 T27685 oass.SolrDispatchFilter.init
    SolrDispatchFilter.init() done
    [junit4] 2> 1216200 T27723
    oasha.CollectionsHandler.handleCreateAction Creating Collection :
    action=CREATE&wt=javabin&numShards=2&collection.configName=conf1&version=2&name=solrj_collection&router.field=myOwnField&replicationFactor=2
    [junit4] 2> 1216202 T27735
    oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
    path: /overseer/collection-queue-work state: SyncConnected type
    NodeChildrenChanged
    [junit4] 2> 1216202 T27715 oasc.OverseerCollectionProcessor.run
    Overseer Collection Processor: Get the message
    id:/overseer/collection-queue-work/qn-0000000000 message:{
    [junit4] 2> "operation":"create",
    [junit4] 2> "fromApi":"true",
    [junit4] 2> "name":"solrj_collection",
    [junit4] 2> "replicationFactor":"2",
    [junit4] 2> "collection.configName":"conf1",
    [junit4] 2> "numShards":"2",
    [junit4] 2> "router.field":"myOwnField"}
    [junit4] 2> 1216204 T27783
    oasc.OverseerCollectionProcessor.processMessage WARN
    OverseerCollectionProcessor.processMessage : create , {
    [junit4] 2> "operation":"create",
    [junit4] 2> "fromApi":"true",
    [junit4] 2> "name":"solrj_collection",
    [junit4] 2> "replicationFactor":"2",
    [junit4] 2> "collection.configName":"conf1",
    [junit4] 2> "numShards":"2",
    [junit4] 2> "router.field":"myOwnField"}
    [junit4] 2> 1216205 T27783
    oasc.OverseerCollectionProcessor.createConfNode creating collections conf
    node /collections/solrj_collection
    [junit4] 2> 1216205 T27783 oascc.SolrZkClient.makePath makePath:
    /collections/solrj_collection
    [junit4] 2> 1216207 T27735
    oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
    path: /overseer/queue state: SyncConnected type NodeChildrenChanged
    [junit4] 2> 1216209 T27714
    oasc.Overseer$ClusterStateUpdater.buildCollection building a new
    collection: solrj_collection
    [junit4] 2> 1216209 T27714
    oasc.Overseer$ClusterStateUpdater.createCollection Create collection
    solrj_collection with shards [shard1, shard2]
    [junit4] 2> 1216209 T27714
    oasc.Overseer$ClusterStateUpdater.createCollection state version
    solrj_collection 1
    [junit4] 2> 1216211 T27734 oascc.ZkStateReader$2.process A cluster
    state change: WatchedEvent state:SyncConnected type:NodeDataChanged
    path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1216211 T27735 oascc.ZkStateReader$2.process A cluster
    state change: WatchedEvent state:SyncConnected type:NodeDataChanged
    path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1216211 T27765 oascc.ZkStateReader$2.process A cluster
    state change: WatchedEvent state:SyncConnected type:NodeDataChanged
    path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1216211 T27750 oascc.ZkStateReader$2.process A cluster
    state change: WatchedEvent state:SyncConnected type:NodeDataChanged
    path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1216211 T27733 oascc.ZkStateReader$2.process A cluster
    state change: WatchedEvent state:SyncConnected type:NodeDataChanged
    path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1216211 T27780 oascc.ZkStateReader$2.process A cluster
    state change: WatchedEvent state:SyncConnected type:NodeDataChanged
    path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1216307 T27783
    oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for
    new collection solrj_collection, shardNames [shard1, shard2] ,
    replicationFactor : 2
    [junit4] 2> 1216308 T27783
    oasc.OverseerCollectionProcessor.createCollection Creating shard
    solrj_collection_shard1_replica1 as part of slice shard1 of collection
    solrj_collection on 127.0.0.1:63988_
    [junit4] 2> 1216309 T27783
    oasc.OverseerCollectionProcessor.createCollection Creating shard
    solrj_collection_shard1_replica2 as part of slice shard1 of collection
    solrj_collection on 127.0.0.1:63976_
    [junit4] 2> 1216311 T27783
    oasc.OverseerCollectionProcessor.createCollection Creating shard
    solrj_collection_shard2_replica1 as part of slice shard2 of collection
    solrj_collection on 127.0.0.1:63979_
    [junit4] 2> 1216314 T27783
    oasc.OverseerCollectionProcessor.createCollection Creating shard
    solrj_collection_shard2_replica2 as part of slice shard2 of collection
    solrj_collection on 127.0.0.1:63982_
    [junit4] 2> 1216432 T27743 oasha.CoreAdminHandler.handleCreateAction
    core create command
    action=CREATE&wt=javabin&numShards=2&collection.configName=conf1&version=2&name=solrj_collection_shard2_replica1&collection=solrj_collection&qt=/admin/cores&shard=shard2
    [junit4] 2> 1216434 T27743 oasc.ZkController.publish publishing
    core=solrj_collection_shard2_replica1 state=down collection=solrj_collection
    [junit4] 2> 1216435 T27743 oasc.ZkController.waitForCoreNodeName look
    for our core node name
    [junit4] 2> 1216435 T27735
    oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
    path: /overseer/queue state: SyncConnected type NodeChildrenChanged
    [junit4] 2> 1216438 T27724 oasha.CoreAdminHandler.handleCreateAction
    core create command
    action=CREATE&wt=javabin&numShards=2&collection.configName=conf1&version=2&name=solrj_collection_shard1_replica2&collection=solrj_collection&qt=/admin/cores&shard=shard1
    [junit4] 2> 1216438 T27769 oasha.CoreAdminHandler.handleCreateAction
    core create command
    action=CREATE&wt=javabin&numShards=2&collection.configName=conf1&version=2&name=solrj_collection_shard1_replica1&collection=solrj_collection&qt=/admin/cores&shard=shard1
    [junit4] 2> 1216438 T27714
    oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2
    message={
    [junit4] 2> "base_url":"https://127.0.0.1:63979",
    [junit4] 2> "operation":"state",
    [junit4] 2> "core":"solrj_collection_shard2_replica1",
    [junit4] 2> "numShards":"2",
    [junit4] 2> "state":"down",
    [junit4] 2> "collection":"solrj_collection",
    [junit4] 2> "roles":null,
    [junit4] 2> "shard":"shard2",
    [junit4] 2> "node_name":"127.0.0.1:63979_"}
    [junit4] 2> 1216441 T27724 oasc.ZkController.publish publishing
    core=solrj_collection_shard1_replica2 state=down collection=solrj_collection
    [junit4] 2> 1216442 T27724 oasc.ZkController.waitForCoreNodeName look
    for our core node name
    [junit4] 2> 1216442 T27754 oasha.CoreAdminHandler.handleCreateAction
    core create command
    action=CREATE&wt=javabin&numShards=2&collection.configName=conf1&version=2&name=solrj_collection_shard2_replica2&collection=solrj_collection&qt=/admin/cores&shard=shard2
    [junit4] 2> 1216442 T27769 oasc.ZkController.publish publishing
    core=solrj_collection_shard1_replica1 state=down collection=solrj_collection
    [junit4] 2> 1216442 T27735
    oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
    path: /overseer/queue state: SyncConnected type NodeChildrenChanged
    [junit4] 2> 1216443 T27769 oasc.ZkController.waitForCoreNodeName look
    for our core node name
    [junit4] 2> 1216444 T27714
    oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2
    message={
    [junit4] 2> "base_url":"https://127.0.0.1:63976",
    [junit4] 2> "operation":"state",
    [junit4] 2> "core":"solrj_collection_shard1_replica2",
    [junit4] 2> "numShards":"2",
    [junit4] 2> "state":"down",
    [junit4] 2> "collection":"solrj_collection",
    [junit4] 2> "roles":null,
    [junit4] 2> "shard":"shard1",
    [junit4] 2> "node_name":"127.0.0.1:63976_"}
    [junit4] 2> 1216445 T27754 oasc.ZkController.publish publishing
    core=solrj_collection_shard2_replica2 state=down collection=solrj_collection
    [junit4] 2> 1216446 T27735
    oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
    path: /overseer/queue state: SyncConnected type NodeChildrenChanged
    [junit4] 2> 1216446 T27714
    oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2
    message={
    [junit4] 2> "base_url":"https://127.0.0.1:63988",
    [junit4] 2> "operation":"state",
    [junit4] 2> "core":"solrj_collection_shard1_replica1",
    [junit4] 2> "numShards":"2",
    [junit4] 2> "state":"down",
    [junit4] 2> "collection":"solrj_collection",
    [junit4] 2> "roles":null,
    [junit4] 2> "shard":"shard1",
    [junit4] 2> "node_name":"127.0.0.1:63988_"}
    [junit4] 2> 1216446 T27754 oasc.ZkController.waitForCoreNodeName look
    for our core node name
    [junit4] 2> 1216447 T27714
    oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2
    message={
    [junit4] 2> "base_url":"https://127.0.0.1:63982",
    [junit4] 2> "operation":"state",
    [junit4] 2> "core":"solrj_collection_shard2_replica2",
    [junit4] 2> "numShards":"2",
    [junit4] 2> "state":"down",
    [junit4] 2> "collection":"solrj_collection",
    [junit4] 2> "roles":null,
    [junit4] 2> "shard":"shard2",
    [junit4] 2> "node_name":"127.0.0.1:63982_"}
    [junit4] 2> 1216448 T27735
    oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
    path: /overseer/queue state: SyncConnected type NodeChildrenChanged
    [junit4] 2> 1216449 T27735 oascc.ZkStateReader$2.process A cluster
    state change: WatchedEvent state:SyncConnected type:NodeDataChanged
    path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1216449 T27733 oascc.ZkStateReader$2.process A cluster
    state change: WatchedEvent state:SyncConnected type:NodeDataChanged
    path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1216449 T27734 oascc.ZkStateReader$2.process A cluster
    state change: WatchedEvent state:SyncConnected type:NodeDataChanged
    path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1216449 T27750 oascc.ZkStateReader$2.process A cluster
    state change: WatchedEvent state:SyncConnected type:NodeDataChanged
    path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1216449 T27765 oascc.ZkStateReader$2.process A cluster
    state change: WatchedEvent state:SyncConnected type:NodeDataChanged
    path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1216449 T27780 oascc.ZkStateReader$2.process A cluster
    state change: WatchedEvent state:SyncConnected type:NodeDataChanged
    path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
    [junit4] 2> 1217436 T27743 oasc.ZkController.waitForShardId waiting
    to find shard id in clusterstate for solrj_collection_shard2_replica1
    [junit4] 2> 1217437 T27743 oasc.ZkController.createCollectionZkNode
    Check for collection zkNode:solrj_collection
    [junit4] 2> 1217437 T27743 oasc.ZkController.createCollectionZkNode
    Collection zkNode exists
    [junit4] 2> 1217438 T27743 oascc.ZkStateReader.readConfigName Load
    collection config from:/collections/solrj_collection
    [junit4] 2> 1217438 T27743 oascc.ZkStateReader.readConfigName
    path=/collections/solrj_collection configName=conf1 specified config exists
    in ZooKeeper
    [junit4] 2> 1217438 T27743 oasc.SolrResourceLoader.<init> new
    SolrResourceLoader for directory:
    '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/'
    [junit4] 2> 1217444 T27724 oasc.ZkController.waitForShardId waiting
    to find shard id in clusterstate for solrj_collection_shard1_replica2
    [junit4] 2> 1217444 T27724 oasc.ZkController.createCollectionZkNode
    Check for collection zkNode:solrj_collection
    [junit4] 2> 1217444 T27769 oasc.ZkController.waitForShardId waiting
    to find shard id in clusterstate for solrj_collection_shard1_replica1
    [junit4] 2> 1217445 T27769 oasc.ZkController.createCollectionZkNode
    Check for collection zkNode:solrj_collection
    [junit4] 2> 1217445 T27724 oasc.ZkController.createCollectionZkNode
    Collection zkNode exists
    [junit4] 2> 1217445 T27724 oascc.ZkStateReader.readConfigName Load
    collection config from:/collections/solrj_collection
    [junit4] 2> 1217445 T27769 oasc.ZkController.createCollectionZkNode
    Collection zkNode exists
    [junit4] 2> 1217445 T27769 oascc.ZkStateReader.readConfigName Load
    collection config from:/collections/solrj_collection
    [junit4] 2> 1217446 T27724 oascc.ZkStateReader.readConfigName
    path=/collections/solrj_collection configName=conf1 specified config exists
    in ZooKeeper
    [junit4] 2> 1217446 T27724 oasc.SolrResourceLoader.<init> new
    SolrResourceLoader for directory:
    '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/'
    [junit4] 2> 1217446 T27769 oascc.ZkStateReader.readConfigName
    path=/collections/solrj_collection configName=conf1 specified config exists
    in ZooKeeper
    [junit4] 2> 1217446 T27769 oasc.SolrResourceLoader.<init> new
    SolrResourceLoader for directory:
    '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/'
    [junit4] 2> 1217446 T27754 oasc.ZkController.waitForShardId waiting
    to find shard id in clusterstate for solrj_collection_shard2_replica2
    [junit4] 2> 1217447 T27754 oasc.ZkController.createCollectionZkNode
    Check for collection zkNode:solrj_collection
    [junit4] 2> 1217448 T27754 oasc.ZkController.createCollectionZkNode
    Collection zkNode exists
    [junit4] 2> 1217448 T27754 oascc.ZkStateReader.readConfigName Load
    collection config from:/collections/solrj_collection
    [junit4] 2> 1217449 T27754 oascc.ZkStateReader.readConfigName
    path=/collections/solrj_collection configName=conf1 specified config exists
    in ZooKeeper
    [junit4] 2> 1217450 T27754 oasc.SolrResourceLoader.<init> new
    SolrResourceLoader for directory:
    '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/'
    [junit4] 2> 1217538 T27769 oasc.SolrConfig.<init> Using Lucene
    MatchVersion: 5.0.0
    [junit4] 2> 1217539 T27743 oasc.SolrConfig.<init> Using Lucene
    MatchVersion: 5.0.0
    [junit4] 2> 1217557 T27724 oasc.SolrConfig.<init> Using Lucene
    MatchVersion: 5.0.0
    [junit4] 2> 1217613 T27754 oasc.SolrConfig.<init> Using Lucene
    MatchVersion: 5.0.0
    [junit4] 2> 1217627 T27743 oasc.SolrConfig.<init> Loaded SolrConfig:
    solrconfig.xml
    [junit4] 2> 1217633 T27769 oasc.SolrConfig.<init> Loaded SolrConfig:
    solrconfig.xml
    [junit4] 2> 1217645 T27724 oasc.SolrConfig.<init> Loaded SolrConfig:
    solrconfig.xml
    [junit4] 2> 1217653 T27754 oasc.SolrConfig.<init> Loaded SolrConfig:
    solrconfig.xml
    [junit4] 2> 1217728 T27743 oass.IndexSchema.readSchema Reading Solr
    Schema from /configs/conf1/schema.xml
    [junit4] 2> 1217734 T27769 oass.IndexSchema.readSchema Reading Solr
    Schema from /configs/conf1/schema.xml
    [junit4] 2> 1217742 T27743 oass.IndexSchema.readSchema
    [solrj_collection_shard2_replica1] Schema name=test
    [junit4] 2> 1217746 T27724 oass.IndexSchema.readSchema Reading Solr
    Schema from /configs/conf1/schema.xml
    [junit4] 2> 1217747 T27769 oass.IndexSchema.readSchema
    [solrj_collection_shard1_replica1] Schema name=test
    [junit4] 2> 1217754 T27754 oass.IndexSchema.readSchema Reading Solr
    Schema from /configs/conf1/schema.xml
    [junit4] 2> 1217756 T27724 oass.IndexSchema.readSchema
    [solrj_collection_shard1_replica2] Schema name=test
    [junit4] 2> 1217807 T27754 oass.IndexSchema.readSchema
    [solrj_collection_shard2_replica2] Schema name=test
    [junit4] 2> 1218411 T27724 oass.OpenExchangeRatesOrgProvider.init
    Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
    [junit4] 2> 1218426 T27724 oass.IndexSchema.readSchema default search
    field in schema is text
    [junit4] 2> 1218438 T27724 oass.IndexSchema.readSchema unique key
    field: id
    [junit4] 2> 1218440 T27743 oass.OpenExchangeRatesOrgProvider.init
    Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
    [junit4] 2> 1218444 T27754 oass.OpenExchangeRatesOrgProvider.init
    Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
    [junit4] 2> 1218448 T27743 oass.IndexSchema.readSchema default search
    field in schema is text
    [junit4] 2> 1218450 T27724 oass.FileExchangeRateProvider.reload
    Reloading exchange rates from file currency.xml
    [junit4] 2> 1218451 T27754 oass.IndexSchema.readSchema default search
    field in schema is text
    [junit4] 2> 1218451 T27743 oass.IndexSchema.readSchema unique key
    field: id
    [junit4] 2> 1218455 T27754 oass.IndexSchema.readSchema unique key
    field: id
    [junit4] 2> 1218455 T27724 oass.FileExchangeRateProvider.reload
    Reloading exchange rates from file currency.xml
    [junit4] 2> 1218460 T27724 oass.OpenExchangeRatesOrgProvider.reload
    Reloading exchange rates from open-exchange-rates.json
    [junit4] 2> 1218461 T27724
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
    IMPORTANT NOTE
    [junit4] 2> 1218464 T27724
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
    key, got STRING
    [junit4] 2> 1218464 T27724 oass.OpenExchangeRatesOrgProvider.reload
    Reloading exchange rates from open-exchange-rates.json
    [junit4] 2> 1218465 T27743 oass.FileExchangeRateProvider.reload
    Reloading exchange rates from file currency.xml
    [junit4] 2> 1218465 T27724
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
    IMPORTANT NOTE
    [junit4] 2> 1218466 T27724
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
    key, got STRING
    [junit4] 2> 1218466 T27724 oasc.CoreContainer.create Creating
    SolrCore 'solrj_collection_shard1_replica2' using configuration from
    collection solrj_collection
    [junit4] 2> 1218466 T27724 oasc.SolrCore.initDirectoryFactory
    org.apache.solr.core.MockDirectoryFactory
    [junit4] 2> 1218467 T27724 oasc.SolrCore.<init>
    [solrj_collection_shard1_replica2] Opening new SolrCore at
    ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/,
    dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/data/
    [junit4] 2> 1218467 T27724 oasc.JmxMonitoredMap.<init> JMX monitoring
    is enabled. Adding Solr mbeans to JMX Server:
    com.sun.jmx.mbeanserver.JmxMBeanServer@3eb6814a
    [junit4] 2> 1218469 T27754 oass.FileExchangeRateProvider.reload
    Reloading exchange rates from file currency.xml
    [junit4] 2> 1218470 T27743 oass.FileExchangeRateProvider.reload
    Reloading exchange rates from file currency.xml
    [junit4] 2> 1218470 T27724 oasc.CachingDirectoryFactory.get return
    new directory for
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/data
    [junit4] 2> 1218472 T27724 oasc.SolrCore.getNewIndexDir New index
    directory detected: old=null
    new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/data/index/
    [junit4] 2> 1218472 T27724 oasc.SolrCore.initIndex WARN
    [solrj_collection_shard1_replica2] Solr index directory
    '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/data/index'
    doesn't exist. Creating new index...
    [junit4] 2> 1218473 T27769 oass.OpenExchangeRatesOrgProvider.init
    Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
    [junit4] 2> 1218473 T27724 oasc.CachingDirectoryFactory.get return
    new directory for
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-003/solrj_collection_shard1_replica2/data/index
    [junit4] 2> 1218474 T27724 oasu.RandomMergePolicy.<init>
    RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy:
    [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=19,
    maxMergedSegmentMB=97.9267578125, floorSegmentMB=1.2958984375,
    forceMergeDeletesPctAllowed=11.900176490131441, segmentsPerTier=49.0,
    maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
    [junit4] 2> 1218474 T27754 oass.FileExchangeRateProvider.reload
    Reloading exchange rates from file currency.xml
    [junit4] 2> 1218475 T27724 oasc.SolrDeletionPolicy.onCommit
    SolrDeletionPolicy.onCommit: commits: num=1
    [junit4] 2>
    commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-7c885a91)),segFN=segments_1,generation=1}
    [junit4] 2> 1218475 T27743 oass.OpenExchangeRatesOrgProvider.reload
    Reloading exchange rates from open-exchange-rates.json
    [junit4] 2> 1218475 T27724 oasc.SolrDeletionPolicy.updateCommits
    newest commit generation = 1
    [junit4] 2> 1218476 T27743
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
    IMPORTANT NOTE
    [junit4] 2> 1218477 T27743
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
    key, got STRING
    [junit4] 2> 1218477 T27743 oass.OpenExchangeRatesOrgProvider.reload
    Reloading exchange rates from open-exchange-rates.json
    [junit4] 2> 1218478 T27743
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
    IMPORTANT NOTE
    [junit4] 2> 1218478 T27743
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
    key, got STRING
    [junit4] 2> 1218479 T27743 oasc.CoreContainer.create Creating
    SolrCore 'solrj_collection_shard2_replica1' using configuration from
    collection solrj_collection
    [junit4] 2> 1218479 T27743 oasc.SolrCore.initDirectoryFactory
    org.apache.solr.core.MockDirectoryFactory
    [junit4] 2> 1218479 T27743 oasc.SolrCore.<init>
    [solrj_collection_shard2_replica1] Opening new SolrCore at
    ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/,
    dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/data/
    [junit4] 2> 1218480 T27769 oass.IndexSchema.readSchema default search
    field in schema is text
    [junit4] 2> 1218480 T27754 oass.OpenExchangeRatesOrgProvider.reload
    Reloading exchange rates from open-exchange-rates.json
    [junit4] 2> 1218480 T27743 oasc.JmxMonitoredMap.<init> JMX monitoring
    is enabled. Adding Solr mbeans to JMX Server:
    com.sun.jmx.mbeanserver.JmxMBeanServer@3eb6814a
    [junit4] 2> 1218481 T27724 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "nodistrib"
    [junit4] 2> 1218481 T27754
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
    IMPORTANT NOTE
    [junit4] 2> 1218482 T27724 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "dedupe"
    [junit4] 2> 1218482 T27754
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
    key, got STRING
    [junit4] 2> 1218483 T27724 oasup.UpdateRequestProcessorChain.init
    inserting DistributedUpdateProcessorFactory into
    updateRequestProcessorChain "dedupe"
    [junit4] 2> 1218483 T27769 oass.IndexSchema.readSchema unique key
    field: id
    [junit4] 2> 1218483 T27754 oass.OpenExchangeRatesOrgProvider.reload
    Reloading exchange rates from open-exchange-rates.json
    [junit4] 2> 1218484 T27743 oasc.CachingDirectoryFactory.get return
    new directory for
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/data
    [junit4] 2> 1218483 T27724 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "stored_sig"
    [junit4] 2> 1218485 T27754
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
    IMPORTANT NOTE
    [junit4] 2> 1218485 T27743 oasc.SolrCore.getNewIndexDir New index
    directory detected: old=null
    new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/data/index/
    [junit4] 2> 1218485 T27724 oasup.UpdateRequestProcessorChain.init
    inserting DistributedUpdateProcessorFactory into
    updateRequestProcessorChain "stored_sig"
    [junit4] 2> 1218486 T27743 oasc.SolrCore.initIndex WARN
    [solrj_collection_shard2_replica1] Solr index directory
    '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/data/index'
    doesn't exist. Creating new index...
    [junit4] 2> 1218486 T27754
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
    key, got STRING
    [junit4] 2> 1218487 T27743 oasc.CachingDirectoryFactory.get return
    new directory for
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-004/solrj_collection_shard2_replica1/data/index
    [junit4] 2> 1218486 T27724 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
    [junit4] 2> 1218488 T27743 oasu.RandomMergePolicy.<init>
    RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy:
    [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=19,
    maxMergedSegmentMB=97.9267578125, floorSegmentMB=1.2958984375,
    forceMergeDeletesPctAllowed=11.900176490131441, segmentsPerTier=49.0,
    maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
    [junit4] 2> 1218487 T27754 oasc.CoreContainer.create Creating
    SolrCore 'solrj_collection_shard2_replica2' using configuration from
    collection solrj_collection
    [junit4] 2> 1218488 T27724 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
    [junit4] 2> 1218489 T27743 oasc.SolrDeletionPolicy.onCommit
    SolrDeletionPolicy.onCommit: commits: num=1
    [junit4] 2>
    commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-68b86762)),segFN=segments_1,generation=1}
    [junit4] 2> 1218489 T27754 oasc.SolrCore.initDirectoryFactory
    org.apache.solr.core.MockDirectoryFactory
    [junit4] 2> 1218490 T27743 oasc.SolrDeletionPolicy.updateCommits
    newest commit generation = 1
    [junit4] 2> 1218489 T27724 oasup.UpdateRequestProcessorChain.init
    inserting DistributedUpdateProcessorFactory into
    updateRequestProcessorChain "distrib-dup-test-chain-implicit"
    [junit4] 2> 1218490 T27754 oasc.SolrCore.<init>
    [solrj_collection_shard2_replica2] Opening new SolrCore at
    ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/,
    dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/data/
    [junit4] 2> 1218491 T27724 oasc.SolrCore.loadUpdateProcessorChains no
    updateRequestProcessorChain defined as default, creating implicit default
    [junit4] 2> 1218491 T27754 oasc.JmxMonitoredMap.<init> JMX monitoring
    is enabled. Adding Solr mbeans to JMX Server:
    com.sun.jmx.mbeanserver.JmxMBeanServer@3eb6814a
    [junit4] 2> 1218492 T27724
    oasc.RequestHandlers.initHandlersFromConfig created /update/json:
    org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1218493 T27724
    oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs:
    org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1218494 T27724
    oasc.RequestHandlers.initHandlersFromConfig created /update/csv:
    org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1218494 T27724
    oasc.RequestHandlers.initHandlersFromConfig created standard:
    solr.StandardRequestHandler
    [junit4] 2> 1218495 T27754 oasc.CachingDirectoryFactory.get return
    new directory for
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/data
    [junit4] 2> 1218495 T27769 oass.FileExchangeRateProvider.reload
    Reloading exchange rates from file currency.xml
    [junit4] 2> 1218495 T27743 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "nodistrib"
    [junit4] 2> 1218496 T27754 oasc.SolrCore.getNewIndexDir New index
    directory detected: old=null
    new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/data/index/
    [junit4] 2> 1218495 T27724
    oasc.RequestHandlers.initHandlersFromConfig created /get:
    solr.RealTimeGetHandler
    [junit4] 2> 1218496 T27754 oasc.SolrCore.initIndex WARN
    [solrj_collection_shard2_replica2] Solr index directory
    '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/data/index'
    doesn't exist. Creating new index...
    [junit4] 2> 1218496 T27743 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "dedupe"
    [junit4] 2> 1218497 T27724
    oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler:
    solr.ReplicationHandler
    [junit4] 2> 1218498 T27743 oasup.UpdateRequestProcessorChain.init
    inserting DistributedUpdateProcessorFactory into
    updateRequestProcessorChain "dedupe"
    [junit4] 2> 1218498 T27754 oasc.CachingDirectoryFactory.get return
    new directory for
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-005/solrj_collection_shard2_replica2/data/index
    [junit4] 2> 1218498 T27743 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "stored_sig"
    [junit4] 2> 1218499 T27754 oasu.RandomMergePolicy.<init>
    RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy:
    [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=19,
    maxMergedSegmentMB=97.9267578125, floorSegmentMB=1.2958984375,
    forceMergeDeletesPctAllowed=11.900176490131441, segmentsPerTier=49.0,
    maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
    [junit4] 2> 1218498 T27724
    oasc.RequestHandlers.initHandlersFromConfig created /replication:
    solr.ReplicationHandler
    [junit4] 2> 1218499 T27743 oasup.UpdateRequestProcessorChain.init
    inserting DistributedUpdateProcessorFactory into
    updateRequestProcessorChain "stored_sig"
    [junit4] 2> 1218500 T27724
    oasc.RequestHandlers.initHandlersFromConfig created /update:
    solr.UpdateRequestHandler
    [junit4] 2> 1218501 T27754 oasc.SolrDeletionPolicy.onCommit
    SolrDeletionPolicy.onCommit: commits: num=1
    [junit4] 2>
    commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-5c32ed3a)),segFN=segments_1,generation=1}
    [junit4] 2> 1218500 T27743 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
    [junit4] 2> 1218501 T27769 oass.FileExchangeRateProvider.reload
    Reloading exchange rates from file currency.xml
    [junit4] 2> 1218502 T27743 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
    [junit4] 2> 1218501 T27754 oasc.SolrDeletionPolicy.updateCommits
    newest commit generation = 1
    [junit4] 2> 1218501 T27724
    oasc.RequestHandlers.initHandlersFromConfig created /admin/:
    org.apache.solr.handler.admin.AdminHandlers
    [junit4] 2> 1218502 T27743 oasup.UpdateRequestProcessorChain.init
    inserting DistributedUpdateProcessorFactory into
    updateRequestProcessorChain "distrib-dup-test-chain-implicit"
    [junit4] 2> 1218503 T27724
    oasc.RequestHandlers.initHandlersFromConfig created /admin/file:
    solr.admin.ShowFileRequestHandler
    [junit4] 2> 1218503 T27743 oasc.SolrCore.loadUpdateProcessorChains no
    updateRequestProcessorChain defined as default, creating implicit default
    [junit4] 2> 1218504 T27724
    oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit:
    solr.admin.EditFileRequestHandler
    [junit4] 2> 1218504 T27743
    oasc.RequestHandlers.initHandlersFromConfig created /update/json:
    org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1218505 T27743
    oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs:
    org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1218506 T27743
    oasc.RequestHandlers.initHandlersFromConfig created /update/csv:
    org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1218506 T27743
    oasc.RequestHandlers.initHandlersFromConfig created standard:
    solr.StandardRequestHandler
    [junit4] 2> 1218507 T27743
    oasc.RequestHandlers.initHandlersFromConfig created /get:
    solr.RealTimeGetHandler
    [junit4] 2> 1218507 T27743
    oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler:
    solr.ReplicationHandler
    [junit4] 2> 1218507 T27769 oass.OpenExchangeRatesOrgProvider.reload
    Reloading exchange rates from open-exchange-rates.json
    [junit4] 2> 1218508 T27743
    oasc.RequestHandlers.initHandlersFromConfig created /replication:
    solr.ReplicationHandler
    [junit4] 2> 1218508 T27754 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "nodistrib"
    [junit4] 2> 1218509 T27769
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
    IMPORTANT NOTE
    [junit4] 2> 1218509 T27754 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "dedupe"
    [junit4] 2> 1218509 T27743
    oasc.RequestHandlers.initHandlersFromConfig created /update:
    solr.UpdateRequestHandler
    [junit4] 2> 1218510 T27754 oasup.UpdateRequestProcessorChain.init
    inserting DistributedUpdateProcessorFactory into
    updateRequestProcessorChain "dedupe"
    [junit4] 2> 1218509 T27769
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
    key, got STRING
    [junit4] 2> 1218510 T27754 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "stored_sig"
    [junit4] 2> 1218510 T27743
    oasc.RequestHandlers.initHandlersFromConfig created /admin/:
    org.apache.solr.handler.admin.AdminHandlers
    [junit4] 2> 1218511 T27754 oasup.UpdateRequestProcessorChain.init
    inserting DistributedUpdateProcessorFactory into
    updateRequestProcessorChain "stored_sig"
    [junit4] 2> 1218511 T27743
    oasc.RequestHandlers.initHandlersFromConfig created /admin/file:
    solr.admin.ShowFileRequestHandler
    [junit4] 2> 1218510 T27769 oass.OpenExchangeRatesOrgProvider.reload
    Reloading exchange rates from open-exchange-rates.json
    [junit4] 2> 1218512 T27743
    oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit:
    solr.admin.EditFileRequestHandler
    [junit4] 2> 1218512 T27754 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
    [junit4] 2> 1218513 T27769
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
    IMPORTANT NOTE
    [junit4] 2> 1218513 T27754 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
    [junit4] 2> 1218513 T27769
    oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
    key, got STRING
    [junit4] 2> 1218514 T27754 oasup.UpdateRequestProcessorChain.init
    inserting DistributedUpdateProcessorFactory into
    updateRequestProcessorChain "distrib-dup-test-chain-implicit"
    [junit4] 2> 1218514 T27769 oasc.CoreContainer.create Creating
    SolrCore 'solrj_collection_shard1_replica1' using configuration from
    collection solrj_collection
    [junit4] 2> 1218514 T27754 oasc.SolrCore.loadUpdateProcessorChains no
    updateRequestProcessorChain defined as default, creating implicit default
    [junit4] 2> 1218515 T27769 oasc.SolrCore.initDirectoryFactory
    org.apache.solr.core.MockDirectoryFactory
    [junit4] 2> 1218515 T27769 oasc.SolrCore.<init>
    [solrj_collection_shard1_replica1] Opening new SolrCore at
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/,
    dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/data/
    [junit4] 2> 1218515 T27754
    oasc.RequestHandlers.initHandlersFromConfig created /update/json:
    org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1218516 T27769 oasc.JmxMonitoredMap.<init> JMX monitoring
    is enabled. Adding Solr mbeans to JMX Server:
    com.sun.jmx.mbeanserver.JmxMBeanServer@3eb6814a
    [junit4] 2> 1218516 T27754
    oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs:
    org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1218517 T27754
    oasc.RequestHandlers.initHandlersFromConfig created /update/csv:
    org.apache.solr.handler.UpdateRequestHandler
    [junit4] 2> 1218518 T27754
    oasc.RequestHandlers.initHandlersFromConfig created standard:
    solr.StandardRequestHandler
    [junit4] 2> 1218518 T27754
    oasc.RequestHandlers.initHandlersFromConfig created /get:
    solr.RealTimeGetHandler
    [junit4] 2> 1218519 T27754
    oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler:
    solr.ReplicationHandler
    [junit4] 2> 1218519 T27754
    oasc.RequestHandlers.initHandlersFromConfig created /replication:
    solr.ReplicationHandler
    [junit4] 2> 1218520 T27754
    oasc.RequestHandlers.initHandlersFromConfig created /update:
    solr.UpdateRequestHandler
    [junit4] 2> 1218520 T27754
    oasc.RequestHandlers.initHandlersFromConfig created /admin/:
    org.apache.solr.handler.admin.AdminHandlers
    [junit4] 2> 1218521 T27769 oasc.CachingDirectoryFactory.get return
    new directory for
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/data
    [junit4] 2> 1218521 T27754
    oasc.RequestHandlers.initHandlersFromConfig created /admin/file:
    solr.admin.ShowFileRequestHandler
    [junit4] 2> 1218522 T27754
    oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit:
    solr.admin.EditFileRequestHandler
    [junit4] 2> 1218523 T27769 oasc.SolrCore.getNewIndexDir New index
    directory detected: old=null
    new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/data/index/
    [junit4] 2> 1218523 T27769 oasc.SolrCore.initIndex WARN
    [solrj_collection_shard1_replica1] Solr index directory
    '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/data/index'
    doesn't exist. Creating new index...
    [junit4] 2> 1218524 T27769 oasc.CachingDirectoryFactory.get return
    new directory for
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/solrj_collection_shard1_replica1/data/index
    [junit4] 2> 1218525 T27769 oasu.RandomMergePolicy.<init>
    RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy:
    [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=19,
    maxMergedSegmentMB=97.9267578125, floorSegmentMB=1.2958984375,
    forceMergeDeletesPctAllowed=11.900176490131441, segmentsPerTier=49.0,
    maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
    [junit4] 2> 1218527 T27769 oasc.SolrDeletionPolicy.onCommit
    SolrDeletionPolicy.onCommit: commits: num=1
    [junit4] 2>
    commit{dir=MockDirectoryWrapper(RAMDirectory(id=lucene-21a88cd6)),segFN=segments_1,generation=1}
    [junit4] 2> 1218527 T27769 oasc.SolrDeletionPolicy.updateCommits
    newest commit generation = 1
    [junit4] 2> 1218533 T27769 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "nodistrib"
    [junit4] 2> 1218533 T27724 oashl.XMLLoader.init
    xsltCacheLifetimeSeconds=60
    [junit4] 2> 1218533 T27754 oashl.XMLLoader.init
    xsltCacheLifetimeSeconds=60
    [junit4] 2> 1218533 T27769 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "dedupe"
    [junit4] 2> 1218533 T27743 oashl.XMLLoader.init
    xsltCacheLifetimeSeconds=60
    [junit4] 2> 1218534 T27769 oasup.UpdateRequestProcessorChain.init
    inserting DistributedUpdateProcessorFactory into
    updateRequestProcessorChain "dedupe"
    [junit4] 2> 1218535 T27769 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "stored_sig"
    [junit4] 2> 1218535 T27769 oasup.UpdateRequestProcessorChain.init
    inserting DistributedUpdateProcessorFactory into
    updateRequestProcessorChain "stored_sig"
    [junit4] 2> 1218536 T27769 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
    [junit4] 2> 1218536 T27754 oashl.XMLLoader.init
    xsltCacheLifetimeSeconds=60
    [junit4] 2> 1218536 T27724 oashl.XMLLoader.init
    xsltCacheLifetimeSeconds=60
    [junit4] 2> 1218537 T27743 oashl.XMLLoader.init
    xsltCacheLifetimeSeconds=60
    [junit4] 2> 1218537 T27769 oasup.UpdateRequestProcessorChain.init
    creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
    [junit4] 2> 1218538 T27769 oasup.UpdateRequestProcessorChain.init
    inserting DistributedUpdateProcessorFactory into
    updateRequestProcessorChain "distrib-dup-t

    [...truncated too long message...]

    home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/addReplicaColl_shard2_replica3/data
    [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/addReplicaColl_shard2_replica3/data;done=false>>]
    [junit4] 2> 1369735 T27685 oasc.CachingDirectoryFactory.close Closing
    directory:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/addReplicaColl_shard2_replica3/data
    [junit4] 2> 1369735 T27685
    oasc.CachingDirectoryFactory.closeCacheValue looking to close
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/addReplicaColl_shard2_replica3/data/index
    [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/addReplicaColl_shard2_replica3/data/index;done=false>>]
    [junit4] 2> 1369736 T27685 oasc.CachingDirectoryFactory.close Closing
    directory:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-901F7D855C61C262-001/tempDir-006/addReplicaColl_shard2_replica3/data/index
    [junit4] 2> 1369737 T27685 oasc.Overseer.close Overseer
    (id=92702442030432271-127.0.0.1:63988_-n_0000000004) closing
    [junit4] 2> 1369738 T29287 oasc.Overseer$ClusterStateUpdater.run
    Overseer Loop exiting : 127.0.0.1:63988_
    [junit4] 2> 1369751 T27976 oascc.ZkStateReader$3.process WARN
    ZooKeeper watch triggered, but Solr cannot talk to ZK
    [junit4] 2> 1369756 T27685 oejsh.ContextHandler.doStop stopped
    o.e.j.s.ServletContextHandler{/,null}
    [junit4] 2> 1370474 T27685 oasc.ChaosMonkey.monkeyLog monkey: stop
    shard! 63972
    [junit4] 2> 1370475 T27685 oas.SolrTestCaseJ4.tearDown ###Ending
    testDistribSearch
    [junit4] 2> 1370477 T27685 oasc.ZkTestServer.send4LetterWord
    connecting to 127.0.0.1:63966 63966
    [junit4] 2> 1370659 T27969 oasc.ZkTestServer.send4LetterWord
    connecting to 127.0.0.1:63966 63966
    [junit4] 2> NOTE: download the large Jenkins line-docs file by
    running 'ant get-jenkins-line-docs' in the lucene directory.
    [junit4] 2> NOTE: reproduce with: ant test
    -Dtestcase=CollectionsAPIDistributedZkTest -Dtests.method=testDistribSearch
    -Dtests.seed=901F7D855C61C262 -Dtests.multiplier=2 -Dtests.nightly=true
    -Dtests.slow=true
    -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt
    -Dtests.locale=pt_BR -Dtests.timezone=Pacific/Norfolk -Dtests.asserts=true
    -Dtests.file.encoding=US-ASCII
    [junit4] ERROR 166s J3 |
    CollectionsAPIDistributedZkTest.testDistribSearch <<<
    [junit4] > Throwable #1:
    com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
    uncaught exception in thread: Thread[id=28050, name=Thread-5840,
    state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
    [junit4] > Caused by: java.lang.NullPointerException
    [junit4] > at
    __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
    [junit4] > at
    org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)Throwable
    #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
    uncaught exception in thread: Thread[id=28048, name=Thread-5838,
    state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
    [junit4] > Caused by: java.lang.NullPointerException
    [junit4] > at
    __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
    [junit4] > at
    org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)Throwable
    #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
    uncaught exception in thread: Thread[id=28049, name=Thread-5839,
    state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
    [junit4] > Caused by: java.lang.NullPointerException
    [junit4] > at
    __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
    [junit4] > at
    org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)Throwable
    #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
    uncaught exception in thread: Thread[id=28053, name=Thread-5843,
    state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
    [junit4] > Caused by: java.lang.NullPointerException
    [junit4] > at
    __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
    [junit4] > at
    org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)Throwable
    #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
    uncaught exception in thread: Thread[id=28051, name=Thread-5841,
    state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
    [junit4] > Caused by: java.lang.NullPointerException
    [junit4] > at
    __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
    [junit4] > at
    org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)Throwable
    #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
    uncaught exception in thread: Thread[id=28052, name=Thread-5842,
    state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
    [junit4] > Caused by: java.lang.NullPointerException
    [junit4] > at
    __randomizedtesting.SeedInfo.seed([901F7D855C61C262]:0)
    [junit4] > at
    org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:1044)
    [junit4] 2> 1370705 T27685 oas.SolrTestCaseJ4.deleteCore ###deleteCore
    [junit4] 2> NOTE: test params are: codec=Asserting(Lucene50):
    {multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval),
    _version_=BlockTreeOrds(blocksize=128),
    intDefault=BlockTreeOrds(blocksize=128),
    range_facet_l=PostingsFormat(name=LuceneVarGapFixedInterval),
    other_tl1=BlockTreeOrds(blocksize=128),
    a_i1=PostingsFormat(name=LuceneVarGapFixedInterval),
    timestamp=PostingsFormat(name=LuceneVarGapFixedInterval),
    text=PostingsFormat(name=MockRandom),
    a_t=PostingsFormat(name=LuceneVarGapFixedInterval), id=FST50},
    docValues:{timestamp=DocValuesFormat(name=Asserting)},
    sim=DefaultSimilarity, locale=pt_BR, timezone=Pacific/Norfolk
    [junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation
    1.7.0_65 (64-bit)/cpus=16,threads=1,free=211097640,total=522190848
    [junit4] 2> NOTE: All tests run in this JVM:
    [StatelessScriptUpdateProcessorFactoryTest, IndexBasedSpellCheckerTest,
    TestSearcherReuse, TestMergePolicyConfig,
    DocumentAnalysisRequestHandlerTest, SpellPossibilityIteratorTest,
    TestFastLRUCache, AliasIntegrationTest, SolrInfoMBeanTest, TestSearchPerf,
    HdfsWriteToMultipleCollectionsTest, TestLMJelinekMercerSimilarityFactory,
    LoggingHandlerTest, CollectionsAPIAsyncDistributedZkTest,
    TimeZoneUtilsTest, TestPhraseSuggestions, TestPHPSerializedResponseWriter,
    DOMUtilTest, TestCSVLoader, TestNonNRTOpen, SOLR749Test, TestIndexSearcher,
    TestOverriddenPrefixQueryForCustomFieldType, DistributedDebugComponentTest,
    UniqFieldsUpdateProcessorFactoryTest, TestMiniSolrCloudCluster,
    TestDistributedMissingSort, TestReplicaProperties,
    CoreContainerCoreInitFailuresTest, TestInfoStreamLogging,
    FileBasedSpellCheckerTest, TestDistributedGrouping, TestLRUCache,
    DistributedFacetPivotLongTailTest, ShardRoutingTest,
    TestAnalyzeInfixSuggestions, TestConfigSets, RequestHandlersTest,
    CursorPagingTest, ResponseLogComponentTest,
    TermVectorComponentDistributedTest, EchoParamsTest, TestModifyConfFiles,
    OverseerStatusTest, FieldMutatingUpdateProcessorTest,
    UnloadDistributedZkTest, ZkCLITest, TestDocSet,
    DocExpirationUpdateProcessorFactoryTest, AtomicUpdatesTest,
    OutputWriterTest, TestSort, ChaosMonkeySafeLeaderTest,
    CollectionsAPIDistributedZkTest]
    [junit4] Completed on J3 in 170.41s, 1 test, 1 error <<< FAILURES!

    [...truncated 850 lines...]
    BUILD FAILED
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:532:
    The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:473:
    The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61:
    The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39:
    The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:189:
    The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:496:
    The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1359:
    The following error occurred while executing this line:
    /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:966:
    There were test failures: 438 suites, 1800 tests, 1 error, 54 ignored (9
    assumptions)

    Total time: 224 minutes 20 seconds
    Build step 'Invoke Ant' marked build as failure
    Archiving artifacts
    Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #654
    Archived 5 artifacts
    Archive block size is 32768
    Received 0 blocks and 419876900 bytes
    Compression is 0.0%
    Took 1 min 15 sec
    Recording test results
    Email was triggered for: Failure
    Sending email for trigger: Failure




    ---------------------------------------------------------------------
    To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
    For additional commands, e-mail: dev-help@lucene.apache.org


    --
    Regards,
    Shalin Shekhar Mangar.
  • Apache Jenkins Server at Oct 30, 2014 at 1:01 am
    Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/662/

    1 tests failed.
    FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler

    Error Message:
    Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4131, name=coreLoadExecutor-1670-thread-1, state=RUNNABLE, group=TGRP-TestReplicationHandler], registration stack trace below.

    Stack Trace:
    com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4131, name=coreLoadExecutor-1670-thread-1, state=RUNNABLE, group=TGRP-TestReplicationHandler], registration stack trace below.
      at __randomizedtesting.SeedInfo.seed([800168CC91F75BB2]:0)
      at java.lang.Thread.getStackTrace(Thread.java:1589)
      at com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:166)
      at org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:728)
      at org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1314)
      at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1205)
      at org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1197)
      at org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:47)
      at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:350)
      at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:276)
      at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:488)
      at org.apache.solr.core.SolrCore.(SolrCore.java:652)
      at org.apache.solr.core.CoreContainer.create(CoreContainer.java:509)
      at org.apache.solr.core.CoreContainer$1.call(CoreContainer.java:273)
      at org.apache.solr.core.CoreContainer$1.call(CoreContainer.java:267)
      at java.util.concurrent.FutureTask.run(FutureTask.java:262)
      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
      at java.lang.Thread.run(Thread.java:745)
    Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(SimpleFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/index-SimpleFSDirectory-116 lockFactory=NativeFSLockFactory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/index-SimpleFSDirectory-116)
      at org.junit.Assert.fail(Assert.java:93)
      at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:47)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$2$1.apply(RandomizedRunner.java:699)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$2$1.apply(RandomizedRunner.java:696)
      at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:183)
      at com.carrotsearch.randomizedtesting.RandomizedRunner$2.afterAlways(RandomizedRunner.java:712)
      at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
      at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
      ... 1 more




    Build Log:
    [...truncated 12631 lines...]
        [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
        [junit4] 2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/init-core-data-001
        [junit4] 2> 1421040 T3509 oas.SolrTestCaseJ4.setUp ###Starting doTestReplicateAfterCoreReload
        [junit4] 2> 1421056 T3509 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 1421060 T3509 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:54137
        [junit4] 2> 1421060 T3509 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 1421061 T3509 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 1421061 T3509 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001
        [junit4] 2> 1421061 T3509 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/'
        [junit4] 2> 1421106 T3509 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/solr.xml
        [junit4] 2> 1421186 T3509 oasc.CoreContainer.<init> New CoreContainer 798608293
        [junit4] 2> 1421186 T3509 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/]
        [junit4] 2> 1421188 T3509 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 1421188 T3509 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 1421188 T3509 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 1421189 T3509 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 1421189 T3509 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 1421189 T3509 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 1421190 T3509 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 1421190 T3509 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 1421190 T3509 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 1421191 T3509 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 1421191 T3509 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 1421192 T3509 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 1421192 T3509 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 1421192 T3509 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 1421198 T3519 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/'
        [junit4] 2> 1421230 T3519 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 1421258 T3519 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 1421259 T3519 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/conf/schema.xml
        [junit4] 2> 1421263 T3519 oass.IndexSchema.readSchema [collection1] Schema name=test
        [junit4] 2> 1421275 T3519 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 1421275 T3519 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/
        [junit4] 2> 1421276 T3519 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 1421276 T3519 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/data/
        [junit4] 2> 1421276 T3519 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
        [junit4] 2> 1421277 T3519 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/data
        [junit4] 2> 1421278 T3519 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/data/index/
        [junit4] 2> 1421278 T3519 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/data/index' doesn't exist. Creating new index...
        [junit4] 2> 1421278 T3519 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/data/index
        [junit4] 2> 1421279 T3519 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=4.3681640625, floorSegmentMB=0.9501953125, forceMergeDeletesPctAllowed=8.299167170192748, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7467634060416238
        [junit4] 2> 1421279 T3519 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-7412dec1)),segFN=segments_1,generation=1}
        [junit4] 2> 1421279 T3519 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 1421280 T3519 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 1421280 T3519 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1421281 T3519 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1421281 T3519 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1421281 T3519 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 1421281 T3519 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 1421281 T3519 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
        [junit4] 2> 1421282 T3519 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
        [junit4] 2> 1421282 T3519 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
        [junit4] 2> 1421282 T3519 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 1421294 T3519 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1421295 T3519 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1421296 T3519 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1421297 T3519 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1421299 T3519 oasu.CommitTracker.<init> Hard AutoCommit: disabled
        [junit4] 2> 1421299 T3519 oasu.CommitTracker.<init> Soft AutoCommit: disabled
        [junit4] 2> 1421299 T3519 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=50, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7751748373476876]
        [junit4] 2> 1421300 T3519 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-7412dec1)),segFN=segments_1,generation=1}
        [junit4] 2> 1421300 T3519 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 1421300 T3519 oass.SolrIndexSearcher.<init> Opening Searcher@cebb37a[collection1] main
        [junit4] 2> 1421301 T3519 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/conf
        [junit4] 2> 1421301 T3519 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/conf}
        [junit4] 2> 1421301 T3519 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/conf
        [junit4] 2> 1421302 T3519 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
        [junit4] 2> 1421302 T3519 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using file:dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/conf
        [junit4] 2> 1421302 T3519 oasr.RestManager.init Initializing 0 registered ManagedResources
        [junit4] 2> 1421302 T3519 oash.ReplicationHandler.inform Replication enabled for following config files: schema.xml,xslt/dummy.xsl
        [junit4] 2> 1421303 T3519 oash.ReplicationHandler.inform Commits will be reserved for 10000
        [junit4] 2> 1421303 T3519 oasc.CoreContainer.registerCore registering core: collection1
        [junit4] 2> 1421303 T3520 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@cebb37a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
        [junit4] 2> 1421303 T3509 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2
        [junit4] 2> 1421304 T3509 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 1421315 T3509 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 1421318 T3509 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:54138
        [junit4] 2> 1421319 T3509 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 1421319 T3509 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 1421319 T3509 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002
        [junit4] 2> 1421320 T3509 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/'
        [junit4] 2> 1421337 T3509 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/solr.xml
        [junit4] 2> 1421383 T3509 oasc.CoreContainer.<init> New CoreContainer 293150267
        [junit4] 2> 1421383 T3509 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/]
        [junit4] 2> 1421385 T3509 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 1421385 T3509 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 1421385 T3509 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 1421386 T3509 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
        [junit4] 2> 1421386 T3509 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
        [junit4] 2> 1421386 T3509 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
        [junit4] 2> 1421387 T3509 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
        [junit4] 2> 1421387 T3509 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
        [junit4] 2> 1421387 T3509 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
        [junit4] 2> 1421388 T3509 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
        [junit4] 2> 1421388 T3509 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
        [junit4] 2> 1421389 T3509 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
        [junit4] 2> 1421389 T3509 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
        [junit4] 2> 1421389 T3509 oasc.CoreContainer.load Host Name: 127.0.0.1
        [junit4] 2> 1421395 T3531 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/'
        [junit4] 2> 1421426 T3531 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
        [junit4] 2> 1421697 T3531 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
        [junit4] 2> 1421698 T3531 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/conf/schema.xml
        [junit4] 2> 1421705 T3531 oass.IndexSchema.readSchema [collection1] Schema name=test
        [junit4] 2> 1421716 T3531 oass.IndexSchema.readSchema unique key field: id
        [junit4] 2> 1421717 T3531 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/
        [junit4] 2> 1421718 T3531 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
        [junit4] 2> 1421718 T3531 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/data/
        [junit4] 2> 1421718 T3531 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
        [junit4] 2> 1421720 T3531 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/data
        [junit4] 2> 1421721 T3531 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/data/index/
        [junit4] 2> 1421721 T3531 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/data/index' doesn't exist. Creating new index...
        [junit4] 2> 1421723 T3531 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/data/index
        [junit4] 2> 1421723 T3531 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=4.3681640625, floorSegmentMB=0.9501953125, forceMergeDeletesPctAllowed=8.299167170192748, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7467634060416238
        [junit4] 2> 1421724 T3531 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-a744469)),segFN=segments_1,generation=1}
        [junit4] 2> 1421724 T3531 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 1421725 T3531 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
        [junit4] 2> 1421725 T3531 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1421726 T3531 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1421726 T3531 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
        [junit4] 2> 1421726 T3531 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
        [junit4] 2> 1421727 T3531 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
        [junit4] 2> 1421727 T3531 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
        [junit4] 2> 1421727 T3531 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
        [junit4] 2> 1421727 T3531 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
        [junit4] 2> 1421727 T3531 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
        [junit4] 2> 1421729 T3531 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1421731 T3531 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1421733 T3531 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1421735 T3531 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
        [junit4] 2> 1421737 T3531 oasu.CommitTracker.<init> Hard AutoCommit: disabled
        [junit4] 2> 1421738 T3531 oasu.CommitTracker.<init> Soft AutoCommit: disabled
        [junit4] 2> 1421738 T3531 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=50, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7751748373476876]
        [junit4] 2> 1421739 T3531 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
        [junit4] 2> commit{dir=BaseDirectoryWrapper(RAMDirectory(id=lucene-a744469)),segFN=segments_1,generation=1}
        [junit4] 2> 1421739 T3531 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
        [junit4] 2> 1421739 T3531 oass.SolrIndexSearcher.<init> Opening Searcher@4bcb8241[collection1] main
        [junit4] 2> 1421740 T3531 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/conf
        [junit4] 2> 1421740 T3531 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/conf}
        [junit4] 2> 1421740 T3531 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/conf
        [junit4] 2> 1421741 T3531 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
        [junit4] 2> 1421742 T3531 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using file:dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/conf
        [junit4] 2> 1421742 T3531 oasr.RestManager.init Initializing 0 registered ManagedResources
        [junit4] 2> 1421748 T3531 oash.SnapPuller.startExecutorService Poll Scheduled at an interval of 1000ms
        [junit4] 2> 1421749 T3532 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4bcb8241[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
        [junit4] 2> 1421749 T3531 oasc.CoreContainer.registerCore registering core: collection1
        [junit4] 2> 1421750 T3509 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2
        [junit4] 2> 1421750 T3509 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
        [junit4] 2> 1421751 T3509 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=293150267
        [junit4] 2> 1421752 T3509 oasc.SolrCore.close [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@340a3fa8
        [junit4] 2> 1421753 T3509 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
        [junit4] 2> 1421753 T3509 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
        [junit4] 2> 1421753 T3509 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
        [junit4] 2> 1421754 T3509 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
        [junit4] 2> 1421755 T3509 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
        [junit4] 2> 1421756 T3509 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
        [junit4] 2> 1421756 T3509 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/data/index;done=false>>]
        [junit4] 2> 1421756 T3509 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/data/index
        [junit4] 2> 1421757 T3509 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/data;done=false>>]
        [junit4] 2> 1421757 T3509 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-002/collection1/data
        [junit4] 2> 1421758 T3509 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
        [junit4] 2> 1426323 T3509 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=798608293
        [junit4] 2> 1426325 T3509 oasc.SolrCore.close [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@6057fb97
        [junit4] 2> 1426325 T3509 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
        [junit4] 2> 1426325 T3509 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
        [junit4] 2> 1426326 T3509 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
        [junit4] 2> 1426326 T3509 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
        [junit4] 2> 1426328 T3509 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
        [junit4] 2> 1426328 T3509 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
        [junit4] 2> 1426328 T3509 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/data/index;done=false>>]
        [junit4] 2> 1426329 T3509 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/data/index
        [junit4] 2> 1426329 T3509 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/data;done=false>>]
        [junit4] 2> 1426330 T3509 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/collection1/data
        [junit4] 2> 1426331 T3509 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
        [junit4] 2> 1431062 T3509 oejs.Server.doStart jetty-8.1.10.v20130312
        [junit4] 2> 1431066 T3509 oejs.AbstractConnector.doStart Started socketconnector@127.0.0.1:54140
        [junit4] 2> 1431066 T3509 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
        [junit4] 2> 1431067 T3509 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
        [junit4] 2> 1431067 T3509 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001
        [junit4] 2> 1431067 T3509 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/'
        [junit4] 2> 1431084 T3509 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/solr.xml
        [junit4] 2> 1431128 T3509 oasc.CoreContainer.<init> New CoreContainer 1650848693
        [junit4] 2> 1431129 T3509 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler-800168CC91F75BB2-001/solr-instance-001/]
        [junit4] 2> 1431130 T3509 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
        [junit4] 2> 1431130 T3509 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to:
        [junit4] 2> 1431131 T3509 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
        [junit4] 2> 1431131 T3509 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHo