lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Policeman Jenkins Server <jenk...@thetaphi.de>
Subject [JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.7.0) - Build # 700 - Still Failing!
Date Sat, 03 Aug 2013 01:02:45 GMT
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/700/
Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseParallelGC

2 tests failed.
REGRESSION:  org.apache.solr.core.OpenCloseCoreStressTest.test30SecondsNew

Error Message:
Core 00000_core bad! expected:<706> but was:<0>

Stack Trace:
java.lang.AssertionError: Core 00000_core bad! expected:<706> but was:<0>
	at __randomizedtesting.SeedInfo.seed([3FFD2EFC80FB9677:5B483228D23B6F32]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.core.OpenCloseCoreStressTest.checkResults(OpenCloseCoreStressTest.java:307)
	at org.apache.solr.core.OpenCloseCoreStressTest.doStress(OpenCloseCoreStressTest.java:201)
	at org.apache.solr.core.OpenCloseCoreStressTest.test30SecondsNew(OpenCloseCoreStressTest.java:121)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
Server refused connection at: http://127.0.0.1:50255/_j/mt

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:50255/_j/mt
	at __randomizedtesting.SeedInfo.seed([3FFD2EFC80FB9677:BE1BA0E4F7A4F64B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:428)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1535)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCustomCollectionsAPI(CollectionsAPIDistributedZkTest.java:677)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:159)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)
Caused by: org.apache.http.conn.HttpHostConnectException: Connection to http://127.0.0.1:50255 refused
	at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:190)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
	at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
	at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
	at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
	at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
	... 45 more
Caused by: java.net.ConnectException: Connection refused
	at java.net.PlainSocketImpl.socketConnect(Native Method)
	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
	at java.net.Socket.connect(Socket.java:579)
	at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
	at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
	... 52 more




Build Log:
[...truncated 9748 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> 1973328 T4356 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_j/mt
   [junit4]   2> 1973336 T4356 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./solrtest-CollectionsAPIDistributedZkTest-1375489643988
   [junit4]   2> 1973338 T4356 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 1973340 T4357 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1973440 T4356 oasc.ZkTestServer.run start zk server on port:50245
   [junit4]   2> 1973446 T4356 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1973462 T4363 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@34bcca42 name:ZooKeeperConnection Watcher:127.0.0.1:50245 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1973462 T4356 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1973463 T4356 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1973481 T4356 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1973487 T4365 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6d5bc478 name:ZooKeeperConnection Watcher:127.0.0.1:50245/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1973488 T4356 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1973488 T4356 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1973498 T4356 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1973520 T4356 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1973527 T4356 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1973538 T4356 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1973541 T4356 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1973591 T4356 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1973593 T4356 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1973614 T4356 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1973619 T4356 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1973628 T4356 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1973629 T4356 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1973643 T4356 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1973644 T4356 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1973653 T4356 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1973654 T4356 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1973677 T4356 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1973678 T4356 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1973691 T4356 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1973692 T4356 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1973710 T4356 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1973712 T4356 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1974372 T4356 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1974381 T4356 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:?????
   [junit4]   2> 1974382 T4356 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1974382 T4356 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1974383 T4356 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374
   [junit4]   2> 1974383 T4356 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374/'
   [junit4]   2> 1974438 T4356 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374/solr.xml
   [junit4]   2> 1974559 T4356 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1974560 T4356 oasc.CoreContainer.<init> New CoreContainer 1853126241
   [junit4]   2> 1974560 T4356 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374/]
   [junit4]   2> 1974563 T4356 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1974564 T4356 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1974564 T4356 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1974564 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1974565 T4356 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1974565 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1974566 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1974567 T4356 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1974567 T4356 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1974568 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1974592 T4356 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50245/solr
   [junit4]   2> 1974593 T4356 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1974597 T4356 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1974601 T4376 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@330e5cab name:ZooKeeperConnection Watcher:127.0.0.1:50245 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1974602 T4356 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1974607 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1974625 T4356 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1974630 T4378 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1cd31057 name:ZooKeeperConnection Watcher:127.0.0.1:50245/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1974631 T4356 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1974634 T4356 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1974643 T4356 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1974651 T4356 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1974656 T4356 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50248__j%2Fmt
   [junit4]   2> 1974660 T4356 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50248__j%2Fmt
   [junit4]   2> 1974674 T4356 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1974687 T4356 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1974693 T4356 oasc.Overseer.start Overseer (id=90144089308921859-127.0.0.1:50248__j%2Fmt-n_0000000000) starting
   [junit4]   2> 1974701 T4356 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1974714 T4379 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1974715 T4380 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1974722 T4356 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1974728 T4356 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1974733 T4356 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1974748 T4356 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374
   [junit4]   2> 1974897 T4356 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374/conf/
   [junit4]   2> 1974905 T4356 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1974906 T4356 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1974907 T4356 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1974907 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1974936 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:
   [junit4]   2> 1974939 T4356 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1974945 T4383 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6cc849d1 name:ZooKeeperConnection Watcher:127.0.0.1:50245/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1974947 T4356 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1974957 T4356 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1974966 T4356 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
   [junit4]   2> 1975576 T4356 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1975583 T4356 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:?????
   [junit4]   2> 1975585 T4356 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1975585 T4356 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1975586 T4356 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1375489645617
   [junit4]   2> 1975586 T4356 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1375489645617/'
   [junit4]   2> 1975658 T4356 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1375489645617/solr.xml
   [junit4]   2> 1975887 T4356 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1975887 T4356 oasc.CoreContainer.<init> New CoreContainer 1836513948
   [junit4]   2> 1975888 T4356 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1375489645617/]
   [junit4]   2> 1975890 T4356 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1975891 T4356 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1975891 T4356 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1975892 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1975892 T4356 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1975893 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1975893 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1975894 T4356 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1975894 T4356 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1975894 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1975913 T4356 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50245/solr
   [junit4]   2> 1975914 T4356 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1975916 T4356 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1975924 T4394 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@32758e1d name:ZooKeeperConnection Watcher:127.0.0.1:50245 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1975928 T4356 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1975937 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1975954 T4356 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1975958 T4396 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38989292 name:ZooKeeperConnection Watcher:127.0.0.1:50245/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1975959 T4356 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1975970 T4356 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1976980 T4356 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50252__j%2Fmt
   [junit4]   2> 1976984 T4356 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50252__j%2Fmt
   [junit4]   2> 1976995 T4378 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1976998 T4396 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1976998 T4383 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1977015 T4356 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1375489645617
   [junit4]   2> 1977225 T4356 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1375489645617/conf/
   [junit4]   2> 1977232 T4356 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1977233 T4356 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1977233 T4356 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1977234 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1977771 T4356 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1977779 T4356 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:?????
   [junit4]   2> 1977780 T4356 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1977781 T4356 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1977782 T4356 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910
   [junit4]   2> 1977783 T4356 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910/'
   [junit4]   2> 1977848 T4356 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910/solr.xml
   [junit4]   2> 1977987 T4356 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1977988 T4356 oasc.CoreContainer.<init> New CoreContainer 2125438562
   [junit4]   2> 1977988 T4356 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910/]
   [junit4]   2> 1977991 T4356 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1977992 T4356 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1977993 T4356 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1977994 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1977994 T4356 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1977995 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1977995 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1977996 T4356 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1977996 T4356 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1977997 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1978020 T4356 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50245/solr
   [junit4]   2> 1978021 T4356 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1978023 T4356 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1978030 T4408 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@390e9a03 name:ZooKeeperConnection Watcher:127.0.0.1:50245 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1978032 T4356 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1978041 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1978066 T4356 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1978069 T4410 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7140e344 name:ZooKeeperConnection Watcher:127.0.0.1:50245/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1978069 T4356 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1978085 T4356 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1979102 T4356 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50255__j%2Fmt
   [junit4]   2> 1979105 T4356 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50255__j%2Fmt
   [junit4]   2> 1979114 T4383 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1979129 T4356 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910
   [junit4]   2> 1979158 T4410 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1979159 T4396 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1979159 T4378 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1979321 T4356 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910/conf/
   [junit4]   2> 1979330 T4356 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1979331 T4356 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1979331 T4356 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1979332 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1979896 T4356 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1979901 T4356 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:?????
   [junit4]   2> 1979902 T4356 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1979902 T4356 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1979903 T4356 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007
   [junit4]   2> 1979903 T4356 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007/'
   [junit4]   2> 1979970 T4356 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007/solr.xml
   [junit4]   2> 1980090 T4356 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1980091 T4356 oasc.CoreContainer.<init> New CoreContainer 1666416585
   [junit4]   2> 1980091 T4356 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007/]
   [junit4]   2> 1980093 T4356 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1980094 T4356 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1980094 T4356 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1980094 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1980095 T4356 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1980095 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1980096 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1980096 T4356 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1980096 T4356 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1980097 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1980119 T4356 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50245/solr
   [junit4]   2> 1980120 T4356 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1980125 T4356 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1980127 T4422 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f5e84b9 name:ZooKeeperConnection Watcher:127.0.0.1:50245 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1980128 T4356 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1980131 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1980157 T4356 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1980161 T4424 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e42d784 name:ZooKeeperConnection Watcher:127.0.0.1:50245/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1980162 T4356 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1980176 T4356 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1981184 T4356 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50258__j%2Fmt
   [junit4]   2> 1981187 T4356 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50258__j%2Fmt
   [junit4]   2> 1981209 T4383 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1981210 T4396 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1981210 T4378 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1981210 T4410 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1981215 T4356 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007
   [junit4]   2> 1981219 T4424 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1981406 T4356 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007/conf/
   [junit4]   2> 1981415 T4356 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1981416 T4356 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1981417 T4356 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1981417 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1982045 T4356 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1982054 T4356 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:?????
   [junit4]   2> 1982055 T4356 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1982056 T4356 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1982056 T4356 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100
   [junit4]   2> 1982057 T4356 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100/'
   [junit4]   2> 1982116 T4356 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100/solr.xml
   [junit4]   2> 1982252 T4356 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1982253 T4356 oasc.CoreContainer.<init> New CoreContainer 1810013405
   [junit4]   2> 1982253 T4356 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100/]
   [junit4]   2> 1982256 T4356 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1982257 T4356 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1982257 T4356 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1982258 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1982259 T4356 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1982259 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1982260 T4356 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1982261 T4356 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1982262 T4356 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1982263 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1982286 T4356 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50245/solr
   [junit4]   2> 1982287 T4356 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1982292 T4356 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1982302 T4436 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7fb38a7 name:ZooKeeperConnection Watcher:127.0.0.1:50245 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1982302 T4356 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1982309 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1982334 T4356 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1982338 T4438 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f365f89 name:ZooKeeperConnection Watcher:127.0.0.1:50245/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1982339 T4356 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1982350 T4356 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1983362 T4356 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50261__j%2Fmt
   [junit4]   2> 1983366 T4356 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50261__j%2Fmt
   [junit4]   2> 1983377 T4410 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1983384 T4438 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1983385 T4378 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1983385 T4383 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1983386 T4424 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1983386 T4396 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1983401 T4356 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100
   [junit4]   2> 1983650 T4356 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100/conf/
   [junit4]   2> 1983661 T4356 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1983663 T4356 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1983663 T4356 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1983664 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1983701 T4356 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1983730 T4389 oasha.CollectionsHandler.handleCreateAction Creating Collection : numShards=2&name=nodes_used_collection&replicationFactor=2&action=CREATE&wt=javabin&version=2
   [junit4]   2> 1983734 T4378 oasc.DistributedQueue$LatchChildWatcher.process Watcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1983742 T4380 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2"}
   [junit4]   2> 1983743 T4380 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2"}
   [junit4]   2> 1983820 T4379 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1983822 T4379 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: nodes_used_collection
   [junit4]   2> 1983823 T4379 oasc.Overseer$ClusterStateUpdater.createCollection Create collection nodes_used_collection with shards [shard1, shard2]
   [junit4]   2> 1983837 T4424 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1983837 T4383 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1983837 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1983838 T4396 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1983838 T4438 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1983837 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1983953 T4380 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
   [junit4]   2> 1983954 T4380 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica1 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:50248__j%2Fmt
   [junit4]   2> 1983964 T4380 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica2 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:50258__j%2Fmt
   [junit4]   2> 1983966 T4380 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica1 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:50261__j%2Fmt
   [junit4]   2> 1983977 T4374 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica1 state=down
   [junit4]   2> 1983978 T4380 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica2 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:50255__j%2Fmt
   [junit4]   2> 1983986 T4374 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1983989 T4433 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica1 state=down
   [junit4]   2> 1983993 T4420 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica2 state=down
   [junit4]   2> 1983994 T4402 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica2 state=down
   [junit4]   2> 1983996 T4433 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1983998 T4402 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1983998 T4420 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1985360 T4379 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1985363 T4379 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 1985364 T4379 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica1",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50248__j%2Fmt",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50248/_j/mt"}
   [junit4]   2> 1985378 T4379 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 1985379 T4379 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica1",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50261__j%2Fmt",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50261/_j/mt"}
   [junit4]   2> 1985400 T4379 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 1985400 T4379 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node3",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica2",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50255__j%2Fmt",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50255/_j/mt"}
   [junit4]   2> 1985415 T4379 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 1985415 T4379 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node4",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica2",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50258__j%2Fmt",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50258/_j/mt"}
   [junit4]   2> 1985430 T4378 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1985432 T4383 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1985440 T4396 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1985441 T4424 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1985441 T4438 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1985442 T4410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1985992 T4374 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica1
   [junit4]   2> 1985994 T4374 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374/nodes_used_collection_shard1_replica1/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374/nodes_used_collection_shard1_replica1/core.properties (No such file or directory)
   [junit4]   2> 1986000 T4374 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374/nodes_used_collection_shard1_replica1
   [junit4]   2> 1986002 T4374 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 1986000 T4433 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica1
   [junit4]   2> 1986001 T4402 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica2
   [junit4]   2> 1986004 T4433 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100/nodes_used_collection_shard2_replica1/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100/nodes_used_collection_shard2_replica1/core.properties (No such file or directory)
   [junit4]   2> 1986005 T4433 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100/nodes_used_collection_shard2_replica1
   [junit4]   2> 1986005 T4402 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910/nodes_used_collection_shard2_replica2/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910/nodes_used_collection_shard2_replica2/core.properties (No such file or directory)
   [junit4]   2> 1986007 T4402 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica2' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910/nodes_used_collection_shard2_replica2
   [junit4]   2> 1986007 T4402 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 1986006 T4420 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica2
   [junit4]   2> 1986009 T4420 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007/nodes_used_collection_shard1_replica2/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007/nodes_used_collection_shard1_replica2/core.properties (No such file or directory)
   [junit4]   2> 1986010 T4420 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica2' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007/nodes_used_collection_shard1_replica2
   [junit4]   2> 1986010 T4420 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 1986006 T4433 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 1986005 T4374 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 1986012 T4374 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 1986016 T4420 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 1986017 T4420 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 1986018 T4433 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 1986018 T4433 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 1986021 T4402 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 1986022 T4402 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 1986025 T4374 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 1986025 T4374 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 1986029 T4433 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 1986029 T4433 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 1986030 T4420 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 1986031 T4420 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 1986041 T4402 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 1986041 T4402 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 1986044 T4420 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 1986044 T4374 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 1986045 T4433 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 1986051 T4420 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007/nodes_used_collection_shard1_replica2/'
   [junit4]   2> 1986052 T4374 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374/nodes_used_collection_shard1_replica1/'
   [junit4]   2> 1986051 T4433 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100/nodes_used_collection_shard2_replica1/'
   [junit4]   2> 1986068 T4402 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 1986070 T4402 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910/nodes_used_collection_shard2_replica2/'
   [junit4]   2> 1986445 T4420 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 1986456 T4433 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 1986480 T4402 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 1986483 T4374 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 1986724 T4433 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1986740 T4433 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1986741 T4374 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1986775 T4433 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica1] Schema name=test
   [junit4]   2> 1986775 T4420 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1986784 T4420 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1986800 T4374 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1986820 T4402 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1986831 T4402 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1986835 T4420 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica2] Schema name=test
   [junit4]   2> 1986845 T4374 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica1] Schema name=test
   [junit4]   2> 1986857 T4402 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica2] Schema name=test
   [junit4]   2> 1989655 T4420 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1989692 T4420 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1989694 T4433 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1989701 T4420 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1989724 T4433 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1989744 T4433 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1989770 T4374 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1989801 T4374 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1989803 T4402 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1989806 T4374 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1989818 T4402 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1989824 T4402 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1989853 T4420 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1989867 T4420 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1989872 T4433 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1989889 T4420 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1989897 T4433 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1989899 T4420 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1989901 T4420 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1989902 T4420 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1989913 T4420 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1989914 T4420 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1989916 T4420 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1989917 T4420 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica2] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007/nodes_used_collection_shard1_replica2/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007//nodes_used_collection_shard1_replica2/data/
   [junit4]   2> 1989918 T4420 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@35cc031
   [junit4]   2> 1989919 T4420 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007//nodes_used_collection_shard1_replica2/data
   [junit4]   2> 1989920 T4420 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007//nodes_used_collection_shard1_replica2/data/index/
   [junit4]   2> 1989924 T4420 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard1_replica2] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007/nodes_used_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1989921 T4433 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1989926 T4420 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1375489650007//nodes_used_collection_shard1_replica2/data/index
   [junit4]   2> 1989935 T4420 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@264fa400 lockFactory=org.apache.lucene.store.NativeFSLockFactory@54a4b898),segFN=segments_1,generation=1}
   [junit4]   2> 1989936 T4420 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1989939 T4374 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1989941 T4402 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1989946 T4433 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1989946 T4433 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1989947 T4433 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1989953 T4420 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1989955 T4420 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1989958 T4420 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1989959 T4420 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1989961 T4420 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1989961 T4420 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1989962 T4420 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1989964 T4420 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1989965 T4420 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1989978 T4433 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1989979 T4433 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1989979 T4433 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1989980 T4433 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100/nodes_used_collection_shard2_replica1/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100//nodes_used_collection_shard2_replica1/data/
   [junit4]   2> 1989980 T4433 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@35cc031
   [junit4]   2> 1989982 T4433 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100//nodes_used_collection_shard2_replica1/data
   [junit4]   2> 1989982 T4433 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100//nodes_used_collection_shard2_replica1/data/index/
   [junit4]   2> 1989989 T4433 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica1] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100/nodes_used_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1989991 T4374 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1989992 T4402 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1989993 T4433 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1375489652100//nodes_used_collection_shard2_replica1/data/index
   [junit4]   2> 1990006 T4374 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1990009 T4402 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1990015 T4402 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1990016 T4402 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1990019 T4402 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1990020 T4374 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1990021 T4374 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1990021 T4374 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1990031 T4402 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1990033 T4402 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1990033 T4402 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1990036 T4402 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica2] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910/nodes_used_collection_shard2_replica2/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910//nodes_used_collection_shard2_replica2/data/
   [junit4]   2> 1990037 T4402 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@35cc031
   [junit4]   2> 1990038 T4402 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910//nodes_used_collection_shard2_replica2/data
   [junit4]   2> 1990038 T4402 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910//nodes_used_collection_shard2_replica2/data/index/
   [junit4]   2> 1990039 T4402 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica2] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910/nodes_used_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1990043 T4420 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1990046 T4402 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1375489647910//nodes_used_collection_shard2_replica2/data/index
   [junit4]   2> 1990047 T4374 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1990051 T4374 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1990052 T4374 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1990053 T4374 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374/nodes_used_collection_shard1_replica1/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374//nodes_used_collection_shard1_replica1/data/
   [junit4]   2> 1990054 T4374 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@35cc031
   [junit4]   2> 1990056 T4374 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374//nodes_used_collection_shard1_replica1/data
   [junit4]   2> 1990058 T4374 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374//nodes_used_collection_shard1_replica1/data/index/
   [junit4]   2> 1990058 T4374 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard1_replica1] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374/nodes_used_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1990061 T4420 oass.SolrIndexSearcher.<init> Opening Searcher@23c9196 main
   [junit4]   2> 1990063 T4433 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@34396444 lockFactory=org.apache.lucene.store.NativeFSLockFactory@2815a96a),segFN=segments_1,generation=1}
   [junit4]   2> 1990064 T4433 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1990065 T4374 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1375489644374//nodes_used_collection_shard1_replica1/data/index
   [junit4]   2> 1990067 T4420 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1990067 T4420 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1990071 T4402 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@11eee54d lockFactory=org.apache.lucene.store.NativeFSLockFactory@3d9bd80d),segFN=segments_1,generation=1}
   [junit4]   2> 1990072 T4402 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1990083 T4374 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@242a0484 lockFactory=org.apache.lucene.store.NativeFSLockFactory@567a0fff),segFN=segments_1,generation=1}
   [junit4]   2> 1990084 T4374 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1990091 T4444 oasc.SolrCore.registerSearcher [nodes_used_collection_shard1_replica2] Registered new searcher Searcher@23c9196 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 1990106 T4433 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1990107 T4433 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1990111 T4433 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1990120 T4433 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1990124 T4433 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1990125 T4433 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1990126 T4433 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1990129 T4433 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1990131 T4420 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard1_replica2
   [junit4]   2> 1990135 T4420 oasc.ZkController.register Register replica - core:nodes_used_collection_shard1_replica2 address:http://127.0.0.1:50258/_j/mt collection:nodes_used_collection shard:shard1
   [junit4]   2> 1990136 T4420 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 1990134 T4402 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1990138 T4402 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1990140 T4402 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1990142 T4402 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1990131 T4433 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1990134 T4374 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1990152 T4374 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1990155 T4374 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1990157 T4374 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1990144 T4402 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1990160 T4402 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1990161 T4402 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1990168 T4402 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1990170 T4402 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1990162 T4374 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1990183 T4374 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1990184 T4374 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1990184 T4433 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1990188 T4402 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1990186 T4374 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1990195 T4374 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1990200 T4374 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1990213 T4420 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leader_elect/shard1/election
   [junit4]   2> 1990214 T4433 oass.SolrIndexSearcher.<init> Opening Searcher@b745fe3 main
   [junit4]   2> 1990220 T4433 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1990222 T4433 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1990228 T4374 oass.SolrIndexSearcher.<init> Opening Searcher@3c08ccb9 main
   [junit4]   2> 1990231 T4374 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1990232 T4374 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1990234 T4445 oasc.SolrCore.registerSearcher [nodes_used_collection_shard2_replica1] Registered new searcher Searcher@b745fe3 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   

[...truncated too long message...]

ng directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/OpenCloseCoreStressTest_/00001_core/data/index
   [junit4]   2> 2435016 T5215 oasc.SolrCore.close [00000_core]  CLOSING SolrCore org.apache.solr.core.SolrCore@7535c2a7
   [junit4]   2> 2435033 T5215 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=7,autocommits=0,soft autocommits=4,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=706,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 2435034 T5215 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2435034 T5215 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2435035 T5215 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2435073 T5215 C1267 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.MMapDirectory@/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/OpenCloseCoreStressTest_/00000_core/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@4fdb4fd5),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.MMapDirectory@/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/OpenCloseCoreStressTest_/00000_core/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@4fdb4fd5),segFN=segments_4,generation=4}
   [junit4]   2> 2435074 T5215 C1267 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 2435289 T5215 oasc.SolrCore.closeSearcher [00000_core] Closing main searcher on request.
   [junit4]   2> 2435427 T5215 oasc.CachingDirectoryFactory.close Closing MockFSDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2435428 T5215 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/OpenCloseCoreStressTest_/00000_core/data/index [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/OpenCloseCoreStressTest_/00000_core/data/index;done=false>>]
   [junit4]   2> 2435429 T5215 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/OpenCloseCoreStressTest_/00000_core/data/index
   [junit4]   2> 2435525 T5215 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/OpenCloseCoreStressTest_/00000_core/data [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/OpenCloseCoreStressTest_/00000_core/data;done=false>>]
   [junit4]   2> 2435526 T5215 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/OpenCloseCoreStressTest_/00000_core/data
   [junit4]   2> 2435527 T5215 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 2438614 T5215 oas.SolrTestCaseJ4.tearDown ###Ending test30SecondsNew
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=OpenCloseCoreStressTest -Dtests.method=test30SecondsNew -Dtests.seed=3FFD2EFC80FB9677 -Dtests.slow=true -Dtests.locale=fr_FR -Dtests.timezone=Asia/Hebron -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 49.3s | OpenCloseCoreStressTest.test30SecondsNew <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Core 00000_core bad! expected:<706> but was:<0>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3FFD2EFC80FB9677:5B483228D23B6F32]:0)
   [junit4]    > 	at org.apache.solr.core.OpenCloseCoreStressTest.checkResults(OpenCloseCoreStressTest.java:307)
   [junit4]    > 	at org.apache.solr.core.OpenCloseCoreStressTest.doStress(OpenCloseCoreStressTest.java:201)
   [junit4]    > 	at org.apache.solr.core.OpenCloseCoreStressTest.test30SecondsNew(OpenCloseCoreStressTest.java:121)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:724)
   [junit4] IGNOR/A 0.00s | OpenCloseCoreStressTest.test1HourOld
   [junit4]    > Assumption #1: 'weekly' test group is disabled (@Weekly)
   [junit4]   2> 2438628 T5215 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: test params are: codec=Lucene42: {id=PostingsFormat(name=Asserting), text=PostingsFormat(name=MockSep)}, docValues:{}, sim=DefaultSimilarity, locale=fr_FR, timezone=Asia/Hebron
   [junit4]   2> NOTE: Mac OS X 10.8.4 x86_64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=2,threads=1,free=135315384,total=434700288
   [junit4]   2> NOTE: All tests run in this JVM: [HdfsLockFactoryTest, PluginInfoTest, LegacyHTMLStripCharFilterTest, TestPluginEnable, DefaultValueUpdateProcessorTest, TestCodecSupport, PeerSyncTest, OutputWriterTest, ResponseLogComponentTest, HdfsSyncSliceTest, SolrRequestParserTest, TestInfoStreamLogging, BadComponentTest, TestSystemIdResolver, TestDistributedGrouping, TestManagedSchemaFieldResource, TestLRUCache, PreAnalyzedUpdateProcessorTest, TestRandomDVFaceting, TestSolrXml, SliceStateUpdateTest, TestDocumentBuilder, SuggesterTest, LeaderElectionTest, SolrTestCaseJ4Test, TestValueSourceCache, HdfsRecoveryZkTest, TestReversedWildcardFilterFactory, TestSchemaVersionResource, TestSolrQueryParserDefaultOperatorResource, TestManagedSchema, TestRTGBase, TestStressLucene, RAMDirectoryFactoryTest, LukeRequestHandlerTest, TestShardHandlerFactory, TestHashPartitioner, SolrIndexConfigTest, SolrInfoMBeanTest, SystemInfoHandlerTest, URLClassifyProcessorTest, PathHierarchyTokenizerFactoryTest, TestDocSet, DocValuesMultiTest, TestRemoteStreaming, TestFiltering, TestCoreContainer, SampleTest, MinimalSchemaTest, TestFunctionQuery, UpdateRequestProcessorFactoryTest, PreAnalyzedFieldTest, TestFastWriter, BlockDirectoryTest, DOMUtilTest, TestDistributedSearch, TestSerializedLuceneMatchVersion, FileUtilsTest, TestMaxScoreQueryParser, BadCopyFieldTest, TestRandomFaceting, TestSurroundQueryParser, SOLR749Test, ClusterStateTest, TestRecoveryHdfs, TestUpdate, FieldAnalysisRequestHandlerTest, TestJmxMonitoredMap, TestLFUCache, TestWordDelimiterFilterFactory, TestSchemaSimilarityResource, TestSweetSpotSimilarityFactory, MBeansHandlerTest, OpenExchangeRatesOrgProviderTest, TestBinaryField, ZkCLITest, TestSolrQueryParser, TestConfig, InfoHandlerTest, TestFoldingMultitermQuery, TermVectorComponentDistributedTest, TestSolrDeletionPolicy2, TestComponentsName, TestBadConfig, EchoParamsTest, TestRealTimeGet, TestPseudoReturnFields, OverseerCollectionProcessorTest, DateFieldTest, TestWriterPerf, TestGroupingSearch, TestBinaryResponseWriter, TestReplicationHandler, SoftAutoCommitTest, SuggesterFSTTest, FieldMutatingUpdateProcessorTest, TestStressReorder, ShardRoutingCustomTest, DebugComponentTest, TestDynamicFieldResource, IndexSchemaRuntimeFieldTest, TestPostingsSolrHighlighter, ChaosMonkeySafeLeaderTest, TestArbitraryIndexDir, TimeZoneUtilsTest, TestQueryUtils, XsltUpdateRequestHandlerTest, UpdateParamsTest, NoCacheHeaderTest, SolrCmdDistributorTest, SuggesterTSTTest, TestPerFieldSimilarity, BinaryUpdateRequestHandlerTest, DateMathParserTest, PrimUtilsTest, NotRequiredUniqueKeyTest, DistributedTermsComponentTest, TestPropInjectDefaults, TestExtendedDismaxParser, TestRangeQuery, LeaderElectionIntegrationTest, SolrCoreTest, StatelessScriptUpdateProcessorFactoryTest, TestQuerySenderListener, HighlighterConfigTest, TestIBSimilarityFactory, TestSolrXMLSerializer, TestMergePolicyConfig, IndexBasedSpellCheckerTest, ScriptEngineTest, DocValuesTest, HdfsBasicDistributedZk2Test, HighlighterTest, TestClassNameShortening, TestElisionMultitermQuery, SpellingQueryConverterTest, TestSolrJ, CacheHeaderTest, QueryResultKeyTest, TestTrie, AutoCommitTest, DocumentBuilderTest, FileBasedSpellCheckerTest, TestSuggestSpellingConverter, CircularListTest, UniqFieldsUpdateProcessorFactoryTest, TestFieldResource, CSVRequestHandlerTest, TestCSVResponseWriter, MoreLikeThisHandlerTest, DirectUpdateHandlerTest, HdfsBasicDistributedZkTest, BlockCacheTest, BasicDistributedZkTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest]
   [junit4] Completed in 93.46s, 6 tests, 1 failure, 4 skipped <<< FAILURES!

[...truncated 481 lines...]
BUILD FAILED
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:389: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:369: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:181: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:449: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1230: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:873: There were test failures: 320 suites, 1347 tests, 1 error, 1 failure, 39 ignored (5 assumptions)

Total time: 108 minutes 46 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Mime
View raw message