lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Apache Jenkins Server <jenk...@builds.apache.org>
Subject [JENKINS] Lucene-Solr-BadApples-NightlyTests-7.x - Build # 9 - Failure
Date Wed, 09 May 2018 23:44:39 GMT
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-7.x/9/

20 tests failed.
FAILED:  org.apache.lucene.codecs.simpletext.TestSimpleTextLiveDocsFormat.testOverflow

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([84B2D01F5935B264:BF1AF8F05A043352]:0)
	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:78)
	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:51)
	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:164)
	at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:150)
	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
	at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
	at org.apache.lucene.codecs.simpletext.SimpleTextUtil.write(SimpleTextUtil.java:48)
	at org.apache.lucene.codecs.simpletext.SimpleTextLiveDocsFormat.writeLiveDocs(SimpleTextLiveDocsFormat.java:113)
	at org.apache.lucene.index.BaseLiveDocsFormatTestCase.testSerialization(BaseLiveDocsFormatTestCase.java:129)
	at org.apache.lucene.index.BaseLiveDocsFormatTestCase.testOverflow(BaseLiveDocsFormatTestCase.java:78)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)


FAILED:  org.apache.lucene.codecs.lucene50.TestLucene50LiveDocsFormat.testOverflow

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([ECD9E68EE67C6B0B:D771CE61E54DEA3D]:0)
	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:78)
	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:51)
	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:164)
	at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:150)
	at org.apache.lucene.store.MockIndexOutputWrapper.writeBytes(MockIndexOutputWrapper.java:141)
	at org.apache.lucene.store.MockIndexOutputWrapper.writeByte(MockIndexOutputWrapper.java:126)
	at org.apache.lucene.store.DataOutput.writeInt(DataOutput.java:71)
	at org.apache.lucene.store.DataOutput.writeLong(DataOutput.java:213)
	at org.apache.lucene.codecs.lucene50.Lucene50LiveDocsFormat.writeLiveDocs(Lucene50LiveDocsFormat.java:110)
	at org.apache.lucene.index.BaseLiveDocsFormatTestCase.testSerialization(BaseLiveDocsFormatTestCase.java:129)
	at org.apache.lucene.index.BaseLiveDocsFormatTestCase.testOverflow(BaseLiveDocsFormatTestCase.java:78)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)


FAILED:  org.apache.solr.cloud.RollingRestartTest.test

Error Message:
Unable to restart (#4): CloudJettyRunner [url=https://127.0.0.1:34865/sz/collection1_shard1_replica_n51]

Stack Trace:
java.lang.AssertionError: Unable to restart (#4): CloudJettyRunner [url=https://127.0.0.1:34865/sz/collection1_shard1_replica_n51]
	at __randomizedtesting.SeedInfo.seed([2B4B8691C5CC7C26:A31FB94B6B3011DE]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.RollingRestartTest.restartWithRolesTest(RollingRestartTest.java:103)
	at org.apache.solr.cloud.RollingRestartTest.test(RollingRestartTest.java:52)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.Overseer.start(Overseer.java:545)  at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851)  at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)  at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)  at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)  at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2081)  at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [Overseer]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.cloud.Overseer.start(Overseer.java:545)
	at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:851)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
	at org.apache.solr.cloud.LeaderElector.retryElection(LeaderElector.java:393)
	at org.apache.solr.cloud.ZkController.rejoinOverseerElection(ZkController.java:2081)
	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:331)
	at java.lang.Thread.run(Thread.java:748)


	at __randomizedtesting.SeedInfo.seed([2B4B8691C5CC7C26]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:303)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ZkControllerTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.ZkControllerTest:     1) Thread[id=1708, name=OverseerAutoScalingTriggerThread-73771535134621698-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]         at org.apache.lucene.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:128)         at java.io.PrintStream.write(PrintStream.java:480)         at org.apache.logging.log4j.core.util.CloseShieldOutputStream.write(CloseShieldOutputStream.java:53)         at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:263)         at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:295)         at org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:304)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:179)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)         at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)         at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)         at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)         at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)         at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)         at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)         at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)         at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)         at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)         at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)         at org.apache.logging.log4j.core.config.DefaultReliabilityStrategy.log(DefaultReliabilityStrategy.java:49)         at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)         at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2170)         at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2125)         at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2108)         at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2002)         at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1974)         at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)         at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:146)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.ZkControllerTest: 
   1) Thread[id=1708, name=OverseerAutoScalingTriggerThread-73771535134621698-127.0.0.1:8983_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
        at org.apache.lucene.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:128)
        at java.io.PrintStream.write(PrintStream.java:480)
        at org.apache.logging.log4j.core.util.CloseShieldOutputStream.write(CloseShieldOutputStream.java:53)
        at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:263)
        at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:295)
        at org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:304)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:179)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
        at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
        at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
        at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
        at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
        at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:439)
        at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:434)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
        at org.apache.logging.log4j.core.config.DefaultReliabilityStrategy.log(DefaultReliabilityStrategy.java:49)
        at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
        at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2170)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2125)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2108)
        at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2002)
        at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1974)
        at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)
        at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:146)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([2B4B8691C5CC7C26]:0)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testNodeLost

Error Message:
last state: DocCollection(testNodeLost//clusterstate.json/327)={   "replicationFactor":"10",   "pullReplicas":"0",   "router":{"name":"compositeId"},   "maxShardsPerNode":"5",   "autoAddReplicas":"false",   "nrtReplicas":"10",   "tlogReplicas":"0",   "autoCreated":"true",   "shards":{     "shard20":{       "replicas":{         "core_node191":{           "core":"testNodeLost_shard20_replica_n191",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10093_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node234":{           "node_name":"127.0.0.1:10075_solr",           "core":"testNodeLost_shard20_replica_n233",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node200":{           "core":"testNodeLost_shard20_replica_n200",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10106_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node199":{           "core":"testNodeLost_shard20_replica_n199",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10044_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node198":{           "core":"testNodeLost_shard20_replica_n198",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10062_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node197":{           "core":"testNodeLost_shard20_replica_n197",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10087_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node196":{           "core":"testNodeLost_shard20_replica_n196",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10052_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node195":{           "core":"testNodeLost_shard20_replica_n195",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10054_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node193":{           "core":"testNodeLost_shard20_replica_n193",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10058_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node192":{           "core":"testNodeLost_shard20_replica_n192",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10024_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"73330000-7fffffff",       "state":"active"},     "shard10":{       "replicas":{         "core_node92":{           "core":"testNodeLost_shard10_replica_n92",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10024_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node91":{           "core":"testNodeLost_shard10_replica_n91",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10093_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node236":{           "node_name":"127.0.0.1:10050_solr",           "core":"testNodeLost_shard10_replica_n235",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node93":{           "core":"testNodeLost_shard10_replica_n93",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10058_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node96":{           "core":"testNodeLost_shard10_replica_n96",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10052_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node95":{           "core":"testNodeLost_shard10_replica_n95",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10054_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node98":{           "core":"testNodeLost_shard10_replica_n98",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10062_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node100":{           "core":"testNodeLost_shard10_replica_n100",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10106_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node97":{           "core":"testNodeLost_shard10_replica_n97",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10087_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node99":{           "core":"testNodeLost_shard10_replica_n99",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10044_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"f3330000-ffffffff",       "state":"active"},     "shard11":{       "replicas":{         "core_node108":{           "core":"testNodeLost_shard11_replica_n108",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10095_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node107":{           "core":"testNodeLost_shard11_replica_n107",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10042_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node106":{           "core":"testNodeLost_shard11_replica_n106",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10099_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node105":{           "core":"testNodeLost_shard11_replica_n105",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10083_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node268":{           "node_name":"127.0.0.1:10000_solr",           "core":"testNodeLost_shard11_replica_n267",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node102":{           "core":"testNodeLost_shard11_replica_n102",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10068_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node101":{           "core":"testNodeLost_shard11_replica_n101",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10114_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node110":{           "core":"testNodeLost_shard11_replica_n110",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10040_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node252":{           "node_name":"127.0.0.1:10115_solr",           "core":"testNodeLost_shard11_replica_n251",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node109":{           "core":"testNodeLost_shard11_replica_n109",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10057_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"0-ccbffff",       "state":"active"},     "shard12":{       "replicas":{         "core_node118":{           "core":"testNodeLost_shard12_replica_n118",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10048_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node228":{           "node_name":"127.0.0.1:10002_solr",           "core":"testNodeLost_shard12_replica_n227",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node116":{           "core":"testNodeLost_shard12_replica_n116",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10032_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node115":{           "core":"testNodeLost_shard12_replica_n115",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10063_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node113":{           "core":"testNodeLost_shard12_replica_n113",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10027_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node112":{           "core":"testNodeLost_shard12_replica_n112",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10100_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node111":{           "core":"testNodeLost_shard12_replica_n111",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10097_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node220":{           "node_name":"127.0.0.1:10094_solr",           "core":"testNodeLost_shard12_replica_n219",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node120":{           "core":"testNodeLost_shard12_replica_n120",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10075_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node260":{           "node_name":"127.0.0.1:10085_solr",           "core":"testNodeLost_shard12_replica_n259",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"}},       "range":"ccc0000-1998ffff",       "state":"active"},     "shard2":{       "replicas":{         "core_node18":{           "core":"testNodeLost_shard2_replica_n18",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10048_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node218":{           "node_name":"127.0.0.1:10106_solr",           "core":"testNodeLost_shard2_replica_n217",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node226":{           "node_name":"127.0.0.1:10097_solr",           "core":"testNodeLost_shard2_replica_n225",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node258":{           "node_name":"127.0.0.1:10046_solr",           "core":"testNodeLost_shard2_replica_n257",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node20":{           "core":"testNodeLost_shard2_replica_n20",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10075_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node12":{           "core":"testNodeLost_shard2_replica_n12",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10100_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node11":{           "core":"testNodeLost_shard2_replica_n11",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10097_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node13":{           "core":"testNodeLost_shard2_replica_n13",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10027_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node16":{           "core":"testNodeLost_shard2_replica_n16",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10032_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node15":{           "core":"testNodeLost_shard2_replica_n15",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10063_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"8ccc0000-9998ffff",       "state":"active"},     "shard13":{       "replicas":{         "core_node129":{           "core":"testNodeLost_shard13_replica_n129",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10034_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node128":{           "core":"testNodeLost_shard13_replica_n128",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10043_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node127":{           "core":"testNodeLost_shard13_replica_n127",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10029_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node126":{           "core":"testNodeLost_shard13_replica_n126",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10108_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node246":{           "node_name":"127.0.0.1:10062_solr",           "core":"testNodeLost_shard13_replica_n245",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node125":{           "core":"testNodeLost_shard13_replica_n125",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10059_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node124":{           "core":"testNodeLost_shard13_replica_n124",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10115_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node122":{           "core":"testNodeLost_shard13_replica_n122",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10124_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node254":{           "node_name":"127.0.0.1:10063_solr",           "core":"testNodeLost_shard13_replica_n253",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node121":{           "core":"testNodeLost_shard13_replica_n121",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10113_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"19990000-2665ffff",       "state":"active"},     "shard3":{       "replicas":{         "core_node29":{           "core":"testNodeLost_shard3_replica_n29",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10034_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node28":{           "core":"testNodeLost_shard3_replica_n28",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10043_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node248":{           "node_name":"127.0.0.1:10110_solr",           "core":"testNodeLost_shard3_replica_n247",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node256":{           "node_name":"127.0.0.1:10068_solr",           "core":"testNodeLost_shard3_replica_n255",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node21":{           "core":"testNodeLost_shard3_replica_n21",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10113_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node22":{           "core":"testNodeLost_shard3_replica_n22",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10124_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node25":{           "core":"testNodeLost_shard3_replica_n25",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10059_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node24":{           "core":"testNodeLost_shard3_replica_n24",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10115_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node27":{           "core":"testNodeLost_shard3_replica_n27",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10029_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node26":{           "core":"testNodeLost_shard3_replica_n26",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10108_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"99990000-a665ffff",       "state":"active"},     "shard14":{       "replicas":{         "core_node139":{           "core":"testNodeLost_shard14_replica_n139",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10101_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node138":{           "core":"testNodeLost_shard14_replica_n138",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10109_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node137":{           "core":"testNodeLost_shard14_replica_n137",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10036_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node136":{           "core":"testNodeLost_shard14_replica_n136",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10045_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node135":{           "core":"testNodeLost_shard14_replica_n135",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10074_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node222":{           "node_name":"127.0.0.1:10034_solr",           "core":"testNodeLost_shard14_replica_n221",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node133":{           "core":"testNodeLost_shard14_replica_n133",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10008_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node131":{           "core":"testNodeLost_shard14_replica_n131",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10056_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node262":{           "node_name":"127.0.0.1:10040_solr",           "core":"testNodeLost_shard14_replica_n261",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node140":{           "core":"testNodeLost_shard14_replica_n140",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10116_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"26660000-3332ffff",       "state":"active"},     "shard15":{       "replicas":{         "core_node206":{           "node_name":"127.0.0.1:10103_solr",           "core":"testNodeLost_shard15_replica_n205",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node149":{           "core":"testNodeLost_shard15_replica_n149",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10050_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node148":{           "core":"testNodeLost_shard15_replica_n148",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10092_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node147":{           "core":"testNodeLost_shard15_replica_n147",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10110_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node146":{           "core":"testNodeLost_shard15_replica_n146",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10037_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node145":{           "core":"testNodeLost_shard15_replica_n145",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10125_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node144":{           "core":"testNodeLost_shard15_replica_n144",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10055_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node143":{           "core":"testNodeLost_shard15_replica_n143",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10025_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node142":{           "core":"testNodeLost_shard15_replica_n142",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10011_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node141":{           "core":"testNodeLost_shard15_replica_n141",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10084_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"33330000-3fffffff",       "state":"active"},     "shard4":{       "replicas":{         "core_node39":{           "core":"testNodeLost_shard4_replica_n39",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10101_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node224":{           "node_name":"127.0.0.1:10031_solr",           "core":"testNodeLost_shard4_replica_n223",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node40":{           "core":"testNodeLost_shard4_replica_n40",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10116_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node264":{           "node_name":"127.0.0.1:10011_solr",           "core":"testNodeLost_shard4_replica_n263",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node31":{           "core":"testNodeLost_shard4_replica_n31",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10056_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node33":{           "core":"testNodeLost_shard4_replica_n33",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10008_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node36":{           "core":"testNodeLost_shard4_replica_n36",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10045_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node35":{           "core":"testNodeLost_shard4_replica_n35",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10074_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node38":{           "core":"testNodeLost_shard4_replica_n38",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10109_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node37":{           "core":"testNodeLost_shard4_replica_n37",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10036_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"a6660000-b332ffff",       "state":"active"},     "shard5":{       "replicas":{         "core_node41":{           "core":"testNodeLost_shard5_replica_n41",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10084_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node43":{           "core":"testNodeLost_shard5_replica_n43",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10025_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node42":{           "core":"testNodeLost_shard5_replica_n42",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10011_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node45":{           "core":"testNodeLost_shard5_replica_n45",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10125_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node44":{           "core":"testNodeLost_shard5_replica_n44",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10055_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node47":{           "core":"testNodeLost_shard5_replica_n47",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10110_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node46":{           "core":"testNodeLost_shard5_replica_n46",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10037_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node49":{           "core":"testNodeLost_shard5_replica_n49",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10050_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node48":{           "core":"testNodeLost_shard5_replica_n48",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10092_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node208":{           "node_name":"127.0.0.1:10059_solr",           "core":"testNodeLost_shard5_replica_n207",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"}},       "range":"b3330000-bfffffff",       "state":"active"},     "shard16":{       "replicas":{         "core_node159":{           "core":"testNodeLost_shard16_replica_n159",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10102_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node157":{           "core":"testNodeLost_shard16_replica_n157",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10094_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node156":{           "core":"testNodeLost_shard16_replica_n156",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10090_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node155":{           "core":"testNodeLost_shard16_replica_n155",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10039_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node154":{           "core":"testNodeLost_shard16_replica_n154",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10103_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node153":{           "core":"testNodeLost_shard16_replica_n153",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10010_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node230":{           "node_name":"127.0.0.1:10078_solr",           "core":"testNodeLost_shard16_replica_n229",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node152":{           "core":"testNodeLost_shard16_replica_n152",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10089_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node151":{           "core":"testNodeLost_shard16_replica_n151",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10122_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node160":{           "core":"testNodeLost_shard16_replica_n160",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10007_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"40000000-4ccbffff",       "state":"active"},     "shard17":{       "replicas":{         "core_node169":{           "core":"testNodeLost_shard17_replica_n169",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10016_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node168":{           "core":"testNodeLost_shard17_replica_n168",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10009_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node166":{           "core":"testNodeLost_shard17_replica_n166",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10076_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node210":{           "node_name":"127.0.0.1:10053_solr",           "core":"testNodeLost_shard17_replica_n209",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node165":{           "core":"testNodeLost_shard17_replica_n165",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10071_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node164":{           "core":"testNodeLost_shard17_replica_n164",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10053_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node163":{           "core":"testNodeLost_shard17_replica_n163",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10085_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node161":{           "core":"testNodeLost_shard17_replica_n161",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10002_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node270":{           "node_name":"127.0.0.1:10009_solr",           "core":"testNodeLost_shard17_replica_n269",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node170":{           "core":"testNodeLost_shard17_replica_n170",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10079_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"4ccc0000-5998ffff",       "state":"active"},     "shard18":{       "replicas":{         "core_node180":{           "core":"testNodeLost_shard18_replica_n180",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10112_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node202":{           "node_name":"127.0.0.1:10072_solr",           "core":"testNodeLost_shard18_replica_n201",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node179":{           "core":"testNodeLost_shard18_replica_n179",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10046_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node178":{           "core":"testNodeLost_shard18_replica_n178",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10000_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node177":{           "core":"testNodeLost_shard18_replica_n177",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10123_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node176":{           "core":"testNodeLost_shard18_replica_n176",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10077_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node175":{           "core":"testNodeLost_shard18_replica_n175",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10069_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node174":{           "core":"testNodeLost_shard18_replica_n174",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10031_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node172":{           "core":"testNodeLost_shard18_replica_n172",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10006_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node171":{           "core":"testNodeLost_shard18_replica_n171",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10086_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"59990000-6665ffff",       "state":"active"},     "shard19":{       "replicas":{         "core_node190":{           "core":"testNodeLost_shard19_replica_n190",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10030_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node238":{           "node_name":"127.0.0.1:10122_solr",           "core":"testNodeLost_shard19_replica_n237",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node214":{           "node_name":"127.0.0.1:10037_solr",           "core":"testNodeLost_shard19_replica_n213",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node189":{           "core":"testNodeLost_shard19_replica_n189",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10013_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node188":{           "core":"testNodeLost_shard19_replica_n188",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10072_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node242":{           "node_name":"127.0.0.1:10116_solr",           "core":"testNodeLost_shard19_replica_n241",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node185":{           "core":"testNodeLost_shard19_replica_n185",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10078_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node184":{           "core":"testNodeLost_shard19_replica_n184",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10028_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node182":{           "core":"testNodeLost_shard19_replica_n182",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10111_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node181":{           "core":"testNodeLost_shard19_replica_n181",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10001_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"66660000-7332ffff",       "state":"active"},     "shard1":{       "replicas":{         "core_node1":{           "core":"testNodeLost_shard1_replica_n1",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10114_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node266":{           "node_name":"127.0.0.1:10030_solr",           "core":"testNodeLost_shard1_replica_n265",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node6":{           "core":"testNodeLost_shard1_replica_n6",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10099_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node10":{           "core":"testNodeLost_shard1_replica_n10",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10040_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node7":{           "core":"testNodeLost_shard1_replica_n7",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10042_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node8":{           "core":"testNodeLost_shard1_replica_n8",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10095_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node9":{           "core":"testNodeLost_shard1_replica_n9",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10057_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node2":{           "core":"testNodeLost_shard1_replica_n2",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10068_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node250":{           "node_name":"127.0.0.1:10069_solr",           "core":"testNodeLost_shard1_replica_n249",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node5":{           "core":"testNodeLost_shard1_replica_n5",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10083_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"80000000-8ccbffff",       "state":"active"},     "shard6":{       "replicas":{         "core_node60":{           "core":"testNodeLost_shard6_replica_n60",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10007_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node52":{           "core":"testNodeLost_shard6_replica_n52",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10089_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node51":{           "core":"testNodeLost_shard6_replica_n51",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10122_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node232":{           "node_name":"127.0.0.1:10125_solr",           "core":"testNodeLost_shard6_replica_n231",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node54":{           "core":"testNodeLost_shard6_replica_n54",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10103_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node53":{           "core":"testNodeLost_shard6_replica_n53",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10010_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node56":{           "core":"testNodeLost_shard6_replica_n56",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10090_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node55":{           "core":"testNodeLost_shard6_replica_n55",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10039_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node57":{           "core":"testNodeLost_shard6_replica_n57",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10094_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node59":{           "core":"testNodeLost_shard6_replica_n59",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10102_solr",           "state":"down",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"c0000000-cccbffff",       "state":"active"},     "shard7":{       "replicas":{         "core_node70":{           "core":"testNodeLost_shard7_replica_n70",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10079_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node61":{           "core":"testNodeLost_shard7_replica_n61",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10002_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node63":{           "core":"testNodeLost_shard7_replica_n63",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10085_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node212":{           "node_name":"127.0.0.1:10109_solr",           "core":"testNodeLost_shard7_replica_n211",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node65":{           "core":"testNodeLost_shard7_replica_n65",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10071_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node64":{           "core":"testNodeLost_shard7_replica_n64",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10053_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node66":{           "core":"testNodeLost_shard7_replica_n66",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10076_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node69":{           "core":"testNodeLost_shard7_replica_n69",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10016_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node272":{           "node_name":"127.0.0.1:10087_solr",           "core":"testNodeLost_shard7_replica_n271",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node68":{           "core":"testNodeLost_shard7_replica_n68",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10009_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"cccc0000-d998ffff",       "state":"active"},     "shard8":{       "replicas":{         "core_node80":{           "core":"testNodeLost_shard8_replica_n80",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10112_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node204":{           "node_name":"127.0.0.1:10056_solr",           "core":"testNodeLost_shard8_replica_n203",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node72":{           "core":"testNodeLost_shard8_replica_n72",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10006_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node71":{           "core":"testNodeLost_shard8_replica_n71",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10086_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node74":{           "core":"testNodeLost_shard8_replica_n74",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10031_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node76":{           "core":"testNodeLost_shard8_replica_n76",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10077_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node75":{           "core":"testNodeLost_shard8_replica_n75",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10069_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node78":{           "core":"testNodeLost_shard8_replica_n78",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10000_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node77":{           "core":"testNodeLost_shard8_replica_n77",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10123_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node79":{           "core":"testNodeLost_shard8_replica_n79",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10046_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"d9990000-e665ffff",       "state":"active"},     "shard9":{       "replicas":{         "core_node90":{           "core":"testNodeLost_shard9_replica_n90",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10030_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node81":{           "core":"testNodeLost_shard9_replica_n81",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10001_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node216":{           "node_name":"127.0.0.1:10100_solr",           "core":"testNodeLost_shard9_replica_n215",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node82":{           "core":"testNodeLost_shard9_replica_n82",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10111_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node85":{           "core":"testNodeLost_shard9_replica_n85",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10078_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node84":{           "core":"testNodeLost_shard9_replica_n84",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10028_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node244":{           "node_name":"127.0.0.1:10008_solr",           "core":"testNodeLost_shard9_replica_n243",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node89":{           "core":"testNodeLost_shard9_replica_n89",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10013_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0},         "core_node240":{           "node_name":"127.0.0.1:10024_solr",           "core":"testNodeLost_shard9_replica_n239",           "state":"active",           "INDEX.sizeInBytes":1,           "type":"NRT"},         "core_node88":{           "core":"testNodeLost_shard9_replica_n88",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":1,           "node_name":"127.0.0.1:10072_solr",           "state":"active",           "type":"NRT",           "SEARCHER.searcher.numDocs":0}},       "range":"e6660000-f332ffff",       "state":"active"}}}

Stack Trace:
java.util.concurrent.TimeoutException: last state: DocCollection(testNodeLost//clusterstate.json/327)={
  "replicationFactor":"10",
  "pullReplicas":"0",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"5",
  "autoAddReplicas":"false",
  "nrtReplicas":"10",
  "tlogReplicas":"0",
  "autoCreated":"true",
  "shards":{
    "shard20":{
      "replicas":{
        "core_node191":{
          "core":"testNodeLost_shard20_replica_n191",
          "leader":"true",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":1,
          "node_name":"127.0.0.1:10093_solr",
          "state":"active",
          "type":"NRT",
          "SEARCHER.searcher.numDocs":0},
        "core_node234":{
          "node_name":"127.0.0.1:10075_solr",
          "core":"testNodeLost_shard20_replica_n233",
          "state":"active",
          "INDEX.sizeInBytes":1,
          "type":"NRT"},
        "core_node200":{
          "core":"testNodeLost_shard20_replica_n200",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":1,
          "node_name":"127.0.0.1:10106_solr",
          "state":"active",
          "type":"NRT",
          "SEARCHER.searcher.numDocs":0},
        "core_node199":{
          "core":"testNodeLost_shard20_replica_n199",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":1,
          "node_name":"127.0.0.1:10044_solr",
          "state":"active",
          "type":"NRT",
          "SEARCHER.searcher.numDocs":0},
        "core_node198":{
          "core":"testNodeLost_shard20_replica_n198",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":1,
          "node_name":"127.0.0.1:10062_solr",
          "state":"active",
          "type":"NRT",
          "SEARCHER.searcher.numDocs":0},
        "core_node197":{
          "core":"testNodeLost_shard20_replica_n197",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":1,
          "node_name":"127.0.0.1:10087_solr",
          "state":"active",
          "type":"NRT",
          

[...truncated too long message...]


   [junit4]    >    6) Thread[id=95737, name=searcherExecutor-14160-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=95771, name=searcherExecutor-14171-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2B4B8691C5CC7C26]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=95584, name=searcherExecutor-14121-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=95703, name=searcherExecutor-14149-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=95548, name=searcherExecutor-14110-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=95506, name=searcherExecutor-14099-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=95626, name=searcherExecutor-14132-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=95737, name=searcherExecutor-14160-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=95771, name=searcherExecutor-14171-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2B4B8691C5CC7C26]:0)
   [junit4] Completed [618/805 (7!)] on J2 in 461.37s, 1 test, 1 failure, 2 errors <<< FAILURES!

[...truncated 557 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-core/test/temp/junit4-J0-20180509_211453_82688593682244439369.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/heapdumps/java_pid14928.hprof ...
   [junit4] Heap dump file created [467286832 bytes in 3.935 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 9437 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/build.xml:661: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid14928.hprof
* java_pid3672.hprof
* java_pid9480.hprof

Total time: 385 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

Mime
View raw message