lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Policeman Jenkins Server <jenk...@thetaphi.de>
Subject [JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 919 - Still Unstable!
Date Sat, 22 Oct 2016 04:22:11 GMT
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/919/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.core.HdfsDirectoryFactoryTest

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([99E5F7D6E1DA3D1F]:0)


FAILED:  org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail

Error Message:
expected:<200> but was:<404>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
	at __randomizedtesting.SeedInfo.seed([99E5F7D6E1DA3D1F:F15AC2FC31402FF3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:294)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11308 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_99E5F7D6E1DA3D1F-001/init-core-data-001
   [junit4]   2> 1422609 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1422617 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_99E5F7D6E1DA3D1F-001/tempDir-001
   [junit4]   2> 1422617 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1422617 INFO  (Thread-3095) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1422618 INFO  (Thread-3095) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1422719 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:56341
   [junit4]   2> 1422729 INFO  (jetty-launcher-1995-thread-1) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1422729 INFO  (jetty-launcher-1995-thread-2) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1422732 INFO  (jetty-launcher-1995-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5d991ba{/solr,null,AVAILABLE}
   [junit4]   2> 1422733 INFO  (jetty-launcher-1995-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2a7a9b5d{/solr,null,AVAILABLE}
   [junit4]   2> 1422734 INFO  (jetty-launcher-1995-thread-1) [    ] o.e.j.s.ServerConnector Started ServerConnector@219700d8{HTTP/1.1,[http/1.1]}{127.0.0.1:54675}
   [junit4]   2> 1422734 INFO  (jetty-launcher-1995-thread-1) [    ] o.e.j.s.Server Started @1426674ms
   [junit4]   2> 1422734 INFO  (jetty-launcher-1995-thread-2) [    ] o.e.j.s.ServerConnector Started ServerConnector@453efee0{HTTP/1.1,[http/1.1]}{127.0.0.1:61509}
   [junit4]   2> 1422734 INFO  (jetty-launcher-1995-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54675}
   [junit4]   2> 1422734 INFO  (jetty-launcher-1995-thread-2) [    ] o.e.j.s.Server Started @1426675ms
   [junit4]   2> 1422734 INFO  (jetty-launcher-1995-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=61509}
   [junit4]   2> 1422734 ERROR (jetty-launcher-1995-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1422735 INFO  (jetty-launcher-1995-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 1422735 INFO  (jetty-launcher-1995-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1422736 INFO  (jetty-launcher-1995-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1422736 INFO  (jetty-launcher-1995-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-21T00:40:24.243Z
   [junit4]   2> 1422734 ERROR (jetty-launcher-1995-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1422737 INFO  (jetty-launcher-1995-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 1422737 INFO  (jetty-launcher-1995-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1422737 INFO  (jetty-launcher-1995-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1422737 INFO  (jetty-launcher-1995-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-21T00:40:24.244Z
   [junit4]   2> 1422740 INFO  (jetty-launcher-1995-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1422741 INFO  (jetty-launcher-1995-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1422749 INFO  (jetty-launcher-1995-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56341/solr
   [junit4]   2> 1422750 INFO  (jetty-launcher-1995-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56341/solr
   [junit4]   2> 1422799 INFO  (jetty-launcher-1995-thread-1) [n:127.0.0.1:54675_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:54675_solr
   [junit4]   2> 1422801 INFO  (jetty-launcher-1995-thread-2) [n:127.0.0.1:61509_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61509_solr
   [junit4]   2> 1422802 INFO  (jetty-launcher-1995-thread-1) [n:127.0.0.1:54675_solr    ] o.a.s.c.Overseer Overseer (id=96797355155587078-127.0.0.1:54675_solr-n_0000000000) starting
   [junit4]   2> 1422804 INFO  (zkCallback-2006-thread-1-processing-n:127.0.0.1:54675_solr) [n:127.0.0.1:54675_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1422805 INFO  (zkCallback-2005-thread-1-processing-n:127.0.0.1:61509_solr) [n:127.0.0.1:61509_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1422819 INFO  (jetty-launcher-1995-thread-1) [n:127.0.0.1:54675_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54675_solr
   [junit4]   2> 1422822 INFO  (zkCallback-2005-thread-1-processing-n:127.0.0.1:61509_solr) [n:127.0.0.1:61509_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1422825 INFO  (zkCallback-2006-thread-1-processing-n:127.0.0.1:54675_solr) [n:127.0.0.1:54675_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1422857 INFO  (jetty-launcher-1995-thread-1) [n:127.0.0.1:54675_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 1422873 INFO  (jetty-launcher-1995-thread-1) [n:127.0.0.1:54675_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 1422887 INFO  (jetty-launcher-1995-thread-2) [n:127.0.0.1:61509_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 1422888 INFO  (jetty-launcher-1995-thread-2) [n:127.0.0.1:61509_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 1422990 INFO  (jetty-launcher-1995-thread-1) [n:127.0.0.1:54675_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1422990 INFO  (jetty-launcher-1995-thread-2) [n:127.0.0.1:61509_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 1423020 INFO  (jetty-launcher-1995-thread-1-EventThread) [n:127.0.0.1:54675_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1423020 INFO  (jetty-launcher-1995-thread-2-EventThread) [n:127.0.0.1:61509_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 1423171 INFO  (jetty-launcher-1995-thread-1) [n:127.0.0.1:54675_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_99E5F7D6E1DA3D1F-001/tempDir-001/node1/.
   [junit4]   2> 1423171 INFO  (jetty-launcher-1995-thread-2) [n:127.0.0.1:61509_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_99E5F7D6E1DA3D1F-001/tempDir-001/node2/.
   [junit4]   2> 1423217 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
   [junit4]   2> 1423291 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1423313 INFO  (qtp756699722-12873) [n:127.0.0.1:54675_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroOsigFYCLsHrAEDFGHqtM7BZ7PeA1CT6Ztzq-P_QWkLB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1423313 INFO  (qtp756699722-12873) [n:127.0.0.1:54675_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroOsigFYCLsHrAEDFGHqtM7BZ7PeA1CT6Ztzq-P_QWkLB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1423321 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1423341 INFO  (qtp756699722-12873) [n:127.0.0.1:54675_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroOsigFYCLsHrAEDFGHqtM7BZ7PeA1CT6Ztzq-P_QWkLB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1423342 INFO  (qtp756699722-12873) [n:127.0.0.1:54675_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroOsigFYCLsHrAEDFGHqtM7BZ7PeA1CT6Ztzq-P_QWkLB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1423347 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1423354 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
   [junit4]   2> 1423383 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
   [junit4]   2> 1423410 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithDelegationTokens -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=99E5F7D6E1DA3D1F -Dtests.slow=true -Dtests.locale=en-ZA -Dtests.timezone=Etc/GMT+0 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.06s J0 | TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([99E5F7D6E1DA3D1F:F15AC2FC31402FF3]:0)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:140)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:294)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1423449 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
   [junit4]   2> 1423453 INFO  (qtp1550475368-12883) [n:127.0.0.1:61509_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1423453 INFO  (qtp1550475368-12883) [n:127.0.0.1:61509_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1423455 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
   [junit4]   2> 1423482 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
   [junit4]   2> 1423491 WARN  (qtp1550475368-12874) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 1423491 WARN  (qtp1550475368-12874) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
   [junit4]   2> 1423492 WARN  (qtp756699722-12881) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 1423495 WARN  (qtp756699722-12881) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
   [junit4]   2> 1423501 WARN  (qtp1550475368-12880) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 1423501 WARN  (qtp1550475368-12880) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
   [junit4]   2> 1423502 WARN  (qtp756699722-12873) [n:127.0.0.1:54675_solr    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 1423502 WARN  (qtp756699722-12873) [n:127.0.0.1:54675_solr    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
   [junit4]   2> 1423503 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
   [junit4]   2> 1423537 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
   [junit4]   2> 1423547 INFO  (qtp1550475368-12885) [n:127.0.0.1:61509_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroTUigFYCLsI1AUDFJtdZxr3W4eEJV_36UJMRtNf8x5yB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1423547 INFO  (qtp1550475368-12885) [n:127.0.0.1:61509_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroTUigFYCLsI1AUDFJtdZxr3W4eEJV_36UJMRtNf8x5yB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1423547 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
   [junit4]   2> 1423583 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
   [junit4]   2> 1423598 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1423602 ERROR (qtp756699722-12876) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_6]
   [junit4]   2> 1423612 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1423631 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1423647 ERROR (qtp756699722-12881) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_7]
   [junit4]   2> 1423656 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
   [junit4]   2> 1423685 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
   [junit4]   2> 1423693 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
   [junit4]   2> 1423728 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
   [junit4]   2> 1423741 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1423744 INFO  (qtp756699722-12877) [n:127.0.0.1:54675_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroWTigFYCLsJkwkDFO8akowYx8ZOdr0a8PkxRK3kS48LB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1423744 INFO  (qtp756699722-12877) [n:127.0.0.1:54675_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroWTigFYCLsJkwkDFO8akowYx8ZOdr0a8PkxRK3kS48LB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1423751 INFO  (qtp756699722-12873) [n:127.0.0.1:54675_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroWTigFYCLsJkwkDFO8akowYx8ZOdr0a8PkxRK3kS48LB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1423751 INFO  (qtp756699722-12873) [n:127.0.0.1:54675_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroWTigFYCLsJkwkDFO8akowYx8ZOdr0a8PkxRK3kS48LB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1423757 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1423778 INFO  (qtp1550475368-12880) [n:127.0.0.1:61509_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroW2igFYCLsJtgoDFLDwYlQKjlFEevzF30jk0aiVaRsJB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1423778 INFO  (qtp1550475368-12880) [n:127.0.0.1:61509_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroW2igFYCLsJtgoDFLDwYlQKjlFEevzF30jk0aiVaRsJB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1423787 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1423791 INFO  (qtp756699722-12873) [n:127.0.0.1:54675_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroW2igFYCLsJtgoDFLDwYlQKjlFEevzF30jk0aiVaRsJB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1423791 INFO  (qtp756699722-12873) [n:127.0.0.1:54675_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroW2igFYCLsJtgoDFLDwYlQKjlFEevzF30jk0aiVaRsJB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1423799 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1423818 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1423821 INFO  (qtp756699722-12881) [n:127.0.0.1:54675_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroXgigFYCLsJ4AsDFCFvGWH3wB_7Cqp-c8JyIPVFqKKeB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1423821 INFO  (qtp756699722-12881) [n:127.0.0.1:54675_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroXgigFYCLsJ4AsDFCFvGWH3wB_7Cqp-c8JyIPVFqKKeB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1423826 INFO  (qtp756699722-12876) [n:127.0.0.1:54675_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroXgigFYCLsJ4AsDFCFvGWH3wB_7Cqp-c8JyIPVFqKKeB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1423827 INFO  (qtp756699722-12876) [n:127.0.0.1:54675_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroXgigFYCLsJ4AsDFCFvGWH3wB_7Cqp-c8JyIPVFqKKeB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1423845 INFO  (qtp1550475368-12883) [n:127.0.0.1:61509_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroX5igFYCLsJ-QwDFPGTy4XGDURdQ8hiyVa48oaLpTb3B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1423846 INFO  (qtp1550475368-12883) [n:127.0.0.1:61509_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroX5igFYCLsJ-QwDFPGTy4XGDURdQ8hiyVa48oaLpTb3B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1423850 INFO  (qtp756699722-12876) [n:127.0.0.1:54675_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroX5igFYCLsJ-QwDFPGTy4XGDURdQ8hiyVa48oaLpTb3B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1423850 INFO  (qtp756699722-12876) [n:127.0.0.1:54675_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVfkroX5igFYCLsJ-QwDFPGTy4XGDURdQ8hiyVa48oaLpTb3B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1423856 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1423860 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[99E5F7D6E1DA3D1F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
   [junit4]   2> 1423862 INFO  (jetty-launcher-1995-thread-1) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@453efee0{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1423863 INFO  (jetty-launcher-1995-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1191154718
   [junit4]   2> 1423863 INFO  (jetty-launcher-1995-thread-2) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@219700d8{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1423864 INFO  (jetty-launcher-1995-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1712351029
   [junit4]   2> 1423872 INFO  (jetty-launcher-1995-thread-2) [    ] o.a.s.c.Overseer Overseer (id=96797355155587078-127.0.0.1:54675_solr-n_0000000000) closing
   [junit4]   2> 1423872 INFO  (OverseerStateUpdate-96797355155587078-127.0.0.1:54675_solr-n_0000000000) [n:127.0.0.1:54675_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:54675_solr
   [junit4]   2> 1423875 INFO  (zkCallback-2006-thread-2-processing-n:127.0.0.1:54675_solr) [n:127.0.0.1:54675_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1423876 ERROR (Thread[Thread-3113,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:61509_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1423886 INFO  (jetty-launcher-1995-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2a7a9b5d{/solr,null,UNAVAILABLE}
   [junit4]   2> 1425378 WARN  (zkCallback-2006-thread-2-processing-n:127.0.0.1:54675_solr) [n:127.0.0.1:54675_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1425379 ERROR (Thread[Thread-3112,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:54675_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 1425387 INFO  (jetty-launcher-1995-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5d991ba{/solr,null,UNAVAILABLE}
   [junit4]   2> 1425390 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:56341 56341
   [junit4]   2> 1425404 INFO  (Thread-3095) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:56341 56341
   [junit4]   2> 1425406 WARN  (Thread-3095) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	14	/solr/aliases.json
   [junit4]   2> 	14	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	14	/solr/clusterstate.json
   [junit4]   2> 	4	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	14	/solr/live_nodes
   [junit4]   2> 	14	/solr/collections
   [junit4]   2> 	4	/solr/overseer/queue
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 1425407 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_99E5F7D6E1DA3D1F-001
   [junit4]   2> Oct 21, 2016 12:40:26 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=972, maxMBSortInHeap=7.486117568307131, sim=RandomSimilarity(queryNorm=true): {}, locale=en-ZA, timezone=Etc/GMT+0
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=1,free=245510544,total=473956352
   [junit4]   2> NOTE: All tests run in this JVM: [ClassificationUpdateProcessorFactoryTest, TermsComponentTest, PrimUtilsTest, DocumentBuilderTest, SystemInfoHandlerTest, SolrPluginUtilsTest, TolerantUpdateProcessorTest, CdcrReplicationDistributedZkTest, TestReplicationHandler, CursorPagingTest, MultiTermTest, DistribDocExpirationUpdateProcessorTest, AnalysisAfterCoreReloadTest, ZkCLITest, TestSolrQueryParserResource, DistributedFacetPivotLongTailTest, OpenExchangeRatesOrgProviderTest, PingRequestHandlerTest, TestExtendedDismaxParser, TestReloadAndDeleteDocs, TestLegacyNumericRangeQueryBuilder, DistributedSpellCheckComponentTest, DistributedMLTComponentTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestJsonFacets, BlockDirectoryTest, DocValuesMissingTest, TestReloadDeadlock, CursorMarkTest, TestSolrConfigHandlerConcurrent, SuggesterFSTTest, TestRTimerTree, TestStandardQParsers, DOMUtilTest, TestSweetSpotSimilarityFactory, TestDocumentBuilder, TestLockTree, TestXmlQParserPlugin, HdfsCollectionsAPIDistributedZkTest, TestMacros, ChangedSchemaMergeTest, InfoHandlerTest, ShardSplitTest, TestCryptoKeys, ReplaceNodeTest, TestBulkSchemaAPI, TestRTGBase, TestCloudManagedSchema, SearchHandlerTest, TestHashQParserPlugin, PathHierarchyTokenizerFactoryTest, AlternateDirectoryTest, TestOnReconnectListenerSupport, DeleteLastCustomShardedReplicaTest, BinaryUpdateRequestHandlerTest, TestConfigSetProperties, TestBinaryField, TestFieldCacheReopen, TestAnalyzeInfixSuggestions, ResponseHeaderTest, UniqFieldsUpdateProcessorFactoryTest, TestJettySolrRunner, DirectSolrSpellCheckerTest, TestRandomRequestDistribution, MigrateRouteKeyTest, QueryElevationComponentTest, TestCustomSort, PluginInfoTest, SolrXmlInZkTest, TestSolrJ, DocExpirationUpdateProcessorFactoryTest, RankQueryTest, CustomCollectionTest, TestOrdValues, TestCloudSchemaless, TestElisionMultitermQuery, HighlighterMaxOffsetTest, TestTolerantSearch, ExitableDirectoryReaderTest, TestCollationField, XmlUpdateRequestHandlerTest, TestFunctionQuery, RecoveryAfterSoftCommitTest, DistributedDebugComponentTest, OverseerRolesTest, CloudMLTQParserTest, BlockJoinFacetRandomTest, ActionThrottleTest, TestExpandComponent, TestHighlightDedupGrouping, TestCloudDeleteByQuery, TestBM25SimilarityFactory, DeleteReplicaTest, TestScoreJoinQPScore, TestSQLHandlerNonCloud, TestMergePolicyConfig, SyncSliceTest, HdfsThreadLeakTest, SolrIndexConfigTest, HardAutoCommitTest, CdcrRequestHandlerTest, DocValuesMultiTest, TestSolrDeletionPolicy1, SolrTestCaseJ4Test, PolyFieldTest, ForceLeaderTest, TestSerializedLuceneMatchVersion, TestSolrXml, TestReplicaProperties, TimeZoneUtilsTest, AnalyticsMergeStrategyTest, TestNumericTerms64, TestJmxIntegration, CdcrUpdateLogTest, ZkStateReaderTest, HdfsDirectoryTest, DocValuesTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestFieldCacheVsDocValues, TestRandomDVFaceting, OverseerTest, HdfsRecoverLeaseTest, SimpleFacetsTest, TestSchemaNameResource, StatelessScriptUpdateProcessorFactoryTest, TestShortCircuitedRequests, TestFilteredDocIdSet, TestCollectionAPI, DeleteStatusTest, TestRuleBasedAuthorizationPlugin, RecoveryZkTest, TestRawTransformer, TestConfigSetsAPIExclusivity, TestPerFieldSimilarity, TestFieldCollectionResource, QueryEqualityTest, TestPostingsSolrHighlighter, WrapperMergePolicyFactoryTest, NumericFieldsTest, TestStressVersions, SchemaVersionSpecificBehaviorTest, TestSolrConfigHandler, ZkNodePropsTest, SoftAutoCommitTest, TestCharFilters, HdfsRestartWhileUpdatingTest, TestSolrCloudWithDelegationTokens]
   [junit4] Completed [225/640 (1!)] on J0 in 3.07s, 8 tests, 1 failure <<< FAILURES!

[...truncated 2661 lines...]
   [junit4] Suite: org.apache.solr.core.HdfsDirectoryFactoryTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_99E5F7D6E1DA3D1F-001/init-core-data-001
   [junit4]   2> 539236 INFO  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 540203 WARN  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 541154 WARN  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 541502 INFO  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
   [junit4]   2> 541553 WARN  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 541683 INFO  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 541821 INFO  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_solaris.vm_64789_hdfs____yzw58s/webapp
   [junit4]   2> 543071 INFO  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:64789
   [junit4]   2> 544714 WARN  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 544723 INFO  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 544746 INFO  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_55540_datanode____dhn9dk/webapp
   [junit4]   2> 545531 INFO  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:55540
   [junit4]   2> 546409 WARN  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 546420 INFO  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 546443 INFO  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_33845_datanode____.yvr0w0/webapp
   [junit4]   2> 547658 INFO  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33845
   [junit4]   2> 99358145 WARN  (DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_99E5F7D6E1DA3D1F-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_99E5F7D6E1DA3D1F-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to solaris-vm/127.0.0.1:34251) [    ] o.a.h.h.s.d.DataNode Problem connecting to server: solaris-vm/127.0.0.1:34251
   [junit4]   2> Oct 22, 2016 11:52:39 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.solr.core.HdfsDirectoryFactoryTest
   [junit4]   2> ==== jstack at approximately timeout time ====
   [junit4]   2> "RMI TCP Connection(6)-127.0.0.1" ID=6126 RUNNABLE (in native code)
   [junit4]   2> 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]   2> 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
   [junit4]   2> 	at java.net.SocketInputStream.read(SocketInputStream.java:170)
   [junit4]   2> 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
   [junit4]   2> 	at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
   [junit4]   2> 	at java.io.BufferedInputStream.read(BufferedInputStream.java:265)
   [junit4]   2> 	- locked java.io.BufferedInputStream@8011e20
   [junit4]   2> 	at java.io.FilterInputStream.read(FilterInputStream.java:83)
   [junit4]   2> 	at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:550)
   [junit4]   2> 	at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
   [junit4]   2> 	at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683)
   [junit4]   2> 	at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$193/1990978767.run(Unknown Source)
   [junit4]   2> 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2> 	at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	Locked synchronizers:
   [junit4]   2> 	- java.util.concurrent.ThreadPoolExecutor$Worker@75f9f8ae
   [junit4]   2> 
   [junit4]   2> "IPC Server idle connection scanner for port 44410" ID=6125 RUNNABLE
   [junit4]   2> 	at java.util.TimerThread.run(Timer.java:505)
   [junit4]   2> 
   [junit4]   2> "Socket Reader #1 for port 44410" ID=6124 RUNNABLE (in native code)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]   2> 	at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2> 	- locked sun.nio.ch.Util$3@107622e6
   [junit4]   2> 	- locked java.util.Collections$UnmodifiableSet@3943703f
   [junit4]   2> 	- locked sun.nio.ch.DevPollSelectorImpl@68de124a
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:629)
   [junit4]   2> 	- locked org.apache.hadoop.ipc.Server$Listener$Reader@682ee16f
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:608)
   [junit4]   2> 
   [junit4]   2> "org.apache.hadoop.util.JvmPauseMonitor$Monitor@2cdc0bca" ID=6122 TIMED_WAITING
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:182)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "nioEventLoopGroup-4-1" ID=6110 RUNNABLE (in native code)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]   2> 	at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2> 	- locked io.netty.channel.nio.SelectedSelectionKeySet@49ee8f9e
   [junit4]   2> 	- locked java.util.Collections$UnmodifiableSet@6b8ecc14
   [junit4]   2> 	- locked sun.nio.ch.DevPollSelectorImpl@73dadf6d
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2> 	at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:639)
   [junit4]   2> 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:325)
   [junit4]   2> 	at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112)
   [junit4]   2> 	at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "Timer-2" ID=6109 RUNNABLE
   [junit4]   2> 	at org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager.java:238)
   [junit4]   2> 	at org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManager.java:44)
   [junit4]   2> 	at org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.java:219)
   [junit4]   2> 	at java.util.TimerThread.mainLoop(Timer.java:555)
   [junit4]   2> 	at java.util.TimerThread.run(Timer.java:505)
   [junit4]   2> 
   [junit4]   2> "IPC Parameter Sending Thread #0" ID=6108 TIMED_WAITING on java.util.concurrent.SynchronousQueue$TransferStack@51d425f7
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.SynchronousQueue$TransferStack@51d425f7
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]   2> 	at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]   2> 	at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "IPC Client (273393224) connection to solaris-vm/127.0.0.1:34251 from jenkins" ID=6107 RUNNABLE
   [junit4]   2> 	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:341)
   [junit4]   2> 	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
   [junit4]   2> 	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
   [junit4]   2> 	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
   [junit4]   2> 	at java.io.FilterInputStream.read(FilterInputStream.java:133)
   [junit4]   2> 	at java.io.FilterInputStream.read(FilterInputStream.java:133)
   [junit4]   2> 	at org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:520)
   [junit4]   2> 	at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
   [junit4]   2> 	at java.io.BufferedInputStream.read(BufferedInputStream.java:265)
   [junit4]   2> 	- locked java.io.BufferedInputStream@6c607de0
   [junit4]   2> 	at java.io.DataInputStream.readInt(DataInputStream.java:387)
   [junit4]   2> 	at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1084)
   [junit4]   2> 	at org.apache.hadoop.ipc.Client$Connection.run(Client.java:979)
   [junit4]   2> 
   [junit4]   2> "1431340762@qtp-1342365098-1" ID=6106 TIMED_WAITING on org.mortbay.thread.QueuedThreadPool$PoolThread@555086da
   [junit4]   2> 	at java.lang.Object.wait(Native Method)
   [junit4]   2> 	- timed waiting on org.mortbay.thread.QueuedThreadPool$PoolThread@555086da
   [junit4]   2> 	at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
   [junit4]   2> 
   [junit4]   2> "289889046@qtp-1342365098-0 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33845" ID=6105 RUNNABLE (in native code)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]   2> 	at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2> 	- locked sun.nio.ch.Util$3@51cf9d8d
   [junit4]   2> 	- locked java.util.Collections$UnmodifiableSet@4cbc15cc
   [junit4]   2> 	- locked sun.nio.ch.DevPollSelectorImpl@69b89a81
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2> 	at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
   [junit4]   2> 	at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
   [junit4]   2> 	at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
   [junit4]   2> 	at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
   [junit4]   2> 	at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
   [junit4]   2> 
   [junit4]   2> "pool-9-thread-1" ID=6104 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@38e21bf7
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@38e21bf7
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]   2> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 9 on 44312" ID=6098 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 8 on 44312" ID=6097 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 7 on 44312" ID=6096 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 6 on 44312" ID=6095 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 5 on 44312" ID=6094 RUNNABLE
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 4 on 44312" ID=6093 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 3 on 44312" ID=6092 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 2 on 44312" ID=6091 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 1 on 44312" ID=6090 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 0 on 44312" ID=6089 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@322fe6a5
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server listener on 44312" ID=6084 RUNNABLE (in native code)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]   2> 	at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2> 	- locked sun.nio.ch.Util$3@658752b
   [junit4]   2> 	- locked java.util.Collections$UnmodifiableSet@2b56b398
   [junit4]   2> 	- locked sun.nio.ch.DevPollSelectorImpl@768c14b
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Listener.run(Server.java:682)
   [junit4]   2> 
   [junit4]   2> "IPC Server Responder" ID=6087 RUNNABLE
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:884)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Responder.run(Server.java:839)
   [junit4]   2> 
   [junit4]   2> "org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@6ca3f15d" ID=6065 RUNNABLE (in native code)
   [junit4]   2> 	at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
   [junit4]   2> 	at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
   [junit4]   2> 	at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
   [junit4]   2> 	- locked java.lang.Object@2a292fb5
   [junit4]   2> 	at sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:100)
   [junit4]   2> 	- locked java.lang.Object@6f9de6f6
   [junit4]   2> 	at org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:139)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:135)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_99E5F7D6E1DA3D1F-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.core.HdfsDirectoryFactoryTest_99E5F7D6E1DA3D1F-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to solaris-vm/127.0.0.1:34251" ID=6088 WAITING on org.apache.hadoop.ipc.Client$Call@3979e562
   [junit4]   2> 	at java.lang.Object.wait(Native Method)
   [junit4]   2> 	- waiting on org.apache.hadoop.ipc.Client$Call@3979e562
   [junit4]   2> 	at java.lang.Object.wait(Object.java:502)
   [junit4]   2> 	at org.apache.hadoop.ipc.Client.call(Client.java:1466)
   [junit4]   2> 	at org.apache.hadoop.ipc.Client.call(Client.java:1412)
   [junit4]   2> 	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229)
   [junit4]   2> 	at com.sun.proxy.$Proxy55.versionRequest(Unknown Source)
   [junit4]   2> 	at org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB.versionRequest(DatanodeProtocolClientSideTranslatorPB.java:274)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.retrieveNamespaceInfo(BPServiceActor.java:172)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.connectToNNAndHandshake(BPServiceActor.java:218)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:802)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "IPC Server idle connection scanner for port 44312" ID=6086 RUNNABLE
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$ConnectionManager.closeIdle(Server.java:2749)
   [junit4]   2> 	- locked org.apache.hadoop.ipc.Server$ConnectionManager@714942ee
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$ConnectionManager$1.run(Server.java:2794)
   [junit4]   2> 	at java.util.TimerThread.mainLoop(Timer.java:555)
   [junit4]   2> 	at java.util.TimerThread.run(Timer.java:505)
   [junit4]   2> 
   [junit4]   2> "Socket Reader #1 for port 44312" ID=6085 RUNNABLE (in native code)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]   2> 	at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2> 	- locked sun.nio.ch.Util$3@30eaa715
   [junit4]   2> 	- locked java.util.Collections$UnmodifiableSet@535d723a
   [junit4]   2> 	- locked sun.nio.ch.DevPollSelectorImpl@53f794f1
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:629)
   [junit4]   2> 	- locked org.apache.hadoop.ipc.Server$Listener$Reader@6812e253
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:608)
   [junit4]   2> 
   [junit4]   2> "org.apache.hadoop.util.JvmPauseMonitor$Monitor@1d063afe" ID=6083 TIMED_WAITING
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:182)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "nioEventLoopGroup-2-1" ID=6070 RUNNABLE (in native code)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]   2> 	at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2> 	- locked io.netty.channel.nio.SelectedSelectionKeySet@da5324b
   [junit4]   2> 	- locked java.util.Collections$UnmodifiableSet@35515d67
   [junit4]   2> 	- locked sun.nio.ch.DevPollSelectorImpl@5a6e1720
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2> 	at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:639)
   [junit4]   2> 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:325)
   [junit4]   2> 	at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:112)
   [junit4]   2> 	at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "Timer-1" ID=6069 RUNNABLE
   [junit4]   2> 	at org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager.java:238)
   [junit4]   2> 	at org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManager.java:44)
   [junit4]   2> 	at org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.java:219)
   [junit4]   2> 	at java.util.TimerThread.mainLoop(Timer.java:555)
   [junit4]   2> 	at java.util.TimerThread.run(Timer.java:505)
   [junit4]   2> 
   [junit4]   2> "578875829@qtp-459317686-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:55540" ID=6068 RUNNABLE (in native code)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]   2> 	at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2> 	- locked sun.nio.ch.Util$3@25916644
   [junit4]   2> 	- locked java.util.Collections$UnmodifiableSet@1822fd7
   [junit4]   2> 	- locked sun.nio.ch.DevPollSelectorImpl@3e87e0e5
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2> 	at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
   [junit4]   2> 	at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
   [junit4]   2> 	at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
   [junit4]   2> 	at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
   [junit4]   2> 	at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
   [junit4]   2> 
   [junit4]   2> "1142438795@qtp-459317686-0" ID=6067 TIMED_WAITING on org.mortbay.thread.QueuedThreadPool$PoolThread@44183b8b
   [junit4]   2> 	at java.lang.Object.wait(Native Method)
   [junit4]   2> 	- timed waiting on org.mortbay.thread.QueuedThreadPool$PoolThread@44183b8b
   [junit4]   2> 	at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
   [junit4]   2> 
   [junit4]   2> "pool-8-thread-1" ID=6066 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@a82ab87
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@a82ab87
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]   2> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "CacheReplicationMonitor(2116243098)" ID=6060 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@64be5224
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@64be5224
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:181)
   [junit4]   2> 
   [junit4]   2> "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@625b050f" ID=6059 TIMED_WAITING
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:4768)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@d530b2d" ID=6058 TIMED_WAITING
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:4695)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@30766cc2" ID=6057 TIMED_WAITING
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:4651)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@6cf0595b" ID=6056 TIMED_WAITING
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:431)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 9 on 34251" ID=6055 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 8 on 34251" ID=6054 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 7 on 34251" ID=6053 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 6 on 34251" ID=6052 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 5 on 34251" ID=6051 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 4 on 34251" ID=6050 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 3 on 34251" ID=6049 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 2 on 34251" ID=6048 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 1 on 34251" ID=6047 RUNNABLE
   [junit4]   2> 	at com.google.protobuf.DescriptorProtos$ServiceDescriptorProto.initFields(DescriptorProtos.java:11462)
   [junit4]   2> 	at com.google.protobuf.DescriptorProtos$ServiceDescriptorProto.<clinit>(DescriptorProtos.java:12228)
   [junit4]   2> 	at com.google.protobuf.DescriptorProtos$FileDescriptorProto.<init>(DescriptorProtos.java:1118)
   [junit4]   2> 	at com.google.protobuf.DescriptorProtos$FileDescriptorProto.<init>(DescriptorProtos.java:1031)
   [junit4]   2> 	at com.google.protobuf.DescriptorProtos$FileDescriptorProto$1.parsePartialFrom(DescriptorProtos.java:1248)
   [junit4]   2> 	at com.google.protobuf.DescriptorProtos$FileDescriptorProto$1.parsePartialFrom(DescriptorProtos.java:1243)
   [junit4]   2> 	at com.google.protobuf.AbstractParser.parsePartialFrom(AbstractParser.java:141)
   [junit4]   2> 	at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:176)
   [junit4]   2> 	at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:188)
   [junit4]   2> 	at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:193)
   [junit4]   2> 	at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:49)
   [junit4]   2> 	at com.google.protobuf.DescriptorProtos$FileDescriptorProto.parseFrom(DescriptorProtos.java:1897)
   [junit4]   2> 	at com.google.protobuf.Descriptors$FileDescriptor.internalBuildGeneratedFileFrom(Descriptors.java:291)
   [junit4]   2> 	at org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.<clinit>(DatanodeProtocolProtos.java:29936)
   [junit4]   2> 	at org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos$DatanodeProtocolService.getDescriptor(DatanodeProtocolProtos.java:28964)
   [junit4]   2> 	at org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos$DatanodeProtocolService$2.getDescriptorForType(DatanodeProtocolProtos.java:28737)
   [junit4]   2> 	at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:598)
   [junit4]   2> 	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:969)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2049)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2045)
   [junit4]   2> 	at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2> 	at javax.security.auth.Subject.doAs(Subject.java:422)
   [junit4]   2> 	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2045)
   [junit4]   2> 
   [junit4]   2> "IPC Server handler 0 on 34251" ID=6046 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7025527d
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
   [junit4]   2> 	at org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:109)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2017)
   [junit4]   2> 
   [junit4]   2> "IPC Server listener on 34251" ID=6035 RUNNABLE (in native code)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]   2> 	at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2> 	- locked sun.nio.ch.Util$3@147a2702
   [junit4]   2> 	- locked java.util.Collections$UnmodifiableSet@3af7c08f
   [junit4]   2> 	- locked sun.nio.ch.DevPollSelectorImpl@2a9bac91
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Listener.run(Server.java:682)
   [junit4]   2> 
   [junit4]   2> "IPC Server Responder" ID=6038 RUNNABLE
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:857)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Responder.run(Server.java:839)
   [junit4]   2> 
   [junit4]   2> "org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@54a01ea7" ID=6033 TIMED_WAITING
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor.run(BlockManager.java:3643)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@6fd48e05" ID=6034 TIMED_WAITING
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:384)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "DecommissionMonitor-0" ID=6045 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@654ac916
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@654ac916
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]   2> 	at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor@1bc6c9f1" ID=6044 TIMED_WAITING
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor.run(PendingReplicationBlocks.java:221)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "process reaper" ID=6040 TIMED_WAITING on java.util.concurrent.SynchronousQueue$TransferStack@67c37394
   [junit4]   2> 	at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	- timed waiting on java.util.concurrent.SynchronousQueue$TransferStack@67c37394
   [junit4]   2> 	at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2> 	at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]   2> 	at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]   2> 	at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "org.apache.hadoop.util.JvmPauseMonitor$Monitor@3ede0a19" ID=6039 TIMED_WAITING
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:182)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> "IPC Server idle connection scanner for port 34251" ID=6037 RUNNABLE
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$ConnectionManager.closeIdle(Server.java:2749)
   [junit4]   2> 	- locked org.apache.hadoop.ipc.Server$ConnectionManager@2b1af1ef
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$ConnectionManager$1.run(Server.java:2794)
   [junit4]   2> 	at java.util.TimerThread.mainLoop(Timer.java:555)
   [junit4]   2> 	at java.util.TimerThread.run(Timer.java:505)
   [junit4]   2> 
   [junit4]   2> "Socket Reader #1 for port 34251" ID=6036 RUNNABLE (in native code)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]   2> 	at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]   2> 	at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2> 	- locked sun.nio.ch.Util$3@6e9d91b9
   [junit4]   2> 	- locked java.util.Collections$UnmodifiableSet@6aead75d
   [junit4]   2> 	- locked sun.nio.ch.DevPollSelectorImpl@1d6211b5
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2> 	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]   2> 	at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:629)
   [junit4]   2> 	- locked org.apache.hadoop.ipc.Server$Listener$Reader@618b1e7f
   [juni

[...truncated too long message...]

solaris-vm/127.0.0.1:34251) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-486482448-127.0.0.1-1477009541898 (Datanode Uuid 2527bf4f-0bd3-4d75-b22a-a478f4674ca1) service to solaris-vm/127.0.0.1:34251
   [junit4]   2> 99364561 INFO  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:0
   [junit4]   2> 99364671 ERROR (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.h.m.l.MethodMetric Error invoking method getBlocksTotal
   [junit4]   2> java.lang.reflect.InvocationTargetException
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2> 	at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2> 	at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2> 	at org.apache.solr.core.HdfsDirectoryFactoryTest.teardownClass(HdfsDirectoryFactoryTest.java:61)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> 	... 54 more
   [junit4]   2> 99364711 INFO  (SUITE-HdfsDirectoryFactoryTest-seed#[99E5F7D6E1DA3D1F]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1979, maxMBSortInHeap=6.425574369765796, sim=ClassicSimilarity, locale=cs, timezone=Asia/Ujung_Pandang
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=2,free=370144520,total=450887680
   [junit4]   2> NOTE: All tests run in this JVM: [OverriddenZkACLAndCredentialsProvidersTest, SliceStateTest, TestSolrConfigHandlerCloud, TestJsonFacetRefinement, TestRecovery, TestExceedMaxTermLength, CurrencyFieldOpenExchangeTest, AsyncCallRequestStatusResponseTest, TestSolrQueryParser, SecurityConfHandlerTest, PKIAuthenticationIntegrationTest, DebugComponentTest, PreAnalyzedFieldTest, DirectUpdateHandlerOptimizeTest, ConvertedLegacyTest, TestSolrDynamicMBean, TestJmxMonitoredMap, ConnectionReuseTest, SpellCheckComponentTest, TestCloudPseudoReturnFields, TestUninvertingReader, TestRawResponseWriter, TestRealTimeGet, TestManagedSynonymFilterFactory, TestBlendedInfixSuggestions, TestFieldCache, TestPHPSerializedResponseWriter, TestJoin, TestMinMaxOnMultiValuedField, TestDFISimilarityFactory, ReplicationFactorTest, HttpPartitionTest, TestXmlQParser, TestUseDocValuesAsStored2, TestStressCloudBlindAtomicUpdates, VersionInfoTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, CollectionsAPISolrJTest, HdfsChaosMonkeySafeLeaderTest, NoCacheHeaderTest, UtilsToolTest, TestChildDocTransformer, EchoParamsTest, DirectoryFactoryTest, HdfsRecoveryZkTest, MultiThreadedOCPTest, OverseerStatusTest, TestFuzzyAnalyzedSuggestions, TestQueryWrapperFilter, TestStressRecovery, TestCloudInspectUtil, UUIDUpdateProcessorFallbackTest, HLLSerializationTest, TestRangeQuery, TestDistributedGrouping, MergeStrategyTest, TestDefaultStatsCache, StatsReloadRaceTest, TestSolrQueryParserDefaultOperatorResource, TestPerFieldSimilarityWithDefaultOverride, HdfsTlogReplayBufferedWhileIndexingTest, CdcrVersionReplicationTest, BigEndianAscendingWordSerializerTest, LeaderFailureAfterFreshStartTest, TermVectorComponentTest, TestReversedWildcardFilterFactory, LeaderInitiatedRecoveryOnShardRestartTest, BooleanFieldTest, EnumFieldTest, ConfigSetsAPITest, TestLRUStatsCache, TestSolrCLIRunExample, TestNoOpRegenerator, HdfsDirectoryFactoryTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HdfsDirectoryFactoryTest -Dtests.seed=99E5F7D6E1DA3D1F -Dtests.slow=true -Dtests.locale=cs -Dtests.timezone=Asia/Ujung_Pandang -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | HdfsDirectoryFactoryTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([99E5F7D6E1DA3D1F]:0)
   [junit4] Completed [555/640 (2!)] on J1 in 98828.10s, 0 tests, 1 error <<< FAILURES!

[...truncated 60621 lines...]


Mime
View raw message