Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/3940/ 9 tests failed. FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsThreadLeakTest Error Message: access denied ("java.io.FilePermission" "<>" "execute") Stack Trace: java.security.AccessControlException: access denied ("java.io.FilePermission" "<>" "execute") at __randomizedtesting.SeedInfo.seed([EC24855D66859E5E]:0) at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472) at java.base/java.security.AccessController.checkPermission(AccessController.java:897) at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) at java.base/java.lang.SecurityManager.checkExec(SecurityManager.java:572) at java.base/java.lang.ProcessBuilder.start(ProcessBuilder.java:1096) at java.base/java.lang.ProcessBuilder.start(ProcessBuilder.java:1071) at org.apache.hadoop.util.Shell.runCommand(Shell.java:938) at org.apache.hadoop.util.Shell.run(Shell.java:901) at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74) at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.(NameNodeResourceChecker.java:70) at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166) at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.(NameNodeResourceChecker.java:135) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168) at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791) at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717) at org.apache.hadoop.hdfs.server.namenode.NameNode.(NameNode.java:940) at org.apache.hadoop.hdfs.server.namenode.NameNode.(NameNode.java:913) at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646) at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314) at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083) at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958) at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890) at org.apache.hadoop.hdfs.MiniDFSCluster.(MiniDFSCluster.java:518) at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477) at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134) at org.apache.solr.cloud.hdfs.HdfsThreadLeakTest.beforeClass(HdfsThreadLeakTest.java:42) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsThreadLeakTest Error Message: 15 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsThreadLeakTest: 1) Thread[id=14564, name=qtp1008181857-14564, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 2) Thread[id=14557, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/java.lang.Object.wait(Native Method) at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:553) at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) 3) Thread[id=14574, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.4/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:417) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 4) Thread[id=14558, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@44b7a3a7, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 5) Thread[id=14559, name=pool-15-thread-1, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 6) Thread[id=14560, name=qtp1008181857-14560, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$167/0x0000000100379c40.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 7) Thread[id=14566, name=qtp1008181857-14566, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 8) Thread[id=14567, name=qtp1008181857-14567, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 9) Thread[id=14568, name=Session-HouseKeeper-451e950b, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 10) Thread[id=14577, name=IPC Server idle connection scanner for port 34301, state=WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/java.lang.Object.wait(Native Method) at java.base@11.0.4/java.lang.Object.wait(Object.java:328) at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:527) at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) 11) Thread[id=14563, name=qtp1008181857-14563, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 12) Thread[id=14562, name=qtp1008181857-14562-acceptor-0@14e412b5-ServerConnector@6d9bfe2{HTTP/1.1,[http/1.1]}{localhost.localdomain:35369}, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at java.base@11.0.4/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:533) at java.base@11.0.4/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:285) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 13) Thread[id=14565, name=qtp1008181857-14565, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 14) Thread[id=14561, name=qtp1008181857-14561, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$167/0x0000000100379c40.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 15) Thread[id=14576, name=Socket Reader #1 for port 34301, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsThreadLeakTest: 1) Thread[id=14564, name=qtp1008181857-14564, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 2) Thread[id=14557, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/java.lang.Object.wait(Native Method) at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:553) at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) 3) Thread[id=14574, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.4/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:417) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 4) Thread[id=14558, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@44b7a3a7, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 5) Thread[id=14559, name=pool-15-thread-1, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 6) Thread[id=14560, name=qtp1008181857-14560, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$167/0x0000000100379c40.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 7) Thread[id=14566, name=qtp1008181857-14566, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 8) Thread[id=14567, name=qtp1008181857-14567, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 9) Thread[id=14568, name=Session-HouseKeeper-451e950b, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 10) Thread[id=14577, name=IPC Server idle connection scanner for port 34301, state=WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/java.lang.Object.wait(Native Method) at java.base@11.0.4/java.lang.Object.wait(Object.java:328) at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:527) at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) 11) Thread[id=14563, name=qtp1008181857-14563, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 12) Thread[id=14562, name=qtp1008181857-14562-acceptor-0@14e412b5-ServerConnector@6d9bfe2{HTTP/1.1,[http/1.1]}{localhost.localdomain:35369}, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at java.base@11.0.4/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:533) at java.base@11.0.4/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:285) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 13) Thread[id=14565, name=qtp1008181857-14565, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 14) Thread[id=14561, name=qtp1008181857-14561, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$167/0x0000000100379c40.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 15) Thread[id=14576, name=Socket Reader #1 for port 34301, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) at __randomizedtesting.SeedInfo.seed([EC24855D66859E5E]:0) FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsThreadLeakTest Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=14564, name=qtp1008181857-14564, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 2) Thread[id=14557, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/java.lang.Object.wait(Native Method) at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:553) at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) 3) Thread[id=14559, name=pool-15-thread-1, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 4) Thread[id=14560, name=qtp1008181857-14560, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$167/0x0000000100379c40.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 5) Thread[id=14566, name=qtp1008181857-14566, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 6) Thread[id=14567, name=qtp1008181857-14567, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 7) Thread[id=14568, name=Session-HouseKeeper-451e950b, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 8) Thread[id=14577, name=IPC Server idle connection scanner for port 34301, state=WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/java.lang.Object.wait(Native Method) at java.base@11.0.4/java.lang.Object.wait(Object.java:328) at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:527) at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) 9) Thread[id=14563, name=qtp1008181857-14563, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 10) Thread[id=14562, name=qtp1008181857-14562, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 11) Thread[id=14565, name=qtp1008181857-14565, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 12) Thread[id=14561, name=qtp1008181857-14561, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$167/0x0000000100379c40.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 13) Thread[id=14576, name=Socket Reader #1 for port 34301, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated: 1) Thread[id=14564, name=qtp1008181857-14564, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 2) Thread[id=14557, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/java.lang.Object.wait(Native Method) at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:553) at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) 3) Thread[id=14559, name=pool-15-thread-1, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 4) Thread[id=14560, name=qtp1008181857-14560, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$167/0x0000000100379c40.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 5) Thread[id=14566, name=qtp1008181857-14566, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 6) Thread[id=14567, name=qtp1008181857-14567, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 7) Thread[id=14568, name=Session-HouseKeeper-451e950b, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 8) Thread[id=14577, name=IPC Server idle connection scanner for port 34301, state=WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/java.lang.Object.wait(Native Method) at java.base@11.0.4/java.lang.Object.wait(Object.java:328) at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:527) at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) 9) Thread[id=14563, name=qtp1008181857-14563, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 10) Thread[id=14562, name=qtp1008181857-14562, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 11) Thread[id=14565, name=qtp1008181857-14565, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 12) Thread[id=14561, name=qtp1008181857-14561, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$167/0x0000000100379c40.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 13) Thread[id=14576, name=Socket Reader #1 for port 34301, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) at __randomizedtesting.SeedInfo.seed([EC24855D66859E5E]:0) FAILED: junit.framework.TestSuite.org.apache.solr.store.hdfs.HdfsDirectoryTest Error Message: access denied ("java.io.FilePermission" "<>" "execute") Stack Trace: java.security.AccessControlException: access denied ("java.io.FilePermission" "<>" "execute") at __randomizedtesting.SeedInfo.seed([EC24855D66859E5E]:0) at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472) at java.base/java.security.AccessController.checkPermission(AccessController.java:897) at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:322) at java.base/java.lang.SecurityManager.checkExec(SecurityManager.java:572) at java.base/java.lang.ProcessBuilder.start(ProcessBuilder.java:1096) at java.base/java.lang.ProcessBuilder.start(ProcessBuilder.java:1071) at org.apache.hadoop.util.Shell.runCommand(Shell.java:938) at org.apache.hadoop.util.Shell.run(Shell.java:901) at org.apache.hadoop.fs.DF.getFilesystem(DF.java:74) at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker$CheckedVolume.(NameNodeResourceChecker.java:70) at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.addDirToCheck(NameNodeResourceChecker.java:166) at org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker.(NameNodeResourceChecker.java:135) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startCommonServices(FSNamesystem.java:1168) at org.apache.hadoop.hdfs.server.namenode.NameNode.startCommonServices(NameNode.java:791) at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:717) at org.apache.hadoop.hdfs.server.namenode.NameNode.(NameNode.java:940) at org.apache.hadoop.hdfs.server.namenode.NameNode.(NameNode.java:913) at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1646) at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNode(MiniDFSCluster.java:1314) at org.apache.hadoop.hdfs.MiniDFSCluster.configureNameService(MiniDFSCluster.java:1083) at org.apache.hadoop.hdfs.MiniDFSCluster.createNameNodesAndSetConf(MiniDFSCluster.java:958) at org.apache.hadoop.hdfs.MiniDFSCluster.initMiniDFSCluster(MiniDFSCluster.java:890) at org.apache.hadoop.hdfs.MiniDFSCluster.(MiniDFSCluster.java:518) at org.apache.hadoop.hdfs.MiniDFSCluster$Builder.build(MiniDFSCluster.java:477) at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:134) at org.apache.solr.cloud.hdfs.HdfsTestUtil.setupClass(HdfsTestUtil.java:66) at org.apache.solr.store.hdfs.HdfsDirectoryTest.beforeClass(HdfsDirectoryTest.java:60) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750) at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:878) at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57) at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) at java.base/java.lang.Thread.run(Thread.java:834) FAILED: junit.framework.TestSuite.org.apache.solr.store.hdfs.HdfsDirectoryTest Error Message: 15 threads leaked from SUITE scope at org.apache.solr.store.hdfs.HdfsDirectoryTest: 1) Thread[id=3233, name=qtp324421839-3233-acceptor-0@38ff5f7e-ServerConnector@41e820e4{HTTP/1.1,[http/1.1]}{localhost.localdomain:35423}, state=RUNNABLE, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at java.base@11.0.4/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:533) at java.base@11.0.4/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:285) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 2) Thread[id=3247, name=Socket Reader #1 for port 42325, state=RUNNABLE, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) 3) Thread[id=3234, name=qtp324421839-3234, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 4) Thread[id=3228, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/java.lang.Object.wait(Native Method) at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:553) at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) 5) Thread[id=3236, name=qtp324421839-3236, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 6) Thread[id=3245, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.4/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:417) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 7) Thread[id=3235, name=qtp324421839-3235, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 8) Thread[id=3232, name=qtp324421839-3232, state=RUNNABLE, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$200/0x000000010049dc40.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 9) Thread[id=3239, name=Session-HouseKeeper-62ae0ad4, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 10) Thread[id=3231, name=qtp324421839-3231, state=RUNNABLE, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$200/0x000000010049dc40.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 11) Thread[id=3238, name=qtp324421839-3238, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 12) Thread[id=3230, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 13) Thread[id=3237, name=qtp324421839-3237, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 14) Thread[id=3229, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@1d155c52, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 15) Thread[id=3248, name=IPC Server idle connection scanner for port 42325, state=WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/java.lang.Object.wait(Native Method) at java.base@11.0.4/java.lang.Object.wait(Object.java:328) at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:527) at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) Stack Trace: com.carrotsearch.randomizedtesting.ThreadLeakError: 15 threads leaked from SUITE scope at org.apache.solr.store.hdfs.HdfsDirectoryTest: 1) Thread[id=3233, name=qtp324421839-3233-acceptor-0@38ff5f7e-ServerConnector@41e820e4{HTTP/1.1,[http/1.1]}{localhost.localdomain:35423}, state=RUNNABLE, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at java.base@11.0.4/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:533) at java.base@11.0.4/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:285) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:385) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:648) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 2) Thread[id=3247, name=Socket Reader #1 for port 42325, state=RUNNABLE, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) 3) Thread[id=3234, name=qtp324421839-3234, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 4) Thread[id=3228, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/java.lang.Object.wait(Native Method) at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:553) at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) 5) Thread[id=3236, name=qtp324421839-3236, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 6) Thread[id=3245, name=FSEditLogAsync, state=WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.4/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:417) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:221) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:229) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 7) Thread[id=3235, name=qtp324421839-3235, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 8) Thread[id=3232, name=qtp324421839-3232, state=RUNNABLE, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$200/0x000000010049dc40.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 9) Thread[id=3239, name=Session-HouseKeeper-62ae0ad4, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 10) Thread[id=3231, name=qtp324421839-3231, state=RUNNABLE, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$200/0x000000010049dc40.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 11) Thread[id=3238, name=qtp324421839-3238, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 12) Thread[id=3230, name=pool-3-thread-1, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 13) Thread[id=3237, name=qtp324421839-3237, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 14) Thread[id=3229, name=org.apache.hadoop.util.JvmPauseMonitor$Monitor@1d155c52, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:192) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 15) Thread[id=3248, name=IPC Server idle connection scanner for port 42325, state=WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/java.lang.Object.wait(Native Method) at java.base@11.0.4/java.lang.Object.wait(Object.java:328) at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:527) at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) at __randomizedtesting.SeedInfo.seed([EC24855D66859E5E]:0) FAILED: junit.framework.TestSuite.org.apache.solr.store.hdfs.HdfsDirectoryTest Error Message: There are still zombie threads that couldn't be terminated: 1) Thread[id=3233, name=qtp324421839-3233, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 2) Thread[id=3247, name=Socket Reader #1 for port 42325, state=RUNNABLE, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) 3) Thread[id=3234, name=qtp324421839-3234, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 4) Thread[id=3228, name=Timer for 'NameNode' metrics system, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/java.lang.Object.wait(Native Method) at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:553) at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) 5) Thread[id=3236, name=qtp324421839-3236, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 6) Thread[id=3235, name=qtp324421839-3235, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 7) Thread[id=3232, name=qtp324421839-3232, state=RUNNABLE, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$200/0x000000010049dc40.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 8) Thread[id=3239, name=Session-HouseKeeper-62ae0ad4, state=TIMED_WAITING, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) 9) Thread[id=3231, name=qtp324421839-3231, state=RUNNABLE, group=TGRP-HdfsDirectoryTest] at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce( [...truncated too long message...] at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) [junit4] > at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) [junit4] > at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) [junit4] > at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) [junit4] > at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) [junit4] > at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$167/0x0000000100379c40.run(Unknown Source) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) [junit4] > at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) [junit4] > 5) Thread[id=14566, name=qtp1008181857-14566, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] [junit4] > at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) [junit4] > at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) [junit4] > at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) [junit4] > at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) [junit4] > 6) Thread[id=14567, name=qtp1008181857-14567, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] [junit4] > at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) [junit4] > at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) [junit4] > at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) [junit4] > at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) [junit4] > 7) Thread[id=14568, name=Session-HouseKeeper-451e950b, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] [junit4] > at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) [junit4] > at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) [junit4] > at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) [junit4] > at java.base@11.0.4/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) [junit4] > at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) [junit4] > at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) [junit4] > at java.base@11.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [junit4] > at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) [junit4] > 8) Thread[id=14577, name=IPC Server idle connection scanner for port 34301, state=WAITING, group=TGRP-HdfsThreadLeakTest] [junit4] > at java.base@11.0.4/java.lang.Object.wait(Native Method) [junit4] > at java.base@11.0.4/java.lang.Object.wait(Object.java:328) [junit4] > at java.base@11.0.4/java.util.TimerThread.mainLoop(Timer.java:527) [junit4] > at java.base@11.0.4/java.util.TimerThread.run(Timer.java:506) [junit4] > 9) Thread[id=14563, name=qtp1008181857-14563, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] [junit4] > at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) [junit4] > at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) [junit4] > at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) [junit4] > at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) [junit4] > 10) Thread[id=14562, name=qtp1008181857-14562, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] [junit4] > at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) [junit4] > at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) [junit4] > at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) [junit4] > at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) [junit4] > 11) Thread[id=14565, name=qtp1008181857-14565, state=TIMED_WAITING, group=TGRP-HdfsThreadLeakTest] [junit4] > at java.base@11.0.4/jdk.internal.misc.Unsafe.park(Native Method) [junit4] > at java.base@11.0.4/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) [junit4] > at java.base@11.0.4/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) [junit4] > at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:850) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:889) [junit4] > at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) [junit4] > 12) Thread[id=14561, name=qtp1008181857-14561, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] [junit4] > at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) [junit4] > at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) [junit4] > at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) [junit4] > at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) [junit4] > at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:464) [junit4] > at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:401) [junit4] > at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) [junit4] > at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) [junit4] > at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) [junit4] > at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:132) [junit4] > at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$167/0x0000000100379c40.run(Unknown Source) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:781) [junit4] > at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:917) [junit4] > at java.base@11.0.4/java.lang.Thread.run(Thread.java:834) [junit4] > 13) Thread[id=14576, name=Socket Reader #1 for port 34301, state=RUNNABLE, group=TGRP-HdfsThreadLeakTest] [junit4] > at java.base@11.0.4/sun.nio.ch.EPoll.wait(Native Method) [junit4] > at java.base@11.0.4/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) [junit4] > at java.base@11.0.4/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124) [junit4] > at java.base@11.0.4/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) [junit4] > at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1097) [junit4] > at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1076) [junit4] > at __randomizedtesting.SeedInfo.seed([EC24855D66859E5E]:0) [junit4] Completed [881/890 (3!)] on J2 in 13.91s, 0 tests, 3 errors <<< FAILURES! [...truncated 52911 lines...]