hbase-builds mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Apache Jenkins Server <jenk...@builds.apache.org>
Subject Build failed in Jenkins: HBase-1.0 #678
Date Fri, 23 Jan 2015 18:53:07 GMT
See <https://builds.apache.org/job/HBase-1.0/678/changes>

Changes:

[ndimiduk] HBASE-12728 buffered writes substantially less useful after removal of HTablePool
(Solomon Duskis and Nick Dimiduk)

------------------------------------------
[...truncated 6220 lines...]
	at java.lang.Thread.sleep(Native Method)
	at org.apache.hadoop.hdfs.server.blockmanagement.PendingReplicationBlocks$PendingReplicationMonitor.run(PendingReplicationBlocks.java:221)
	at java.lang.Thread.run(Thread.java:724)

"org.apache.hadoop.util.JvmPauseMonitor$Monitor@13499ef" daemon prio=10 tid=0x602e3000 nid=0x4082
waiting on condition [0x62e5c000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
	at java.lang.Thread.sleep(Native Method)
	at org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:180)
	at java.lang.Thread.run(Thread.java:724)

"IPC Server idle connection scanner for port 49998" daemon prio=10 tid=0x602c1400 nid=0x407a
in Object.wait() [0x62ead000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at java.util.TimerThread.mainLoop(Timer.java:552)
	- locked <0xd7b025f0> (a java.util.TaskQueue)
	at java.util.TimerThread.run(Timer.java:505)

"Socket Reader #1 for port 49998" prio=10 tid=0x6028c000 nid=0x4079 runnable [0x62efe000]
   java.lang.Thread.State: RUNNABLE
	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
	- locked <0xd7b01e90> (a sun.nio.ch.Util$2)
	- locked <0xd7b01e80> (a java.util.Collections$UnmodifiableSet)
	- locked <0xd7b01c38> (a sun.nio.ch.EPollSelectorImpl)
	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
	at org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:616)
	- locked <0xd7b03858> (a org.apache.hadoop.ipc.Server$Listener$Reader)
	at org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:595)

"Timer-0" daemon prio=10 tid=0x62299800 nid=0x4032 in Object.wait() [0x63263000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at java.util.TimerThread.mainLoop(Timer.java:552)
	- locked <0x7cdd3800> (a java.util.TaskQueue)
	at java.util.TimerThread.run(Timer.java:505)

"17362056@qtp-3500109-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:54272"
daemon prio=10 tid=0x6227e400 nid=0x4023 runnable [0x6407d000]
   java.lang.Thread.State: RUNNABLE
	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228)
	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81)
	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
	- locked <0x7cdc8e08> (a sun.nio.ch.Util$2)
	- locked <0x7cdc8e18> (a java.util.Collections$UnmodifiableSet)
	- locked <0x7cdc8dc8> (a sun.nio.ch.EPollSelectorImpl)
	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
	at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
	at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
	at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
	at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
	at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)

"20597513@qtp-3500109-0" daemon prio=10 tid=0x6227dc00 nid=0x4022 in Object.wait() [0x6427d000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
	- locked <0x7cdce488> (a org.mortbay.thread.QueuedThreadPool$PoolThread)

"Timer for 'NameNode' metrics system" daemon prio=10 tid=0x6222a000 nid=0x401c in Object.wait()
[0x64e52000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at java.util.TimerThread.mainLoop(Timer.java:552)
	- locked <0x7cdb8bf0> (a java.util.TaskQueue)
	at java.util.TimerThread.run(Timer.java:505)

"Service Thread" daemon prio=10 tid=0x64cf7800 nid=0x3dd2 runnable [0x00000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" daemon prio=10 tid=0x64cf5c00 nid=0x3dd1 waiting on condition [0x00000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" daemon prio=10 tid=0x64cf3c00 nid=0x3dd0 waiting on condition [0x00000000]
   java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" daemon prio=10 tid=0x64cf2000 nid=0x3dcf runnable [0x00000000]
   java.lang.Thread.State: RUNNABLE

"Finalizer" daemon prio=10 tid=0x64cb7000 nid=0x3dcb in Object.wait() [0x6be7d000]
   java.lang.Thread.State: WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135)
	- locked <0x7cb61ed0> (a java.lang.ref.ReferenceQueue$Lock)
	at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151)
	at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189)

"Reference Handler" daemon prio=10 tid=0x64cb5400 nid=0x3dca in Object.wait() [0x6c07d000]
   java.lang.Thread.State: WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at java.lang.Object.wait(Object.java:503)
	at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133)
	- locked <0x7cb61f58> (a java.lang.ref.Reference$Lock)

"main" prio=10 tid=0xf6706800 nid=0x3db9 in Object.wait() [0xf68ab000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
	at java.lang.Object.wait(Native Method)
	at java.lang.Thread.join(Thread.java:1268)
	- locked <0xd44497a8> (a org.junit.internal.runners.statements.FailOnTimeout$StatementThread)
	at org.junit.internal.runners.statements.FailOnTimeout.evaluateStatement(FailOnTimeout.java:26)
	at org.junit.internal.runners.statements.FailOnTimeout.evaluate(FailOnTimeout.java:17)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
	at org.junit.runners.Suite.runChild(Suite.java:127)
	at org.junit.runners.Suite.runChild(Suite.java:26)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:160)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:138)
	at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.createRequestAndRun(JUnitCoreWrapper.java:107)
	at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.executeEager(JUnitCoreWrapper.java:77)
	at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:53)
	at org.apache.maven.surefire.junitcore.JUnitCoreProvider.invoke(JUnitCoreProvider.java:144)
	at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:203)
	at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:155)
	at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103)

"VM Thread" prio=10 tid=0x64bf8800 nid=0x3dc9 runnable 

"GC task thread#0 (ParallelGC)" prio=10 tid=0xf6710800 nid=0x3dba runnable 

"GC task thread#1 (ParallelGC)" prio=10 tid=0xf6712000 nid=0x3dbb runnable 

"GC task thread#2 (ParallelGC)" prio=10 tid=0xf6713800 nid=0x3dbc runnable 

"GC task thread#3 (ParallelGC)" prio=10 tid=0xf6715000 nid=0x3dbd runnable 

"GC task thread#4 (ParallelGC)" prio=10 tid=0xf6716400 nid=0x3dbe runnable 

"GC task thread#5 (ParallelGC)" prio=10 tid=0xf6717c00 nid=0x3dbf runnable 

"GC task thread#6 (ParallelGC)" prio=10 tid=0xf6719400 nid=0x3dc0 runnable 

"GC task thread#7 (ParallelGC)" prio=10 tid=0xf671a800 nid=0x3dc1 runnable 

"GC task thread#8 (ParallelGC)" prio=10 tid=0xf671c000 nid=0x3dc2 runnable 

"GC task thread#9 (ParallelGC)" prio=10 tid=0xf671d800 nid=0x3dc3 runnable 

"GC task thread#10 (ParallelGC)" prio=10 tid=0xf671ec00 nid=0x3dc4 runnable 

"GC task thread#11 (ParallelGC)" prio=10 tid=0xf6720400 nid=0x3dc5 runnable 

"GC task thread#12 (ParallelGC)" prio=10 tid=0xf6721c00 nid=0x3dc6 runnable 

"VM Periodic Task Thread" prio=10 tid=0x64cf9400 nid=0x3dd3 waiting on condition 

JNI global references: 280


Found one Java-level deadlock:
=============================
"AM.ZK.Worker-pool3-t19":
  waiting to lock monitor 0x6b96f138 (object 0xd7c446f0, a org.apache.hadoop.hbase.master.RegionStates),
  which is held by "AM.ZK.Worker-pool3-t4"
"AM.ZK.Worker-pool3-t4":
  waiting to lock monitor 0x6bd61c84 (object 0xd7ce28a8, a java.util.TreeMap),
  which is held by "B.defaultRpcServer.handler=1,queue=0,port=33890"
"B.defaultRpcServer.handler=1,queue=0,port=33890":
  waiting to lock monitor 0x6b96f138 (object 0xd7c446f0, a org.apache.hadoop.hbase.master.RegionStates),
  which is held by "AM.ZK.Worker-pool3-t4"

Java stack information for the threads listed above:
===================================================
"AM.ZK.Worker-pool3-t19":
	at org.apache.hadoop.hbase.master.RegionStates.getRegionState(RegionStates.java:972)
	- waiting to lock <0xd7c446f0> (a org.apache.hadoop.hbase.master.RegionStates)
	at org.apache.hadoop.hbase.master.AssignmentManager.handleRegion(AssignmentManager.java:928)
	at org.apache.hadoop.hbase.master.AssignmentManager$6.run(AssignmentManager.java:1490)
	at org.apache.hadoop.hbase.master.AssignmentManager$3.run(AssignmentManager.java:1274)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
	at java.util.concurrent.FutureTask.run(FutureTask.java:166)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:724)
"AM.ZK.Worker-pool3-t4":
	at org.apache.hadoop.hbase.master.AssignmentManager.clearRegionPlan(AssignmentManager.java:3178)
	- waiting to lock <0xd7ce28a8> (a java.util.TreeMap)
	at org.apache.hadoop.hbase.master.AssignmentManager.regionOnline(AssignmentManager.java:1445)
	at org.apache.hadoop.hbase.master.AssignmentManager.regionOnline(AssignmentManager.java:1437)
	at org.apache.hadoop.hbase.master.AssignmentManager$4.run(AssignmentManager.java:1331)
	- locked <0xd7c446f0> (a org.apache.hadoop.hbase.master.RegionStates)
	at org.apache.hadoop.hbase.master.AssignmentManager$3.run(AssignmentManager.java:1274)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
	at java.util.concurrent.FutureTask.run(FutureTask.java:166)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:724)
"B.defaultRpcServer.handler=1,queue=0,port=33890":
	at org.apache.hadoop.hbase.master.RegionStates.getRegionAssignments(RegionStates.java:176)
	- waiting to lock <0xd7c446f0> (a org.apache.hadoop.hbase.master.RegionStates)
	at org.apache.hadoop.hbase.master.AssignmentManager.getSnapShotOfAssignment(AssignmentManager.java:4048)
	at org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.getRegionAssignmentsByServer(BaseLoadBalancer.java:1372)
	at org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.createCluster(BaseLoadBalancer.java:1092)
	at org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.randomAssignment(BaseLoadBalancer.java:1161)
	at org.apache.hadoop.hbase.master.AssignmentManager.getRegionPlan(AssignmentManager.java:2357)
	- locked <0xd7ce28a8> (a java.util.TreeMap)
	at org.apache.hadoop.hbase.master.AssignmentManager.getRegionPlan(AssignmentManager.java:2314)
	at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:2031)
	at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1568)
	at org.apache.hadoop.hbase.master.MasterRpcServices.assignRegion(MasterRpcServices.java:361)
	at org.apache.hadoop.hbase.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java:42420)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2028)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
	at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
	at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
	at java.lang.Thread.run(Thread.java:724)

Found 1 deadlock.

+ echo '************ END  zombies jstack extract'
************ END  zombies jstack extract
+ JIRA_COMMENT='

     {color:red}-1 core zombie tests{color}.  There are 1 zombie test(s): 	at org.apache.hadoop.hbase.client.TestAdmin2.testCloseRegionWhenServerNameIsEmpty(TestAdmin2.java:422)'
+ BAD=1
+ jps
+ grep surefirebooter
+ xargs kill -9
+ cut -d ' ' -f 1
POST BUILD TASK : SUCCESS
END OF POST BUILD TASK : 0
Archiving artifacts
Sending artifact delta relative to HBase-1.0 #677
Archived 1847 artifacts
Archive block size is 32768
Received 27 blocks and 401809851 bytes
Compression is 0.2%
Took 1 min 48 sec
Recording test results
Updating HBASE-12728

Mime
View raw message