hadoop-common-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Peng, Wei" <Wei.P...@xerox.com>
Subject jobtracker cannot be started
Date Fri, 21 Oct 2011 07:47:39 GMT
Hi,

 

When I was running a job on hadoop with 75% mappers finished, the
jobtracker hung so that I cannot access
jobtrackerserver:7845/jobtracker.jsp and hadoop job -status hung as
well.

 

Then I stopped jobtracker and restarted it. However, the jobtracker
cannot be started. I received error message from jobtracker.log.out
saying 

 

Exception in thread "LeaseChecker" java.lang.OutOfMemoryError: Java heap
space

        at
java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:59)

        at
java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:42)

        at
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:318)

        at
org.apache.hadoop.ipc.Client$Connection.access$1700(Client.java:176)

        at org.apache.hadoop.ipc.Client.getConnection(Client.java:859)

        at org.apache.hadoop.ipc.Client.call(Client.java:719)

        at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)

        at $Proxy4.renewLease(Unknown Source)

        at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)

        at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:597)

        at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvo
cationHandler.java:82)

        at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocation
Handler.java:59)

        at $Proxy4.renewLease(Unknown Source)

        at
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.renew(DFSClient.java:1016)

        at
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1028)

        at java.lang.Thread.run(Thread.java:619)

Exception in thread "expireTrackers" java.lang.OutOfMemoryError: Java
heap space

        at java.util.Arrays.copyOf(Arrays.java:2882)

        at
java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.jav
a:100)

        at
java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)

        at java.lang.StringBuffer.append(StringBuffer.java:224)

        at org.apache.hadoop.mapred.JobHistory.log(JobHistory.java:354)

        at
org.apache.hadoop.mapred.JobHistory$MapAttempt.logStarted(JobHistory.jav
a:1354)

        at
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:233
2)

        at
org.apache.hadoop.mapred.JobInProgress.updateTaskStatus(JobInProgress.ja
va:849)

        at
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:246
3)

        at
org.apache.hadoop.mapred.JobTracker.lostTaskTracker(JobTracker.java:3474
)

        at
org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:3
48)

        at java.lang.Thread.run(Thread.java:619)

Exception in thread "IPC Server listener on 9001"
java.lang.OutOfMemoryError: Java heap space

java.lang.reflect.InvocationTargetException

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)

        at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:597)

        at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)

        at org.mortbay.log.Log.warn(Log.java:181)

        at
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)

        at
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
16)

        at
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)

        at
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)

        at
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)

        at
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
Collection.java:230)

        at
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)

        at org.mortbay.jetty.Server.handle(Server.java:324)

        at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)

        at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)

        at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)

        at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)

        at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)

        at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)

        at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)

Caused by: java.lang.OutOfMemoryError: Java heap space

java.lang.reflect.InvocationTargetException

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)

        at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:597)

        at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)

        at org.mortbay.log.Log.warn(Log.java:181)

        at org.mortbay.thread.Timeout.tick(Timeout.java:159)

        at
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
va:587)

        at
org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)

        at
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
or.java:124)

        at
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
707)

        at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)

Caused by: java.lang.OutOfMemoryError: Java heap space

java.lang.reflect.InvocationTargetException

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)

        at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:597)

        at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)

        at org.mortbay.log.Log.warn(Log.java:189)

        at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)

        at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)

        at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)

        at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)

        at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)

        at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)

        at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)

Caused by: java.lang.OutOfMemoryError: Java heap space

Java HotSpot(TM) 64-Bit Server VM warning: Exception
java.lang.OutOfMemoryError occurred dispatching signal SIGTERM to
handler- the VM may need to be forcibly terminated

Exception in thread "retireJobs" java.lang.OutOfMemoryError: Java heap
space

java.lang.reflect.InvocationTargetException

Exception in thread "1586500830@qtp0-37" java.lang.OutOfMemoryError:
Java heap space

java.lang.reflect.InvocationTargetException

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

Exception in thread "1513282332@qtp0-29" java.lang.OutOfMemoryError:
Java heap space

java.lang.reflect.InvocationTargetException

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)

        at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:597)

        at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)

        at org.mortbay.log.Log.warn(Log.java:181)

        at
org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager
.java:281)

       at
org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManag
er.java:44)

        at
org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.ja
va:213)

        at java.util.TimerThread.mainLoop(Timer.java:512)

        at java.util.TimerThread.run(Timer.java:462)

Caused by: java.lang.OutOfMemoryError: Java heap space

Exception in thread "IPC Server Responder" java.lang.OutOfMemoryError:
Java heap space

java.lang.reflect.InvocationTargetException

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)

        at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:597)

        at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)

        at org.mortbay.log.Log.warn(Log.java:181)

        at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
429)

        at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)

Caused by: java.lang.OutOfMemoryError: Java heap space

Exception in thread "expireLaunchingTasks" java.lang.OutOfMemoryError:
Java heap space

java.lang.reflect.InvocationTargetException

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)

        at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:597)

        at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)

        at org.mortbay.log.Log.warn(Log.java:181)

        at
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)

        at
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
16)

        at
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)

        at
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)

        at
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)

        at
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
Collection.java:230)

        at
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)

        at org.mortbay.jetty.Server.handle(Server.java:324)

        at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)

        at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)

        at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)

        at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)

        at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)

        at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)

        at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)

Caused by: java.lang.OutOfMemoryError: Java heap space

java.lang.reflect.InvocationTargetException

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)

        at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:597)

        at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)

        at org.mortbay.log.Log.warn(Log.java:189)

        at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)

        at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)

        at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)

        at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)

        at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)

        at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)

        at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)

Caused by: java.lang.OutOfMemoryError: Java heap space

java.lang.reflect.InvocationTargetException

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)

        at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:597)

        at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)

        at org.mortbay.log.Log.warn(Log.java:181)

        at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
429)

        at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)

Caused by: java.lang.OutOfMemoryError: Java heap space

java.lang.reflect.InvocationTargetException

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)

        at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:597)

        at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)

        at org.mortbay.log.Log.warn(Log.java:181)

        at
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)

        at
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
16)

        at
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)

        at
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)

        at
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)

        at
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
Collection.java:230)

        at
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)

        at org.mortbay.jetty.Server.handle(Server.java:324)

        at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)

        at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)

        at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)

        at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)

        at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)

        at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)

        at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)

Caused by: java.lang.OutOfMemoryError: Java heap space

java.lang.reflect.InvocationTargetException

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)

        at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:597)

        at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)

        at org.mortbay.log.Log.warn(Log.java:181)

        at
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)

        at
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
16)

        at
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)

        at
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)

        at
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)

        at
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
Collection.java:230)

        at
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)

        at org.mortbay.jetty.Server.handle(Server.java:324)

        at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)

        at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)

        at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)

        at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)

        at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)

        at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)

        at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)

Caused by: java.lang.OutOfMemoryError: Java heap space

java.lang.reflect.InvocationTargetException

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)

        at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)

        at java.lang.reflect.Method.invoke(Method.java:597)

        at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)

        at org.mortbay.log.Log.warn(Log.java:189)

        at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)

        at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)

        at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)

        at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)

        at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)

        at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)

        at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)

Caused by: java.lang.OutOfMemoryError: Java heap space

 

I have no idea how to solve this problem. The memory on the jobtracker
server seems ok

             total       used       free     shared    buffers
cached

Mem:       7148044    3578024    3570020          0     673768
467340

-/+ buffers/cache:    2436916    4711128

Swap:      1020116        160    1019956

 

Can someone help me on this?

 

Thanks a lot

Wei


Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message