accumulo-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Roshan Punnoose <rosh...@gmail.com>
Subject HDFS Closed Channel
Date Tue, 27 May 2014 22:19:55 GMT
Has anyone seen a case were a large parallel scan in mapreduce causes HDFS
to close on the tablets? My accumulo instance (version 1.5.1) seems to be
dying during an intensive mapreduce job. These are the only errors I see in
the tablet server logs:

2014-05-27 10:59:34,822 [log.DfsLogger] ERROR:
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:1320)
	at org.apache.hadoop.fs.FSOutputSummer.write(FSOutputSummer.java:98)
	at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:59)
	at java.io.DataOutputStream.writeLong(DataOutputStream.java:224)
	at org.apache.accumulo.server.logger.LogFileKey.write(LogFileKey.java:108)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger.write(DfsLogger.java:435)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger.logFileData(DfsLogger.java:448)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger.logManyTablets(DfsLogger.java:479)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger.log(DfsLogger.java:440)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger$5.write(TabletServerLogger.java:345)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:273)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:229)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.log(TabletServerLogger.java:342)
	at org.apache.accumulo.server.tabletserver.TabletServer$ThriftClientHandler.update(TabletServer.java:1680)
	at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.accumulo.trace.instrument.thrift.TraceWrap$1.invoke(TraceWrap.java:63)
	at com.sun.proxy.$Proxy11.update(Unknown Source)
	at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2125)
	at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2109)
	at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
	at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
	at org.apache.accumulo.server.util.TServerUtils$TimedProcessor.process(TServerUtils.java:155)
	at org.apache.thrift.server.AbstractNonblockingServer$FrameBuffer.invoke(AbstractNonblockingServer.java:478)
	at org.apache.accumulo.server.util.TServerUtils$THsHaServer$Invocation.run(TServerUtils.java:207)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
	at org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
	at java.lang.Thread.run(Thread.java:744)
2014-05-27 10:59:34,826 [log.DfsLogger] WARN : Exception syncing
java.lang.reflect.InvocationTargetException
2014-05-27 10:59:34,827 [log.TabletServerLogger] ERROR: Unexpected
error writing to log, retrying attempt 1
java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
	at org.apache.accumulo.server.tabletserver.log.DfsLogger$LoggerOperation.await(DfsLogger.java:178)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:279)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:229)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.log(TabletServerLogger.java:342)
	at org.apache.accumulo.server.tabletserver.TabletServer$ThriftClientHandler.update(TabletServer.java:1680)
	at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.accumulo.trace.instrument.thrift.TraceWrap$1.invoke(TraceWrap.java:63)
	at com.sun.proxy.$Proxy11.update(Unknown Source)
	at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2125)
	at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2109)
	at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
	at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
	at org.apache.accumulo.server.util.TServerUtils$TimedProcessor.process(TServerUtils.java:155)
	at org.apache.thrift.server.AbstractNonblockingServer$FrameBuffer.invoke(AbstractNonblockingServer.java:478)
	at org.apache.accumulo.server.util.TServerUtils$THsHaServer$Invocation.run(TServerUtils.java:207)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
	at org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
	at java.lang.Thread.run(Thread.java:744)
Caused by: java.lang.reflect.InvocationTargetException
	at sun.reflect.GeneratedMethodAccessor8.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger$LogSyncingTask.run(DfsLogger.java:118)
	... 1 more
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:629)
	at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:1635)
	at org.apache.hadoop.hdfs.DFSOutputStream.hsync(DFSOutputStream.java:1618)
	at org.apache.hadoop.hdfs.DFSOutputStream.hsync(DFSOutputStream.java:1601)
	at org.apache.hadoop.fs.FSDataOutputStream.hsync(FSDataOutputStream.java:137)
	... 5 more
2014-05-27 10:59:34,881 [log.DfsLogger] ERROR:
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:1320)
	at org.apache.hadoop.fs.FSOutputSummer.write(FSOutputSummer.java:98)
	at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:59)
	at java.io.DataOutputStream.writeLong(DataOutputStream.java:224)
	at org.apache.accumulo.server.logger.LogFileKey.write(LogFileKey.java:108)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger.write(DfsLogger.java:435)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger.logFileData(DfsLogger.java:448)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger.logManyTablets(DfsLogger.java:479)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger.log(DfsLogger.java:440)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger$5.write(TabletServerLogger.java:345)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:273)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:229)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.log(TabletServerLogger.java:342)
	at org.apache.accumulo.server.tabletserver.TabletServer$ThriftClientHandler.update(TabletServer.java:1680)
	at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.accumulo.trace.instrument.thrift.TraceWrap$1.invoke(TraceWrap.java:63)
	at com.sun.proxy.$Proxy11.update(Unknown Source)
	at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2125)
	at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2109)
	at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
	at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
	at org.apache.accumulo.server.util.TServerUtils$TimedProcessor.process(TServerUtils.java:155)
	at org.apache.thrift.server.AbstractNonblockingServer$FrameBuffer.invoke(AbstractNonblockingServer.java:478)
	at org.apache.accumulo.server.util.TServerUtils$THsHaServer$Invocation.run(TServerUtils.java:207)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
	at org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
	at java.lang.Thread.run(Thread.java:744)
2014-05-27 10:59:34,883 [log.DfsLogger] WARN : Exception syncing
java.lang.reflect.InvocationTargetException
2014-05-27 10:59:34,883 [log.TabletServerLogger] ERROR: Unexpected
error writing to log, retrying attempt 1
java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
	at org.apache.accumulo.server.tabletserver.log.DfsLogger$LoggerOperation.await(DfsLogger.java:178)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:279)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:229)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.log(TabletServerLogger.java:342)
	at org.apache.accumulo.server.tabletserver.TabletServer$ThriftClientHandler.update(TabletServer.java:1680)
	at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.accumulo.trace.instrument.thrift.TraceWrap$1.invoke(TraceWrap.java:63)
	at com.sun.proxy.$Proxy11.update(Unknown Source)
	at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2125)
	at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2109)
	at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
	at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
	at org.apache.accumulo.server.util.TServerUtils$TimedProcessor.process(TServerUtils.java:155)
	at org.apache.thrift.server.AbstractNonblockingServer$FrameBuffer.invoke(AbstractNonblockingServer.java:478)
	at org.apache.accumulo.server.util.TServerUtils$THsHaServer$Invocation.run(TServerUtils.java:207)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
	at org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
	at java.lang.Thread.run(Thread.java:744)
Caused by: java.lang.reflect.InvocationTargetException
	at sun.reflect.GeneratedMethodAccessor8.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger$LogSyncingTask.run(DfsLogger.java:118)
	... 1 more
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:629)
	at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:1635)
	at org.apache.hadoop.hdfs.DFSOutputStream.hsync(DFSOutputStream.java:1618)
	at org.apache.hadoop.hdfs.DFSOutputStream.hsync(DFSOutputStream.java:1601)
	at org.apache.hadoop.fs.FSDataOutputStream.hsync(FSDataOutputStream.java:137)
	... 5 more
2014-05-27 10:59:34,885 [log.DfsLogger] ERROR:
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:1320)
	at org.apache.hadoop.fs.FSOutputSummer.write(FSOutputSummer.java:98)
	at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:59)
	at java.io.DataOutputStream.writeLong(DataOutputStream.java:224)
	at org.apache.accumulo.server.logger.LogFileKey.write(LogFileKey.java:108)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger.write(DfsLogger.java:435)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger.logFileData(DfsLogger.java:448)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger.logManyTablets(DfsLogger.java:479)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger.log(DfsLogger.java:440)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger$5.write(TabletServerLogger.java:345)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:273)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:229)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.log(TabletServerLogger.java:342)
	at org.apache.accumulo.server.tabletserver.TabletServer$ThriftClientHandler.update(TabletServer.java:1680)
	at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.accumulo.trace.instrument.thrift.TraceWrap$1.invoke(TraceWrap.java:63)
	at com.sun.proxy.$Proxy11.update(Unknown Source)
	at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2125)
	at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2109)
	at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
	at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
	at org.apache.accumulo.server.util.TServerUtils$TimedProcessor.process(TServerUtils.java:155)
	at org.apache.thrift.server.AbstractNonblockingServer$FrameBuffer.invoke(AbstractNonblockingServer.java:478)
	at org.apache.accumulo.server.util.TServerUtils$THsHaServer$Invocation.run(TServerUtils.java:207)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
	at org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
	at java.lang.Thread.run(Thread.java:744)
2014-05-27 10:59:34,888 [log.DfsLogger] WARN : Exception syncing
java.lang.reflect.InvocationTargetException
2014-05-27 10:59:34,889 [log.TabletServerLogger] ERROR: Unexpected
error writing to log, retrying attempt 1
java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
	at org.apache.accumulo.server.tabletserver.log.DfsLogger$LoggerOperation.await(DfsLogger.java:178)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:279)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:229)
	at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.log(TabletServerLogger.java:342)
	at org.apache.accumulo.server.tabletserver.TabletServer$ThriftClientHandler.update(TabletServer.java:1680)
	at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.accumulo.trace.instrument.thrift.TraceWrap$1.invoke(TraceWrap.java:63)
	at com.sun.proxy.$Proxy11.update(Unknown Source)
	at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2125)
	at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2109)
	at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
	at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
	at org.apache.accumulo.server.util.TServerUtils$TimedProcessor.process(TServerUtils.java:155)
	at org.apache.thrift.server.AbstractNonblockingServer$FrameBuffer.invoke(AbstractNonblockingServer.java:478)
	at org.apache.accumulo.server.util.TServerUtils$THsHaServer$Invocation.run(TServerUtils.java:207)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
	at org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
	at java.lang.Thread.run(Thread.java:744)
Caused by: java.lang.reflect.InvocationTargetException
	at sun.reflect.GeneratedMethodAccessor8.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.accumulo.server.tabletserver.log.DfsLogger$LogSyncingTask.run(DfsLogger.java:118)
	... 1 more
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:629)
	at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:1635)
	at org.apache.hadoop.hdfs.DFSOutputStream.hsync(DFSOutputStream.java:1618)
	at org.apache.hadoop.hdfs.DFSOutputStream.hsync(DFSOutputStream.java:1601)
	at org.apache.hadoop.fs.FSDataOutputStream.hsync(FSDataOutputStream.java:137)
	... 5 more

Mime
View raw message