hive-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "sreebalineni ." <sreebalin...@gmail.com>
Subject Re: LeaseExpiredException in Hive
Date Thu, 12 May 2016 07:41:53 GMT
Please do test with debug mode I suspect it could be access issue of that
path or no enough space
On May 12, 2016 12:54 PM, "Arun Vasu" <arunvpy@gmail.com> wrote:

> Hi All,
>
> I was running 10 hive queries (INSERT INTO TABLE..... ) parallel and it is
> failing with an exception saying few of the MR jobs cannot find the
> required files.
> I have got the reason for this issue:
> http://stackoverflow.com/questions/7559880/leaseexpiredexception-no-lease-error-on-hdfs
>
> I am calling the Hive queries using scalding Execution.sequence(...)  and
> attached the the exception trace is given below,
> It is really great if someone can shed some light on this issue and how to
> solve it?
>
>
> 12/05/2016 4:36:03 PM INFO: parquet.hadoop.InternalParquetRecordWriter:
> Flushing mem columnStore to file. allocated memory: 80,530,611
>
> 16/05/12 16:42:07 WARN hdfs.DFSClient: DataStreamer Exception
>
> org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException):
> No lease on
> /tmp/hive-dintg/hive_2016-05-12_16-25-33_755_4680854916521511758-7/-mr-10005/0/emptyFile
> (inode 456570980): File does not exist. Holder
> DFSClient_NONMAPREDUCE_-2077717826_17 does not have any open files.
>
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3416)
>
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3218)
>
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3100)
>
> at
> org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:636)
>
> at
> org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:188)
>
> at
> org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:476)
>
> at
> org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
>
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:587)
>
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1026)
>
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2013)
>
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2009)
>
> at java.security.AccessController.doPrivileged(Native Method)
>
> at javax.security.auth.Subject.doAs(Subject.java:415)
>
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1642)
>
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2007)
>
>
> at org.apache.hadoop.ipc.Client.call(Client.java:1411)
>
> at org.apache.hadoop.ipc.Client.call(Client.java:1364)
>
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)
>
> at com.sun.proxy.$Proxy16.addBlock(Unknown Source)
>
> at
> org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:391)
>
> at sun.reflect.GeneratedMethodAccessor12.invoke(Unknown Source)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>
> at java.lang.reflect.Method.invoke(Method.java:606)
>
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187)
>
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
>
> at com.sun.proxy.$Proxy17.addBlock(Unknown Source)
>
> at
> org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1473)
>
> at
> org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1290)
>
> at
> org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:536)
>
> 16/05/12 16:42:07 ERROR hdfs.DFSClient: Failed to close inode 456570980
>
> org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException):
> No lease on
> /tmp/hive-acoe_omnds_dintg/hive_2016-05-12_16-25-33_755_4680854916521511758-7/-mr-10005/0/emptyFile
> (inode 456570980): File does not exist. Holder
> DFSClient_NONMAPREDUCE_-2077717826_17 does not have any open files.
>
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3416)
>
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.analyzeFileState(FSNamesystem.java:3218)
>
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3100)
>
> at
> org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:636)
>
> at
> org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.addBlock(AuthorizationProviderProxyClientProtocol.java:188)
>
> at
> org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:476)
>
> at
> org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
>
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:587)
>
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1026)
>
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2013)
>
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2009)
>
> at java.security.AccessController.doPrivileged(Native Method)
>
> at javax.security.auth.Subject.doAs(Subject.java:415)
>
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1642)
>
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2007)
>
>
> at org.apache.hadoop.ipc.Client.call(Client.java:1411)
>
> at org.apache.hadoop.ipc.Client.call(Client.java:1364)
>
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)
>
> at com.sun.proxy.$Proxy16.addBlock(Unknown Source)
>
> at
> org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:391)
>
> at sun.reflect.GeneratedMethodAccessor12.invoke(Unknown Source)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>
> at java.lang.reflect.Method.invoke(Method.java:606)
>
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187)
>
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
>
> at com.sun.proxy.$Proxy17.addBlock(Unknown Source)
>
> at
> org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1473)
>
> at
> org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1290)
>
> at
> org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:536)
>
> 16/05/12 16:42:07 WARN hdfs.DFSClient: DataStreamer Exception
>
>
> --
> Thanks,
> Arun
>

Mime
View raw message