hive-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From 王锋 <wfeng1...@163.com>
Subject Re:parallel execution for Hive unit tests
Date Fri, 16 Dec 2011 01:47:37 GMT
hey all,


pls help me. during these days ,I found a serious issue with hive .It's an exception thrown
from hiveserver:




java.lang.StackOverflowError
    at java.io.UnixFileSystem.getBooleanAttributes0(Native Method)
    at java.io.UnixFileSystem.getBooleanAttributes(UnixFileSystem.java:242)
    at java.io.File.exists(File.java:772)
    at sun.misc.URLClassPath$FileLoader.getResource(URLClassPath.java:1057)
    at sun.misc.URLClassPath.getResource(URLClassPath.java:195)
    at java.net.URLClassLoader$1.run(URLClassLoader.java:358)
    at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:423)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
   ........................................................................................
at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:356)
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:264)
    at com.sun.beans.finder.ClassFinder.findClass(ClassFinder.java:64)
    at com.sun.beans.finder.ClassFinder.findClass(ClassFinder.java:106)
    at com.sun.beans.finder.InstanceFinder.instantiate(InstanceFinder.java:93)
    at com.sun.beans.finder.InstanceFinder.find(InstanceFinder.java:66)
    at com.sun.beans.finder.PersistenceDelegateFinder.find(PersistenceDelegateFinder.java:66)
    at java.beans.Encoder.getPersistenceDelegate(Encoder.java:197)
    at java.beans.PersistenceDelegate.initialize(PersistenceDelegate.java:211)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:398)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.java_util_Map_PersistenceDelegate.initialize(MetaData.java:679)
    at java.beans.PersistenceDelegate.initialize(PersistenceDelegate.java:212)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:398)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.DefaultPersistenceDelegate.doProperty(DefaultPersistenceDelegate.java:194)
    at java.beans.DefaultPersistenceDelegate.initBean(DefaultPersistenceDelegate.java:253)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:400)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeObject1(Encoder.java:258)
    at java.beans.Encoder.cloneStatement(Encoder.java:271)
    at java.beans.Encoder.writeStatement(Encoder.java:301)
    at java.beans.XMLEncoder.writeStatement(XMLEncoder.java:400)
    at java.beans.DefaultPersistenceDelegate.invokeStatement(DefaultPersistenceDelegate.java:217)
    at java.beans.java_util_List_PersistenceDelegate.initialize(MetaData.java:649)
    at java.beans.PersistenceDelegate.initialize(PersistenceDelegate.java:212)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:398)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.DefaultPersistenceDelegate.doProperty(DefaultPersistenceDelegate.java:194)
    at java.beans.DefaultPersistenceDelegate.initBean(DefaultPersistenceDelegate.java:253)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:400)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeObject1(Encoder.java:258)
    at java.beans.Encoder.cloneStatement(Encoder.java:271)
    at java.beans.Encoder.writeStatement(Encoder.java:301)
    at java.beans.XMLEncoder.writeStatement(XMLEncoder.java:400)
    at java.beans.DefaultPersistenceDelegate.invokeStatement(DefaultPersistenceDelegate.java:217)
    at java.beans.java_util_List_PersistenceDelegate.initialize(MetaData.java:649)
    at java.beans.PersistenceDelegate.initialize(PersistenceDelegate.java:212)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:398)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.DefaultPersistenceDelegate.doProperty(DefaultPersistenceDelegate.java:194)
    at java.beans.DefaultPersistenceDelegate.initBean(DefaultPersistenceDelegate.java:253)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:400)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeObject1(Encoder.java:258)
    at java.beans.Encoder.cloneStatement(Encoder.java:271)
    at java.beans.Encoder.writeStatement(Encoder.java:301)
    at java.beans.XMLEncoder.writeStatement(XMLEncoder.java:400)
    at java.beans.DefaultPersistenceDelegate.invokeStatement(DefaultPersistenceDelegate.java:217)
    at java.beans.java_util_List_PersistenceDelegate.initialize(MetaData.java:649)
    at java.beans.PersistenceDelegate.initialize(PersistenceDelegate.java:212)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:398)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.DefaultPersistenceDelegate.doProperty(DefaultPersistenceDelegate.java:194)
    at java.beans.DefaultPersistenceDelegate.initBean(DefaultPersistenceDelegate.java:253)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:400)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeObject1(Encoder.java:258)
    at java.beans.Encoder.cloneStatement(Encoder.java:271)
    at java.beans.Encoder.writeStatement(Encoder.java:301)
    at java.beans.XMLEncoder.writeStatement(XMLEncoder.java:400)
    at java.beans.DefaultPersistenceDelegate.invokeStatement(DefaultPersistenceDelegate.java:217)
    at java.beans.java_util_List_PersistenceDelegate.initialize(MetaData.java:649)
    at java.beans.PersistenceDelegate.initialize(PersistenceDelegate.java:212)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:398)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.DefaultPersistenceDelegate.doProperty(DefaultPersistenceDelegate.java:194)
    at java.beans.DefaultPersistenceDelegate.initBean(DefaultPersistenceDelegate.java:253)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:400)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeObject1(Encoder.java:258)
    at java.beans.Encoder.cloneStatement(Encoder.java:271)
    at java.beans.Encoder.writeStatement(Encoder.java:301)
    at java.beans.XMLEncoder.writeStatement(XMLEncoder.java:400)
    at java.beans.DefaultPersistenceDelegate.invokeStatement(DefaultPersistenceDelegate.java:217)
    at java.beans.java_util_List_PersistenceDelegate.initialize(MetaData.java:649)
    at java.beans.PersistenceDelegate.initialize(PersistenceDelegate.java:212)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:398)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.DefaultPersistenceDelegate.doProperty(DefaultPersistenceDelegate.java:194)
    at java.beans.DefaultPersistenceDelegate.initBean(DefaultPersistenceDelegate.java:253)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:400)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.java_util_Map_PersistenceDelegate.initialize(MetaData.java:679)
    at java.beans.PersistenceDelegate.initialize(PersistenceDelegate.java:212)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:398)
    at java.beans.PersistenceDelegate.initialize(PersistenceDelegate.java:212)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:398)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.DefaultPersistenceDelegate.doProperty(DefaultPersistenceDelegate.java:194)
    at java.beans.DefaultPersistenceDelegate.initBean(DefaultPersistenceDelegate.java:253)
    at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:400)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeExpression(Encoder.java:330)
    at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454)
    at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115)
    at java.beans.Encoder.writeObject(Encoder.java:74)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327)
    at java.beans.Encoder.writeObject1(Encoder.java:258)
    at java.beans.Encoder.cloneStatement(Encoder.java:271)
    at java.beans.Encoder.writeStatement(Encoder.java:301)
    at java.beans.XMLEncoder.writeStatement(XMLEncoder.java:400)
    at java.beans.XMLEncoder.writeObject(XMLEncoder.java:330)
    at org.apache.hadoop.hive.ql.exec.Utilities.serializeMapRedWork(Utilities.java:457)
    at org.apache.hadoop.hive.ql.exec.Utilities.setMapRedWork(Utilities.java:320)
    at org.apache.hadoop.hive.ql.exec.ExecDriver.execute(ExecDriver.java:634)
    at org.apache.hadoop.hive.ql.exec.MapRedTask.execute(MapRedTask.java:123)
    at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:130)
    at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:57)
    at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:1063)
    at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:900)
    at org.apache.hadoop.hive.ql.Driver.run(Driver.java:748)
    at org.apache.hadoop.hive.service.HiveServer$HiveServerHandler.execute(HiveServer.java:151)
    at org.apache.hadoop.hive.service.ThriftHive$Processor$execute.process(ThriftHive.java:764)
    at org.apache.hadoop.hive.service.ThriftHive$Processor.process(ThriftHive.java:742)
    at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
    at java.lang.Thread.run(Thread.java:722)
FAILED: Execution Error, return code -101 from org.apache.hadoop.hive.ql.exec.MapRedTask
Starting Job = job_201111141853_676751, Tracking URL = http://jobtracker.aer.dip.sina.com.cn:50030/jobdetails.jsp?jobid=job_201111141853_676751
Kill Command = /usr/lib/hadoop/bin/hadoop job  -Dmapred.job.tracker=jobtracker.aer.dip.sina.com.cn:8021
-kill job_201111141853_676751
2011-12-16 00:40:08,917 Stage-1 map = 0%,  reduce = 0%
2011-12-16 00:40:11,929 Stage-1 map = 11%,  reduce = 0%


when this exception was thrown,the hiveserver will be quickly down!









At 2011-12-16 05:18:36,"John Sichi" <jsichi@fb.com> wrote:
>Hey all,
>
>Marek Sapota has put together a doc on the new scripts for spreading Hive unit test execution
across a cluster:
>
>https://cwiki.apache.org/confluence/display/Hive/Unit+Test+Parallel+Execution
>
>Whether you are a committer or someone contributing patches, if you are currently frustrated
by waiting for Hive tests to complete for each patch, please set this up and give it a try.
>
>JVS
>

Mime
View raw message