spark-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shahid <sha...@trialx.com>
Subject getting this error while runing
Date Sat, 28 Feb 2015 19:11:31 GMT

conf =
SparkConf().setAppName("spark_calc3merged").setMaster("spark://ec2-54-145-68-13.compute-1.amazonaws.com:7077")
    sc =
SparkContext(conf=conf,pyFiles=["/root/platinum.py","/root/collections2.py"])
  
15/02/28 19:06:38 WARN scheduler.TaskSetManager: Lost task 5.0 in stage 3.0
(TID 38, ip-10-80-15-145.ec2.internal):
com.esotericsoftware.kryo.KryoException: Buffer overflow. Available: 0,
required: 73065
        com.esotericsoftware.kryo.io.Output.require(Output.java:138)
        com.esotericsoftware.kryo.io.Output.writeBytes(Output.java:220)
        com.esotericsoftware.kryo.io.Output.writeBytes(Output.java:206)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ByteArraySerializer.write(DefaultArraySerializers.java:29)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ByteArraySerializer.write(DefaultArraySerializers.java:18)
        com.esotericsoftware.kryo.Kryo.writeObjectOrNull(Kryo.java:549)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:312)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:293)
        com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:568)
       
org.apache.spark.serializer.KryoSerializerInstance.serialize(KryoSerializer.scala:156)
       
org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:187)
       
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
       
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        java.lang.Thread.run(Thread.java:745)
15/02/28 19:06:38 INFO scheduler.TaskSetManager: Starting task 5.1 in stage
3.0 (TID 44, ip-10-80-15-145.ec2.internal, NODE_LOCAL, 1502 bytes)
15/02/28 19:06:38 INFO scheduler.TaskSetManager: Finished task 8.0 in stage
3.0 (TID 41) in 7040 ms on ip-10-80-98-118.ec2.internal (9/11)
15/02/28 19:06:38 INFO scheduler.TaskSetManager: Finished task 9.0 in stage
3.0 (TID 42) in 7847 ms on ip-10-80-15-145.ec2.internal (10/11)
15/02/28 19:06:50 WARN scheduler.TaskSetManager: Lost task 5.1 in stage 3.0
(TID 44, ip-10-80-15-145.ec2.internal):
com.esotericsoftware.kryo.KryoException: Buffer overflow. Available: 0,
required: 73065
        com.esotericsoftware.kryo.io.Output.require(Output.java:138)
        com.esotericsoftware.kryo.io.Output.writeBytes(Output.java:220)
        com.esotericsoftware.kryo.io.Output.writeBytes(Output.java:206)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ByteArraySerializer.write(DefaultArraySerializers.java:29)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ByteArraySerializer.write(DefaultArraySerializers.java:18)
        com.esotericsoftware.kryo.Kryo.writeObjectOrNull(Kryo.java:549)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:312)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:293)
        com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:568)
       
org.apache.spark.serializer.KryoSerializerInstance.serialize(KryoSerializer.scala:156)
       
org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:187)
       
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
       
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        java.lang.Thread.run(Thread.java:745)
15/02/28 19:06:50 INFO scheduler.TaskSetManager: Starting task 5.2 in stage
3.0 (TID 45, ip-10-80-98-118.ec2.internal, NODE_LOCAL, 1502 bytes)
15/02/28 19:07:01 WARN scheduler.TaskSetManager: Lost task 5.2 in stage 3.0
(TID 45, ip-10-80-98-118.ec2.internal):
com.esotericsoftware.kryo.KryoException: Buffer overflow. Available: 0,
required: 73065
        com.esotericsoftware.kryo.io.Output.require(Output.java:138)
        com.esotericsoftware.kryo.io.Output.writeBytes(Output.java:220)
        com.esotericsoftware.kryo.io.Output.writeBytes(Output.java:206)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ByteArraySerializer.write(DefaultArraySerializers.java:29)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ByteArraySerializer.write(DefaultArraySerializers.java:18)
        com.esotericsoftware.kryo.Kryo.writeObjectOrNull(Kryo.java:549)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:312)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:293)
        com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:568)
       
org.apache.spark.serializer.KryoSerializerInstance.serialize(KryoSerializer.scala:156)
       
org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:187)
       
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
       
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        java.lang.Thread.run(Thread.java:745)
15/02/28 19:07:01 INFO scheduler.TaskSetManager: Starting task 5.3 in stage
3.0 (TID 46, ip-10-80-15-145.ec2.internal, NODE_LOCAL, 1502 bytes)
15/02/28 19:07:13 WARN scheduler.TaskSetManager: Lost task 5.3 in stage 3.0
(TID 46, ip-10-80-15-145.ec2.internal):
com.esotericsoftware.kryo.KryoException: Buffer overflow. Available: 0,
required: 73065
        com.esotericsoftware.kryo.io.Output.require(Output.java:138)
        com.esotericsoftware.kryo.io.Output.writeBytes(Output.java:220)
        com.esotericsoftware.kryo.io.Output.writeBytes(Output.java:206)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ByteArraySerializer.write(DefaultArraySerializers.java:29)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ByteArraySerializer.write(DefaultArraySerializers.java:18)
        com.esotericsoftware.kryo.Kryo.writeObjectOrNull(Kryo.java:549)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:312)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:293)
        com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:568)
       
org.apache.spark.serializer.KryoSerializerInstance.serialize(KryoSerializer.scala:156)
       
org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:187)
       
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
       
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        java.lang.Thread.run(Thread.java:745)
15/02/28 19:07:13 ERROR scheduler.TaskSetManager: Task 5 in stage 3.0 failed
4 times; aborting job
15/02/28 19:07:13 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 3.0,
whose tasks have all completed, from pool
15/02/28 19:07:13 INFO scheduler.TaskSchedulerImpl: Cancelling stage 3
15/02/28 19:07:13 INFO scheduler.DAGScheduler: Failed to run collect at
/root/spark_calculations.py:152
Traceback (most recent call last):
  File "/root/spark_calculations.py", line 348, in <module>
    CTGov_5year_all = C.get_count_five_year(ctgov_data)
  File "/root/spark_calculations.py", line 152, in get_count_five_year
    for year,data in year_data.collect():
  File "/root/spark/python/pyspark/rdd.py", line 723, in collect
    bytesInJava = self._jrdd.collect().iterator()
  File "/root/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py",
line 538, in __call__
  File "/root/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line
300, in get_return_value
py4j.protocol.Py4JJavaError: An error occurred while calling o59.collect.
: org.apache.spark.SparkException: Job aborted due to stage failure: Task 5
in stage 3.0 failed 4 times, most recent failure: Lost task 5.3 in stage 3.0
(TID 46, ip-10-80-15-145.ec2.internal):
com.esotericsoftware.kryo.KryoException: Buffer overflow. Available: 0,
required: 73065
        com.esotericsoftware.kryo.io.Output.require(Output.java:138)
        com.esotericsoftware.kryo.io.Output.writeBytes(Output.java:220)
        com.esotericsoftware.kryo.io.Output.writeBytes(Output.java:206)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ByteArraySerializer.write(DefaultArraySerializers.java:29)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ByteArraySerializer.write(DefaultArraySerializers.java:18)
        com.esotericsoftware.kryo.Kryo.writeObjectOrNull(Kryo.java:549)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:312)
       
com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:293)
        com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:568)
       
org.apache.spark.serializer.KryoSerializerInstance.serialize(KryoSerializer.scala:156)
       
org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:187)
       
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
       
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        java.lang.Thread.run(Thread.java:745)
Driver stacktrace:



--
View this message in context: http://apache-spark-user-list.1001560.n3.nabble.com/getting-this-error-while-runing-tp21860.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.

---------------------------------------------------------------------
To unsubscribe, e-mail: user-unsubscribe@spark.apache.org
For additional commands, e-mail: user-help@spark.apache.org


Mime
View raw message