ignite-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pradeepchanumolu <pradeepb...@gmail.com>
Subject Ignite Context failing with java.lang.NullPointerException: Ouch! Argument cannot be null: cfg
Date Thu, 21 Sep 2017 22:09:57 GMT
I am hitting the following exception when running Ignite with Spark on Yarn.
Here is the snippet of the code.
The same job runs fine in spark local mode (spark-master: local). Only
failing when running on Yarn.  

val config = new IgniteConfiguration()
val tcpDiscoverySpi = new TcpDiscoverySpi()
val ipFinder = new TcpDiscoveryVmIpFinder()
ipFinder.setAddresses(
      util.Arrays.asList(
        "server1-ip",
        "server2-ip",
        "server3-ip",
        "server4-ip",
        "server5-ip:47500"
      ))
tcpDiscoverySpi.setIpFinder(ipFinder)
config.setDiscoverySpi(tcpDiscoverySpi)


val igniteContext = new IgniteContext(spark.sparkContext, () ⇒ config,
standalone = false)

Exception: 


Driver stacktrace:
	at
org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435)
	at
org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423)
	at
org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422)
	at
scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
	at
org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422)
	at
org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
	at
org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
	at scala.Option.foreach(Option.scala:257)
	at
org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802)
	at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650)
	at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605)
	at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594)
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:1944)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:1958)
	at
org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:925)
	at
org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:923)
	at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
	at org.apache.spark.rdd.RDD.withScope(RDD.scala:362)
	at org.apache.spark.rdd.RDD.foreachPartition(RDD.scala:923)
	at org.apache.ignite.spark.IgniteContext.<init>(IgniteContext.scala:54)
	at
BulkLoadFeatures$.delayedEndpoint$BulkLoadFeatures$1(BulkLoadFeatures.scala:37)
	at BulkLoadFeatures$delayedInit$body.apply(BulkLoadFeatures.scala:18)
	at scala.Function0$class.apply$mcV$sp(Function0.scala:34)
	at scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12)
	at scala.App$$anonfun$main$1.apply(App.scala:76)
	at scala.App$$anonfun$main$1.apply(App.scala:76)
	at scala.collection.immutable.List.foreach(List.scala:381)
	at
scala.collection.generic.TraversableForwarder$class.foreach(TraversableForwarder.scala:35)
	at scala.App$class.main(App.scala:76)
	at BulkLoadFeatures$.main(BulkLoadFeatures.scala:18)
	at BulkLoadFeatures.main(BulkLoadFeatures.scala)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at
org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:637)
Caused by: java.lang.NullPointerException: Ouch! Argument cannot be null:
cfg
	at
org.apache.ignite.internal.util.GridArgumentCheck.notNull(GridArgumentCheck.java:48)
	at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:594)
	at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:536)
	at org.apache.ignite.Ignition.getOrStart(Ignition.java:414)
	at org.apache.ignite.spark.IgniteContext.ignite(IgniteContext.scala:143)
	at
org.apache.ignite.spark.IgniteContext$$anonfun$1.apply(IgniteContext.scala:54)
	at
org.apache.ignite.spark.IgniteContext$$anonfun$1.apply(IgniteContext.scala:54)
	at
org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:925)
	at
org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:925)
	at
org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1944)
	at
org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1944)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
	at org.apache.spark.scheduler.Task.run(Task.scala:99)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)
	at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)





Can someone please look into this error? 



--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/

Mime
View raw message