I am hitting the following exception when running Ignite with Spark on Yarn.
Here is the snippet of the code.
The same job runs fine in spark local mode (spark-master: local). Only
failing when running on Yarn.
val config = new IgniteConfiguration()
val tcpDiscoverySpi = new TcpDiscoverySpi()
val ipFinder = new TcpDiscoveryVmIpFinder()
ipFinder.setAddresses(
util.Arrays.asList(
"server1-ip",
"server2-ip",
"server3-ip",
"server4-ip",
"server5-ip:47500"
))
tcpDiscoverySpi.setIpFinder(ipFinder)
config.setDiscoverySpi(tcpDiscoverySpi)
val igniteContext = new IgniteContext(spark.sparkContext, () ⇒ config,
standalone = false)
Exception:
Driver stacktrace:
at
org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422)
at
scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at
org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
at scala.Option.foreach(Option.scala:257)
at
org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
at
org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1944)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1958)
at
org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:925)
at
org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:923)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:362)
at org.apache.spark.rdd.RDD.foreachPartition(RDD.scala:923)
at org.apache.ignite.spark.IgniteContext.<init>(IgniteContext.scala:54)
at
BulkLoadFeatures$.delayedEndpoint$BulkLoadFeatures$1(BulkLoadFeatures.scala:37)
at BulkLoadFeatures$delayedInit$body.apply(BulkLoadFeatures.scala:18)
at scala.Function0$class.apply$mcV$sp(Function0.scala:34)
at
scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12)
at scala.App$$anonfun$main$1.apply(App.scala:76)
at scala.App$$anonfun$main$1.apply(App.scala:76)
at scala.collection.immutable.List.foreach(List.scala:381)
at
scala.collection.generic.TraversableForwarder$class.foreach(TraversableForwarder.scala:35)
at scala.App$class.main(App.scala:76)
at BulkLoadFeatures$.main(BulkLoadFeatures.scala:18)
at BulkLoadFeatures.main(BulkLoadFeatures.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:637)
Caused by: java.lang.NullPointerException: Ouch! Argument cannot be null:
cfg
at
org.apache.ignite.internal.util.GridArgumentCheck.notNull(GridArgumentCheck.java:48)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:594)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:536)
at org.apache.ignite.Ignition.getOrStart(Ignition.java:414)
at org.apache.ignite.spark.IgniteContext.ignite(IgniteContext.scala:143)
at
org.apache.ignite.spark.IgniteContext$$anonfun$1.apply(IgniteContext.scala:54)
at
org.apache.ignite.spark.IgniteContext$$anonfun$1.apply(IgniteContext.scala:54)
at
org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:925)
at
org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$29.apply(RDD.scala:925)
at
org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1944)
at
org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1944)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:99)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Can someone please look into this error?
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
