I use the attached program to test checkpoint. It's quite simple.
When I run the program second time, it will load checkpoint data, that's
expected, however I see NPE in driver log.
Do you have any idea about the issue? I'm on Spark 1.4.0, thank you very
much!
====== logs ======
15/06/15 15:27:17 [THREAD ID=main] INFO FileInputDStream: Restoring
checkpoint data
15/06/15 15:27:17 [THREAD ID=main] INFO
FileInputDStream$FileInputDStreamCheckpointData: Restoring files for
time 1434353130000 ms - []
15/06/15 15:27:17 [THREAD ID=main] INFO
FileInputDStream$FileInputDStreamCheckpointData: Restoring files for
time 1434353140000 ms - []
15/06/15 15:27:17 [THREAD ID=main] INFO
FileInputDStream$FileInputDStreamCheckpointData: Restoring files for
time 1434353150000 ms - []
15/06/15 15:27:17 [THREAD ID=main] INFO
FileInputDStream$FileInputDStreamCheckpointData: Restoring files for
time 1434353160000 ms - []
15/06/15 15:27:17 [THREAD ID=main] INFO
FileInputDStream$FileInputDStreamCheckpointData: Restoring files for
time 1434353170000 ms - []
15/06/15 15:27:17 [THREAD ID=main] INFO
FileInputDStream$FileInputDStreamCheckpointData: Restoring files for
time 1434353180000 ms - []
15/06/15 15:27:17 [THREAD ID=main] INFO
FileInputDStream$FileInputDStreamCheckpointData: Restoring files for
time 1434353190000 ms - []
15/06/15 15:27:17 [THREAD ID=main] INFO FileInputDStream: Restored
checkpoint data
15/06/15 15:27:17 [THREAD ID=main] INFO MappedDStream: Restored
checkpoint data
15/06/15 15:27:17 [THREAD ID=main] INFO MappedDStream: Restored
checkpoint data
15/06/15 15:27:17 [THREAD ID=main] INFO ForEachDStream: Restored
checkpoint data
15/06/15 15:27:17 [THREAD ID=main] INFO DStreamGraph: Restored
checkpoint data
15/06/15 15:27:17 [THREAD ID=main] ERROR StreamingContext: Error
starting the context, marking it as stopped
java.io.IOException: java.lang.NullPointerException
at
org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1242)
at
org.apache.spark.streaming.dstream.DStreamCheckpointData.writeObject(DSt
reamCheckpointData.scala:123)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:988)
at
java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1495)
at
java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1
431)
at
java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177)
at
java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:15
47)
at
java.io.ObjectOutputStream.defaultWriteObject(ObjectOutputStream.java:44
0)
at
org.apache.spark.streaming.dstream.DStream$$anonfun$writeObject$1.apply$
mcV$sp(DStream.scala:498)
at
org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1239)
at
org.apache.spark.streaming.dstream.DStream.writeObject(DStream.scala:493
)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:988)
at
java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1495)
at
java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1
431)
at
java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177)
at
java.io.ObjectOutputStream.writeArray(ObjectOutputStream.java:1377)
at
java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1173)
at
java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:15
47)
at
java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1508)
at
java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1
431)
at
java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177)
at
java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:15
47)
at
java.io.ObjectOutputStream.defaultWriteObject(ObjectOutputStream.java:44
0)
at
org.apache.spark.streaming.DStreamGraph$$anonfun$writeObject$1.apply$mcV
$sp(DStreamGraph.scala:181)
at
org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1239)
at
org.apache.spark.streaming.DStreamGraph.writeObject(DStreamGraph.scala:1
76)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:988)
at
java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1495)
at
java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1
431)
at
java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177)
at
java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:15
47)
at
java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1508)
at
java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1
431)
at
java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177)
at
java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:347)
at
org.apache.spark.streaming.Checkpoint$$anonfun$serialize$1.apply$mcV$sp(
Checkpoint.scala:113)
at
org.apache.spark.streaming.Checkpoint$$anonfun$serialize$1.apply(Checkpo
int.scala:113)
at
org.apache.spark.streaming.Checkpoint$$anonfun$serialize$1.apply(Checkpo
int.scala:113)
at
org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1285)
at
org.apache.spark.streaming.Checkpoint$.serialize(Checkpoint.scala:114)
at
org.apache.spark.streaming.StreamingContext.validate(StreamingContext.sc
ala:547)
at
org.apache.spark.streaming.StreamingContext.liftedTree1$1(StreamingConte
xt.scala:587)
at
org.apache.spark.streaming.StreamingContext.start(StreamingContext.scala
:586)
at
CheckPointTest$delayedInit$body.apply(CheckPointTest.scala:116)
at scala.Function0$class.apply$mcV$sp(Function0.scala:40)
at
scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12)
at scala.App$$anonfun$main$1.apply(App.scala:71)
at scala.App$$anonfun$main$1.apply(App.scala:71)
at scala.collection.immutable.List.foreach(List.scala:318)
at
scala.collection.generic.TraversableForwarder$class.foreach(TraversableF
orwarder.scala:32)
at scala.App$class.main(App.scala:71)
at CheckPointTest$.main(CheckPointTest.scala:7)
at CheckPointTest.main(CheckPointTest.scala)
Caused by: java.lang.NullPointerException
at
org.apache.spark.streaming.dstream.DStreamCheckpointData$$anonfun$writeO
bject$1.apply$mcV$sp(DStreamCheckpointData.scala:125)
at
org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1239)
... 64 more
CheckPointTest.scala
Description: CheckPointTest.scala
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@spark.apache.org For additional commands, e-mail: dev-h...@spark.apache.org
