It turns out to be a bug in application code. Thank you!
________________________________ From: Haopu Wang Sent: 2016年11月4日 17:23 To: user@spark.apache.org; Cody Koeninger Subject: InvalidClassException when load KafkaDirectStream from checkpoint (Spark 2.0.0) When I load spark checkpoint, I get below error. Do you have any idea? Much thanks! ************************* 2016-11-04 17:12:19,582 INFO [org.apache.spark.streaming.CheckpointReader] (main;) Checkpoint files found: file:/d:/temp/checkpoint/checkpoint-1478250700000,file:/d:/temp/checkpoint/checkpoint-1478250700000.bk,file:/d:/temp/checkpoint/checkpoint-1478250690000,file:/d:/temp/checkpoint/checkpoint-1478250690000.bk,file:/d:/temp/checkpoint/checkpoint-1478250680000,file:/d:/temp/checkpoint/checkpoint-1478250680000.bk,file:/d:/temp/checkpoint/checkpoint-1478250670000,file:/d:/temp/checkpoint/checkpoint-1478250670000.bk 2016-11-04 17:12:19,584 INFO [org.apache.spark.streaming.CheckpointReader] (main;) Attempting to load checkpoint from file file:/d:/temp/checkpoint/checkpoint-1478250700000 2016-11-04 17:12:19,640 DEBUG [org.apache.spark.streaming.DStreamGraph] (main;) DStreamGraph.readObject used 2016-11-04 17:12:19,661 DEBUG [org.apache.spark.streaming.kafka010.DirectKafkaInputDStream] (main;) DirectKafkaInputDStream.readObject used 2016-11-04 17:12:19,664 DEBUG [org.apache.spark.streaming.dstream.DStreamCheckpointData] (main;) DStreamCheckpointData.readObject used 2016-11-04 17:12:19,679 DEBUG [org.apache.spark.streaming.kafka010.DirectKafkaInputDStream$DirectKafkaInputDStreamCheckpointData] (main;) DirectKafkaInputDStreamCheckpointData.readObject used 2016-11-04 17:12:19,685 ERROR [org.apache.spark.util.Utils] (main;) Exception encountered java.io.InvalidClassException: scala.collection.convert.Wrappers$MutableMapWrapper; no valid constructor at java.io.ObjectStreamClass$ExceptionInfo.newInvalidClassException(ObjectStreamClass.java:150) at java.io.ObjectStreamClass.checkDeserialize(ObjectStreamClass.java:768) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1772) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) at java.io.ObjectInputStream.readArray(ObjectInputStream.java:1706) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1344) at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990) at java.io.ObjectInputStream.defaultReadObject(ObjectInputStream.java:500) at org.apache.spark.streaming.DStreamGraph$$anonfun$readObject$1.apply$mcV$sp(DStreamGraph.scala:193) at org.apache.spark.streaming.DStreamGraph$$anonfun$readObject$1.apply(DStreamGraph.scala:189) at org.apache.spark.streaming.DStreamGraph$$anonfun$readObject$1.apply(DStreamGraph.scala:189) at org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1253) at org.apache.spark.streaming.DStreamGraph.readObject(DStreamGraph.scala:189) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1017) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1893) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370) at org.apache.spark.streaming.Checkpoint$$anonfun$deserialize$2.apply(Checkpoint.scala:164) at org.apache.spark.streaming.Checkpoint$$anonfun$deserialize$2.apply(Checkpoint.scala:154) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1287) at org.apache.spark.streaming.Checkpoint$.deserialize(Checkpoint.scala:167) at org.apache.spark.streaming.CheckpointReader$$anonfun$read$2.apply(Checkpoint.scala:346) at org.apache.spark.streaming.CheckpointReader$$anonfun$read$2.apply(Checkpoint.scala:342) at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33) at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:35) at org.apache.spark.streaming.CheckpointReader$.read(Checkpoint.scala:342) at org.apache.spark.streaming.StreamingContext$.getOrCreate(StreamingContext.scala:824) at org.apache.spark.streaming.api.java.JavaStreamingContext$.getOrCreate(JavaStreamingContext.scala:624) at org.apache.spark.streaming.api.java.JavaStreamingContext.getOrCreate(JavaStreamingContext.scala) at com.vitria.spark.MainClass.createStreamingContext(MainClass.java:251) at com.vitria.spark.MainClass.main(MainClass.java:126) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:729) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:185) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:210) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:124) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)