// observed in Spark 1.0
Scala devs,
I was observing an unusual NPE in my code recently, and came up with the
below minimal test case:
class Super extends Serializable {
lazy val superVal: String = null
}
class Sub extends Super {
lazy val subVal: String = {
try {
"literal"
} catch {
case _:Throwable => return superVal
}
}
}
Save this to a file, open the Spark shell, and load with ":l
/tmp/test.scala"
I got the below really unusual exception. It goes away though when
removing the try/catch inside subVal and just returning either a straight
literal or superVal.
Is this a bug in Spark, or Scala, or my code, or what? I think it might be
related to the Spark Repl doing magic but I'm unsure what.
Cheers!
Andrew
scala> :l /tmp/test.scala
Loading /tmp/test.scala...
defined class Super
while compiling: <console>
during phase: mixin
library version: version 2.10.4
compiler version: version 2.10.4
reconstructed args:
last tree to typer: Apply(constructor $read)
symbol: constructor $read in class $read (flags: <method>
<triedcooking>)
symbol definition: def <init>(): $line9.$read
tpe: $line9.$read
symbol owners: constructor $read -> class $read -> package $line9
context owners: class iwC$Sub -> package $line9
== Enclosing template or block ==
Template( // val <local Sub>: <notype>, tree.tpe=$line9.iwC$Sub
"$line5.$read$$iwC$$iwC$$iwC$$iwC$Super" // parents
ValDef(
private
"_"
<tpt>
<empty>
)
// 6 statements
ValDef( // lazy private[this] var subVal: String
private <mutable> <local> lazy <triedcooking>
"subVal "
<tpt> // tree.tpe=String
<empty>
)
DefDef( // lazy val subVal(): String
<method> <stable> <accessor> lazy
"subVal"
[]
List(Nil)
<tpt> // tree.tpe=String
Block( // tree.tpe=String
ValDef( // val nonLocalReturnKey1: Object
<synthetic> <triedcooking>
"nonLocalReturnKey1"
<tpt> // tree.tpe=Object
Apply( // def <init>(): Object in class Object, tree.tpe=Object
new Object."<init>" // def <init>(): Object in class Object,
tree.tpe=()Object
Nil
)
)
Try( // tree.tpe=String
Block( // tree.tpe=String
Assign( // tree.tpe=Unit
$read$$iwC$$iwC$$iwC$$iwC$Sub.this."subVal " // lazy
private[this] var subVal: String, tree.tpe=String
Block( // tree.tpe=String
{}
Apply( // final private[this] def
liftedTree1$1(nonLocalReturnKey1$1: Object): String, tree.tpe=String
$read$$iwC$$iwC$$iwC$$iwC$Sub.this."liftedTree1$1" // final
private[this] def liftedTree1$1(nonLocalReturnKey1$1: Object): String,
tree.tpe=(nonLocalReturnKey1$1: Object)String
"nonLocalReturnKey1" // val nonLocalReturnKey1: Object,
tree.tpe=Object
)
)
)
$read$$iwC$$iwC$$iwC$$iwC$Sub.this."subVal " // lazy
private[this] var subVal: String, tree.tpe=String
)
CaseDef( // tree.tpe=String
Bind( // val ex: runtime.NonLocalReturnControl,
tree.tpe=runtime.NonLocalReturnControl
"ex"
Typed( // tree.tpe=runtime.NonLocalReturnControl
"_" // tree.tpe=runtime.NonLocalReturnControl
<tpt> // tree.tpe=runtime.NonLocalReturnControl
)
)
If( // tree.tpe=String
Apply( // final def eq(x$1: Object): Boolean in class Object,
tree.tpe=Boolean
ex.key()."eq" // final def eq(x$1: Object): Boolean in class
Object, tree.tpe=(x$1: Object)Boolean
"nonLocalReturnKey1" // val nonLocalReturnKey1: Object,
tree.tpe=Object
)
Apply( // final def $asInstanceOf[T0 >: ? <: ?](): T0 in class
Object, tree.tpe=String
TypeApply( // final def $asInstanceOf[T0 >: ? <: ?](): T0 in
class Object, tree.tpe=()String
ex.value()."$asInstanceOf" // final def $asInstanceOf[T0 >:
? <: ?](): T0 in class Object, tree.tpe=[T0 >: ? <: ?]()T0
<tpt> // tree.tpe=String
)
Nil
)
Throw("ex")tree.tpe=Nothing
)
)
)
)
)
ValDef( // protected val $outer: $line9.iwC
protected <synthetic> <paramaccessor> <triedcooking>
"$outer "
<tpt> // tree.tpe=$line9.iwC
<empty>
)
DefDef( // val $outer(): $line9.iwC
<method> <synthetic> <stable> <expandedname>
"$line9$$read$$iwC$$iwC$$iwC$$iwC$Sub$$$outer"
[]
List(Nil)
<tpt> // tree.tpe=$line9.iwC
$read$$iwC$$iwC$$iwC$$iwC$Sub.this."$outer " // protected val $outer:
$line9.iwC, tree.tpe=$line9.iwC
)
DefDef( // final private[this] def liftedTree1$1(nonLocalReturnKey1$1:
Object): String
<method> private final <local> <lifted> <triedcooking>
"liftedTree1"
[]
// 1 parameter list
ValDef( // nonLocalReturnKey1$1: Object
<param> <synthetic>
"nonLocalReturnKey1$1"
<tpt> // tree.tpe=Object
<empty>
)
<tpt> // tree.tpe=String
Try( // tree.tpe=String
"literal"
CaseDef( // tree.tpe=Nothing
Typed( // tree.tpe=Throwable
"_" // tree.tpe=Throwable
<tpt> // tree.tpe=Throwable
)
Throw( // tree.tpe=Nothing
Apply( // def <init>(key: Object,value: Object):
scala.runtime.NonLocalReturnControl in class NonLocalReturnControl,
tree.tpe=scala.runtime.NonLocalReturnControl
new runtime.NonLocalReturnControl."<init>" // def <init>(key:
Object,value: Object): scala.runtime.NonLocalReturnControl in class
NonLocalReturnControl, tree.tpe=(key: Object, value:
Object)scala.runtime.NonLocalReturnControl
// 2 arguments
"nonLocalReturnKey1$1" // nonLocalReturnKey1$1: Object,
tree.tpe=Object
Apply( // lazy val superVal(): String, tree.tpe=String
$read$$iwC$$iwC$$iwC$$iwC$Sub.this."superVal" // lazy val
superVal(): String, tree.tpe=()String
Nil
)
)
)
)
)
)
DefDef( // def <init>(arg$outer: $line9.iwC): $line9.iwC$Sub
<method>
"<init>"
[]
// 1 parameter list
ValDef( // $outer: $line9.iwC
<param> <triedcooking>
"$outer"
<tpt> // tree.tpe=$line9.iwC
<empty>
)
<tpt> // tree.tpe=$line9.iwC$Sub
Block( // tree.tpe=Unit
// 2 statements
If( // tree.tpe=Unit
Apply( // final def eq(x$1: Object): Boolean in class Object,
tree.tpe=Boolean
"$outer"."eq" // final def eq(x$1: Object): Boolean in class
Object, tree.tpe=(x$1: Object)Boolean
null
)
Throw( // tree.tpe=Nothing
Apply( // def <init>(): NullPointerException in class
NullPointerException, tree.tpe=NullPointerException
new NullPointerException."<init>" // def <init>():
NullPointerException in class NullPointerException,
tree.tpe=()NullPointerException
Nil
)
)
Assign( // tree.tpe=Unit
$read$$iwC$$iwC$$iwC$$iwC$Sub.this."$outer " // protected val
$outer: $line9.iwC, tree.tpe=$line9.iwC
"$outer" // $outer: $line9.iwC, tree.tpe=$line9.iwC
)
)
Apply( // def <init>(arg$outer: $line5.iwC): $line5.iwC$Super,
tree.tpe=$line5.iwC$Super
$read$$iwC$$iwC$$iwC$$iwC$Sub.super."<init>" // def
<init>(arg$outer: $line5.iwC): $line5.iwC$Super, tree.tpe=(arg$outer:
$line5.iwC)$line5.iwC$Super
Apply( // val $iw(): $line5.iwC, tree.tpe=$line5.iwC
$outer.$line9$$read$$iwC$$iwC$$iwC$$iwC$$$outer().$VAL1().$iw().$iw().$iw()."$iw"
// val $iw(): $line5.iwC, tree.tpe=()$line5.iwC
Nil
)
)
()
)
)
)
== Expanded type of tree ==
TypeRef(TypeSymbol(class $read extends Serializable))
unhandled exception while transforming <console>
error: uncaught exception during compilation: java.lang.NullPointerException
java.lang.NullPointerException
at scala.reflect.internal.Trees$class.Select(Trees.scala:1066)
at scala.reflect.internal.SymbolTable.Select(SymbolTable.scala:13)
at
scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$scala$tools$nsc$transform$Mixin$MixinTransformer$$dd$1$2.apply(Mixin.scala:908)
at
scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$scala$tools$nsc$transform$Mixin$MixinTransformer$$dd$1$2.apply(Mixin.scala:904)
at scala.reflect.internal.Trees$class.deriveDefDef(Trees.scala:1598)
at scala.reflect.internal.SymbolTable.deriveDefDef(SymbolTable.scala:13)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.scala$tools$nsc$transform$Mixin$MixinTransformer$$dd$1(Mixin.scala:904)
at
scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$addCheckedGetters$1$1.apply(Mixin.scala:945)
at
scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$addCheckedGetters$1$1.apply(Mixin.scala:945)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at scala.collection.immutable.List.foreach(List.scala:318)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
at scala.collection.AbstractTraversable.map(Traversable.scala:105)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.addCheckedGetters$1(Mixin.scala:945)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.addNewDefs(Mixin.scala:1013)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.scala$tools$nsc$transform$Mixin$MixinTransformer$$postTransform(Mixin.scala:1147)
at
scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$transform$1.apply(Mixin.scala:1261)
at
scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$transform$1.apply(Mixin.scala:1261)
at scala.reflect.internal.SymbolTable.atPhase(SymbolTable.scala:207)
at scala.reflect.internal.SymbolTable.afterPhase(SymbolTable.scala:216)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:1261)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:471)
at scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)
at
scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)
at
scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)
at scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)
at scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)
at scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)
at scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)
at scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:1258)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:471)
at
scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)
at
scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)
at scala.collection.immutable.List.loop$1(List.scala:170)
at scala.collection.immutable.List.mapConserve(List.scala:186)
at scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)
at
scala.reflect.internal.Trees$$anonfun$itransform$7.apply(Trees.scala:1298)
at
scala.reflect.internal.Trees$$anonfun$itransform$7.apply(Trees.scala:1298)
at scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)
at scala.reflect.internal.Trees$class.itransform(Trees.scala:1297)
at scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)
at scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)
at scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:1258)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:471)
at scala.tools.nsc.ast.Trees$Transformer.transformUnit(Trees.scala:227)
at scala.tools.nsc.transform.Transform$Phase.apply(Transform.scala:30)
at scala.tools.nsc.Global$GlobalPhase.applyPhase(Global.scala:464)
at scala.tools.nsc.Global$GlobalPhase$$anonfun$run$1.apply(Global.scala:431)
at
scala.tools.nsc.Global$GlobalPhase$$anonfun$run$1.apply(Global.scala:431)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.tools.nsc.Global$GlobalPhase.run(Global.scala:431)
at scala.tools.nsc.Global$Run.compileUnitsInternal(Global.scala:1583)
at scala.tools.nsc.Global$Run.compileUnits(Global.scala:1557)
at scala.tools.nsc.Global$Run.compileSources(Global.scala:1553)
at
org.apache.spark.repl.SparkIMain.compileSourcesKeepingRun(SparkIMain.scala:468)
at
org.apache.spark.repl.SparkIMain$ReadEvalPrint.compileAndSaveRun(SparkIMain.scala:859)
at
org.apache.spark.repl.SparkIMain$ReadEvalPrint.compile(SparkIMain.scala:815)
at
org.apache.spark.repl.SparkIMain$Request.compile$lzycompute(SparkIMain.scala:1009)
at org.apache.spark.repl.SparkIMain$Request.compile(SparkIMain.scala:1004)
at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:644)
at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:609)
at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:796)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:753)
at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:601)
at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:608)
at org.apache.spark.repl.SparkILoop.loop(SparkILoop.scala:611)
at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1$$anonfun$apply$mcV$sp$2.apply(SparkILoop.scala:621)
at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1$$anonfun$apply$mcV$sp$2.apply(SparkILoop.scala:618)
at
scala.reflect.io.Streamable$Chars$class.applyReader(Streamable.scala:104)
at scala.reflect.io.File.applyReader(File.scala:82)
at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SparkILoop.scala:618)
at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1.apply(SparkILoop.scala:618)
at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1.apply(SparkILoop.scala:618)
at org.apache.spark.repl.SparkILoop.savingReplayStack(SparkILoop.scala:150)
at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1.apply$mcV$sp(SparkILoop.scala:617)
at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1.apply(SparkILoop.scala:617)
at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1.apply(SparkILoop.scala:617)
at org.apache.spark.repl.SparkILoop.savingReader(SparkILoop.scala:155)
at org.apache.spark.repl.SparkILoop.interpretAllFrom(SparkILoop.scala:616)
at
org.apache.spark.repl.SparkILoop$$anonfun$loadCommand$1.apply(SparkILoop.scala:681)
at
org.apache.spark.repl.SparkILoop$$anonfun$loadCommand$1.apply(SparkILoop.scala:680)
at org.apache.spark.repl.SparkILoop.withFile(SparkILoop.scala:674)
at org.apache.spark.repl.SparkILoop.loadCommand(SparkILoop.scala:680)
at
org.apache.spark.repl.SparkILoop$$anonfun$standardCommands$7.apply(SparkILoop.scala:294)
at
org.apache.spark.repl.SparkILoop$$anonfun$standardCommands$7.apply(SparkILoop.scala:294)
at
scala.tools.nsc.interpreter.LoopCommands$LineCmd.apply(LoopCommands.scala:81)
at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:748)
at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:601)
at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:608)
at org.apache.spark.repl.SparkILoop.loop(SparkILoop.scala:611)
at
org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply$mcZ$sp(SparkILoop.scala:936)
at
org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
at
org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
at
scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:884)
at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:983)
at org.apache.spark.repl.Main$.main(Main.scala:31)
at org.apache.spark.repl.Main.main(Main.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:256)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:54)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Abandoning crashed session.
scala>
