spark-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Andrew Ash <and...@andrewash.com>
Subject Scala Language NPE
Date Tue, 03 Jun 2014 06:35:41 GMT
// observed in Spark 1.0

Scala devs,

I was observing an unusual NPE in my code recently, and came up with the
below minimal test case:

class Super extends Serializable {
    lazy val superVal: String = null
}

class Sub extends Super {
    lazy val subVal: String = {
        try {
            "literal"
        } catch {
            case _:Throwable => return superVal
        }
    }
}

Save this to a file, open the Spark shell, and load with ":l
/tmp/test.scala"

I got the below really unusual exception.  It goes away though when
removing the try/catch inside subVal and just returning either a straight
literal or superVal.


Is this a bug in Spark, or Scala, or my code, or what?  I think it might be
related to the Spark Repl doing magic but I'm unsure what.

Cheers!
Andrew




scala> :l /tmp/test.scala
Loading /tmp/test.scala...
defined class Super

     while compiling: <console>
        during phase: mixin
     library version: version 2.10.4
    compiler version: version 2.10.4
  reconstructed args:

  last tree to typer: Apply(constructor $read)
              symbol: constructor $read in class $read (flags: <method>
<triedcooking>)
   symbol definition: def <init>(): $line9.$read
                 tpe: $line9.$read
       symbol owners: constructor $read -> class $read -> package $line9
      context owners: class iwC$Sub -> package $line9

== Enclosing template or block ==

Template( // val <local Sub>: <notype>, tree.tpe=$line9.iwC$Sub
  "$line5.$read$$iwC$$iwC$$iwC$$iwC$Super" // parents
  ValDef(
    private
    "_"
    <tpt>
    <empty>
  )
  // 6 statements
  ValDef( // lazy private[this] var subVal: String
    private <mutable> <local> lazy <triedcooking>
    "subVal "
    <tpt> // tree.tpe=String
    <empty>
  )
  DefDef( // lazy val subVal(): String
    <method> <stable> <accessor> lazy
    "subVal"
    []
    List(Nil)
    <tpt> // tree.tpe=String
    Block( // tree.tpe=String
      ValDef( // val nonLocalReturnKey1: Object
        <synthetic> <triedcooking>
        "nonLocalReturnKey1"
        <tpt> // tree.tpe=Object
        Apply( // def <init>(): Object in class Object, tree.tpe=Object
          new Object."<init>" // def <init>(): Object in class Object,
tree.tpe=()Object
          Nil
        )
      )
      Try( // tree.tpe=String
        Block( // tree.tpe=String
          Assign( // tree.tpe=Unit
            $read$$iwC$$iwC$$iwC$$iwC$Sub.this."subVal " // lazy
private[this] var subVal: String, tree.tpe=String
            Block( // tree.tpe=String
              {}
              Apply( // final private[this] def
liftedTree1$1(nonLocalReturnKey1$1: Object): String, tree.tpe=String
                $read$$iwC$$iwC$$iwC$$iwC$Sub.this."liftedTree1$1" // final
private[this] def liftedTree1$1(nonLocalReturnKey1$1: Object): String,
tree.tpe=(nonLocalReturnKey1$1: Object)String
                "nonLocalReturnKey1" // val nonLocalReturnKey1: Object,
tree.tpe=Object
              )
            )
          )
          $read$$iwC$$iwC$$iwC$$iwC$Sub.this."subVal " // lazy
private[this] var subVal: String, tree.tpe=String
        )
        CaseDef( // tree.tpe=String
          Bind( // val ex: runtime.NonLocalReturnControl,
tree.tpe=runtime.NonLocalReturnControl
            "ex"
            Typed( // tree.tpe=runtime.NonLocalReturnControl
              "_" // tree.tpe=runtime.NonLocalReturnControl
              <tpt> // tree.tpe=runtime.NonLocalReturnControl
            )
          )
          If( // tree.tpe=String
            Apply( // final def eq(x$1: Object): Boolean in class Object,
tree.tpe=Boolean
              ex.key()."eq" // final def eq(x$1: Object): Boolean in class
Object, tree.tpe=(x$1: Object)Boolean
              "nonLocalReturnKey1" // val nonLocalReturnKey1: Object,
tree.tpe=Object
            )
            Apply( // final def $asInstanceOf[T0 >: ? <: ?](): T0 in class
Object, tree.tpe=String
              TypeApply( // final def $asInstanceOf[T0 >: ? <: ?](): T0 in
class Object, tree.tpe=()String
                ex.value()."$asInstanceOf" // final def $asInstanceOf[T0 >:
? <: ?](): T0 in class Object, tree.tpe=[T0 >: ? <: ?]()T0
                <tpt> // tree.tpe=String
              )
              Nil
            )
            Throw("ex")tree.tpe=Nothing
          )
        )
      )
    )
  )
  ValDef( // protected val $outer: $line9.iwC
    protected <synthetic> <paramaccessor> <triedcooking>
    "$outer "
    <tpt> // tree.tpe=$line9.iwC
    <empty>
  )
  DefDef( // val $outer(): $line9.iwC
    <method> <synthetic> <stable> <expandedname>
    "$line9$$read$$iwC$$iwC$$iwC$$iwC$Sub$$$outer"
    []
    List(Nil)
    <tpt> // tree.tpe=$line9.iwC
    $read$$iwC$$iwC$$iwC$$iwC$Sub.this."$outer " // protected val $outer:
$line9.iwC, tree.tpe=$line9.iwC
  )
  DefDef( // final private[this] def liftedTree1$1(nonLocalReturnKey1$1:
Object): String
    <method> private final <local> <lifted> <triedcooking>
    "liftedTree1"
    []
    // 1 parameter list
    ValDef( // nonLocalReturnKey1$1: Object
      <param> <synthetic>
      "nonLocalReturnKey1$1"
      <tpt> // tree.tpe=Object
      <empty>
    )
    <tpt> // tree.tpe=String
    Try( // tree.tpe=String
      "literal"
      CaseDef( // tree.tpe=Nothing
        Typed( // tree.tpe=Throwable
          "_" // tree.tpe=Throwable
          <tpt> // tree.tpe=Throwable
        )
        Throw( // tree.tpe=Nothing
          Apply( // def <init>(key: Object,value: Object):
scala.runtime.NonLocalReturnControl in class NonLocalReturnControl,
tree.tpe=scala.runtime.NonLocalReturnControl
            new runtime.NonLocalReturnControl."<init>" // def <init>(key:
Object,value: Object): scala.runtime.NonLocalReturnControl in class
NonLocalReturnControl, tree.tpe=(key: Object, value:
Object)scala.runtime.NonLocalReturnControl
            // 2 arguments
            "nonLocalReturnKey1$1" // nonLocalReturnKey1$1: Object,
tree.tpe=Object
            Apply( // lazy val superVal(): String, tree.tpe=String
              $read$$iwC$$iwC$$iwC$$iwC$Sub.this."superVal" // lazy val
superVal(): String, tree.tpe=()String
              Nil
            )
          )
        )
      )
    )
  )
  DefDef( // def <init>(arg$outer: $line9.iwC): $line9.iwC$Sub
    <method>
    "<init>"
    []
    // 1 parameter list
    ValDef( // $outer: $line9.iwC
      <param> <triedcooking>
      "$outer"
      <tpt> // tree.tpe=$line9.iwC
      <empty>
    )
    <tpt> // tree.tpe=$line9.iwC$Sub
    Block( // tree.tpe=Unit
      // 2 statements
      If( // tree.tpe=Unit
        Apply( // final def eq(x$1: Object): Boolean in class Object,
tree.tpe=Boolean
          "$outer"."eq" // final def eq(x$1: Object): Boolean in class
Object, tree.tpe=(x$1: Object)Boolean
          null
        )
        Throw( // tree.tpe=Nothing
          Apply( // def <init>(): NullPointerException in class
NullPointerException, tree.tpe=NullPointerException
            new NullPointerException."<init>" // def <init>():
NullPointerException in class NullPointerException,
tree.tpe=()NullPointerException
            Nil
          )
        )
        Assign( // tree.tpe=Unit
          $read$$iwC$$iwC$$iwC$$iwC$Sub.this."$outer " // protected val
$outer: $line9.iwC, tree.tpe=$line9.iwC
          "$outer" // $outer: $line9.iwC, tree.tpe=$line9.iwC
        )
      )
      Apply( // def <init>(arg$outer: $line5.iwC): $line5.iwC$Super,
tree.tpe=$line5.iwC$Super
        $read$$iwC$$iwC$$iwC$$iwC$Sub.super."<init>" // def
<init>(arg$outer: $line5.iwC): $line5.iwC$Super, tree.tpe=(arg$outer:
$line5.iwC)$line5.iwC$Super
        Apply( // val $iw(): $line5.iwC, tree.tpe=$line5.iwC

$outer.$line9$$read$$iwC$$iwC$$iwC$$iwC$$$outer().$VAL1().$iw().$iw().$iw()."$iw"
// val $iw(): $line5.iwC, tree.tpe=()$line5.iwC
          Nil
        )
      )
      ()
    )
  )
)

== Expanded type of tree ==

TypeRef(TypeSymbol(class $read extends Serializable))

unhandled exception while transforming <console>
error: uncaught exception during compilation: java.lang.NullPointerException
java.lang.NullPointerException
at scala.reflect.internal.Trees$class.Select(Trees.scala:1066)
 at scala.reflect.internal.SymbolTable.Select(SymbolTable.scala:13)
at
scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$scala$tools$nsc$transform$Mixin$MixinTransformer$$dd$1$2.apply(Mixin.scala:908)
 at
scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$scala$tools$nsc$transform$Mixin$MixinTransformer$$dd$1$2.apply(Mixin.scala:904)
 at scala.reflect.internal.Trees$class.deriveDefDef(Trees.scala:1598)
at scala.reflect.internal.SymbolTable.deriveDefDef(SymbolTable.scala:13)
 at
scala.tools.nsc.transform.Mixin$MixinTransformer.scala$tools$nsc$transform$Mixin$MixinTransformer$$dd$1(Mixin.scala:904)
 at
scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$addCheckedGetters$1$1.apply(Mixin.scala:945)
at
scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$addCheckedGetters$1$1.apply(Mixin.scala:945)
 at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
 at scala.collection.immutable.List.foreach(List.scala:318)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
 at scala.collection.AbstractTraversable.map(Traversable.scala:105)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.addCheckedGetters$1(Mixin.scala:945)
 at
scala.tools.nsc.transform.Mixin$MixinTransformer.addNewDefs(Mixin.scala:1013)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.scala$tools$nsc$transform$Mixin$MixinTransformer$$postTransform(Mixin.scala:1147)
 at
scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$transform$1.apply(Mixin.scala:1261)
at
scala.tools.nsc.transform.Mixin$MixinTransformer$$anonfun$transform$1.apply(Mixin.scala:1261)
 at scala.reflect.internal.SymbolTable.atPhase(SymbolTable.scala:207)
at scala.reflect.internal.SymbolTable.afterPhase(SymbolTable.scala:216)
 at
scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:1261)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:471)
 at scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)
at
scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)
 at
scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)
at scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)
 at scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)
at scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)
 at scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)
at scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)
 at
scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:1258)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:471)
 at
scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)
at
scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)
 at scala.collection.immutable.List.loop$1(List.scala:170)
at scala.collection.immutable.List.mapConserve(List.scala:186)
 at scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)
at
scala.reflect.internal.Trees$$anonfun$itransform$7.apply(Trees.scala:1298)
 at
scala.reflect.internal.Trees$$anonfun$itransform$7.apply(Trees.scala:1298)
at scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)
 at scala.reflect.internal.Trees$class.itransform(Trees.scala:1297)
at scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)
 at scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)
at scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)
 at
scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:1258)
at
scala.tools.nsc.transform.Mixin$MixinTransformer.transform(Mixin.scala:471)
 at scala.tools.nsc.ast.Trees$Transformer.transformUnit(Trees.scala:227)
at scala.tools.nsc.transform.Transform$Phase.apply(Transform.scala:30)
 at scala.tools.nsc.Global$GlobalPhase.applyPhase(Global.scala:464)
at scala.tools.nsc.Global$GlobalPhase$$anonfun$run$1.apply(Global.scala:431)
 at
scala.tools.nsc.Global$GlobalPhase$$anonfun$run$1.apply(Global.scala:431)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
 at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.tools.nsc.Global$GlobalPhase.run(Global.scala:431)
 at scala.tools.nsc.Global$Run.compileUnitsInternal(Global.scala:1583)
at scala.tools.nsc.Global$Run.compileUnits(Global.scala:1557)
 at scala.tools.nsc.Global$Run.compileSources(Global.scala:1553)
at
org.apache.spark.repl.SparkIMain.compileSourcesKeepingRun(SparkIMain.scala:468)
 at
org.apache.spark.repl.SparkIMain$ReadEvalPrint.compileAndSaveRun(SparkIMain.scala:859)
at
org.apache.spark.repl.SparkIMain$ReadEvalPrint.compile(SparkIMain.scala:815)
 at
org.apache.spark.repl.SparkIMain$Request.compile$lzycompute(SparkIMain.scala:1009)
at org.apache.spark.repl.SparkIMain$Request.compile(SparkIMain.scala:1004)
 at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:644)
at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:609)
 at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:796)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
 at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
 at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
 at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
 at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
 at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
 at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
 at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
 at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814)
at
org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:841)
 at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:753)
at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:601)
 at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:608)
at org.apache.spark.repl.SparkILoop.loop(SparkILoop.scala:611)
 at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1$$anonfun$apply$mcV$sp$2.apply(SparkILoop.scala:621)
 at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1$$anonfun$apply$mcV$sp$2.apply(SparkILoop.scala:618)
 at
scala.reflect.io.Streamable$Chars$class.applyReader(Streamable.scala:104)
at scala.reflect.io.File.applyReader(File.scala:82)
 at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SparkILoop.scala:618)
 at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1.apply(SparkILoop.scala:618)
 at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1$$anonfun$apply$mcV$sp$1.apply(SparkILoop.scala:618)
at org.apache.spark.repl.SparkILoop.savingReplayStack(SparkILoop.scala:150)
 at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1.apply$mcV$sp(SparkILoop.scala:617)
at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1.apply(SparkILoop.scala:617)
 at
org.apache.spark.repl.SparkILoop$$anonfun$interpretAllFrom$1.apply(SparkILoop.scala:617)
at org.apache.spark.repl.SparkILoop.savingReader(SparkILoop.scala:155)
 at org.apache.spark.repl.SparkILoop.interpretAllFrom(SparkILoop.scala:616)
at
org.apache.spark.repl.SparkILoop$$anonfun$loadCommand$1.apply(SparkILoop.scala:681)
 at
org.apache.spark.repl.SparkILoop$$anonfun$loadCommand$1.apply(SparkILoop.scala:680)
at org.apache.spark.repl.SparkILoop.withFile(SparkILoop.scala:674)
 at org.apache.spark.repl.SparkILoop.loadCommand(SparkILoop.scala:680)
at
org.apache.spark.repl.SparkILoop$$anonfun$standardCommands$7.apply(SparkILoop.scala:294)
 at
org.apache.spark.repl.SparkILoop$$anonfun$standardCommands$7.apply(SparkILoop.scala:294)
at
scala.tools.nsc.interpreter.LoopCommands$LineCmd.apply(LoopCommands.scala:81)
 at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:748)
at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:601)
 at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:608)
at org.apache.spark.repl.SparkILoop.loop(SparkILoop.scala:611)
 at
org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply$mcZ$sp(SparkILoop.scala:936)
at
org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
 at
org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:884)
at
scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
 at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:884)
at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:983)
 at org.apache.spark.repl.Main$.main(Main.scala:31)
at org.apache.spark.repl.Main.main(Main.scala)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
 at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
 at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:256)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:54)
 at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)


Abandoning crashed session.

scala>

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message