Hi
I am trying to get
* Apache Spark 3.0
* Spark Cassandra Connector 3.0.0
To work together from Zeppelin, but are running into issues that seems to be
related to Scala 2.12
Like for example NoSuchMethodError, see below
Has anyone been able to get this to work ?
There are files in the Zeppelin project that kind of indicates that this should
indeed work, but I can’t figure out what I am doing wrong here
What I have done is
1. Using Zeppelin Docker
2. Installed Spark 3.0 inside of Docker
3. Set SPARK_HOME=/opt/spark (pointing to the newly installed Spark 3)
4. In Zeppelin I have added the dependency for Spark
com.datastax.spark:spark-cassandra-connector_2.12:3.0.0
I can see that inside of the directory :
/opt/zeppelin/local-repo/spark
There are a mix of 2.11 and 2.12 libraries
scala-library-2.11.12.jar
scala-logging_2.11-3.5.0.jar
scala-reflect-2.11.12.jar
I suspect that there is a Scala 2.11 vs 2.12 issue here, but I am at the moment
unable to figure out how to fix it
So any help would be much appreciated 😊
-Tobias
zeppelin | INFO [2021-07-01 08:34:56,064] ({SchedulerFactory60}
RemoteInterpreter.java[pushAngularObjectRegistryToRemote]:393) - Push local
angular object registry from ZeppelinServer to remote interpreter group
spark-shared_process
zeppelin | INFO [2021-07-01 08:34:56,146]
({JobStatusPoller-paragraph_1625079989375_1248319040}
NotebookServer.java[onStatusChange]:1931) - Job
paragraph_1625079989375_1248319040 starts to RUNNING
zeppelin | INFO [2021-07-01 08:34:56,147]
({JobStatusPoller-paragraph_1625079989375_1248319040}
VFSNotebookRepo.java[save]:144) - Saving note 2GAM4MGRX to T3_A_2GAM4MGRX.zpln
zeppelin | WARN [2021-07-01 08:35:03,829] ({SchedulerFactory60}
NotebookServer.java[onStatusChange]:1928) - Job
paragraph_1625079989375_1248319040 is finished, status: ERROR, exception: null,
result: %text org.apache.zeppelin.interpreter.InterpreterException:
java.lang.BootstrapMethodError: java.lang.NoSuchMethodError:
scala.collection.TraversableOnce.$anonfun$count$1$adapted(Lscala/Function1;Lscala/runtime/IntRef;Ljava/lang/Object;)Ljava/lang/Object;
zeppelin | at
org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:76)
zeppelin | at
org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:836)
zeppelin | at
org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:744)
zeppelin | at org.apache.zeppelin.scheduler.Job.run(Job.java:172)
zeppelin | at
org.apache.zeppelin.scheduler.AbstractScheduler.runJob(AbstractScheduler.java:132)
zeppelin | at
org.apache.zeppelin.scheduler.FIFOScheduler.lambda$runJobInScheduler$0(FIFOScheduler.java:42)
zeppelin | at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
zeppelin | at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
zeppelin | at java.lang.Thread.run(Thread.java:748)
zeppelin | Caused by: java.lang.BootstrapMethodError:
java.lang.NoSuchMethodError:
scala.collection.TraversableOnce.$anonfun$count$1$adapted(Lscala/Function1;Lscala/runtime/IntRef;Ljava/lang/Object;)Ljava/lang/Object;
zeppelin | at
scala.tools.nsc.interpreter.Naming.unmangle(Naming.scala:30)
zeppelin | at
scala.tools.nsc.interpreter.Naming.unmangle$(Naming.scala:24)
zeppelin | at
scala.tools.nsc.interpreter.IMain$naming$.unmangle(IMain.scala:188)
zeppelin | at
scala.tools.nsc.interpreter.IMain$ReplStrippingWriter.stripImpl(IMain.scala:1340)
zeppelin | at
scala.tools.nsc.interpreter.IMain$StrippingWriter.strip(IMain.scala:1314)
zeppelin | at
scala.tools.nsc.interpreter.IMain$StrippingWriter.strip$(IMain.scala:1314)
zeppelin | at
scala.tools.nsc.interpreter.IMain$StrippingTruncatingWriter.strip(IMain.scala:1325)
zeppelin | at
scala.tools.nsc.interpreter.IMain$StrippingTruncatingWriter.clean(IMain.scala:1331)
zeppelin | at
scala.tools.nsc.interpreter.IMain$StrippingTruncatingWriter.write(IMain.scala:1332)
zeppelin | at java.io.PrintWriter.print(PrintWriter.java:603)
zeppelin | at java.io.PrintWriter.println(PrintWriter.java:739)
zeppelin | at
scala.tools.nsc.reporters.ConsoleReporter.printMessage(ConsoleReporter.scala:49)
zeppelin | at
scala.tools.nsc.interpreter.ReplReporter.printMessage(ReplReporter.scala:93)
zeppelin | at
scala.tools.nsc.interpreter.IMain.$anonfun$interpret$1(IMain.scala:582)
zeppelin | at
scala.reflect.internal.util.ScalaClassLoader.asContext(ScalaClassLoader.scala:41)
zeppelin | at
scala.reflect.internal.util.ScalaClassLoader.asContext$(ScalaClassLoader.scala:37)
zeppelin | at
scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:41)
zeppelin | at
scala.tools.nsc.interpreter.IMain.loadAndRunReq$1(IMain.scala:573)
zeppelin | at
scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:600)
zeppelin | at
scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:570)
zeppelin | at
org.apache.zeppelin.spark.SparkScala212Interpreter.scalaInterpret(SparkScala212Interpreter.scala:120)
zeppelin | at
org.apache.zeppelin.spark.BaseSparkScalaInterpreter.spark2CreateContext(BaseSparkScalaInterpreter.scala:325)
zeppelin | at
org.apache.zeppelin.spark.BaseSparkScalaInterpreter.createSparkContext(BaseSparkScalaInterpreter.scala:230)
zeppelin | at
org.apache.zeppelin.spark.SparkScala212Interpreter.open(SparkScala212Interpreter.scala:90)
zeppelin | at
org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:106)
zeppelin | at
org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:70)
zeppelin | ... 8 more
zeppelin | Caused by: java.lang.NoSuchMethodError:
scala.collection.TraversableOnce.$anonfun$count$1$adapted(Lscala/Function1;Lscala/runtime/IntRef;Ljava/lang/Object;)Ljava/lang/Object;
zeppelin | at java.lang.invoke.MethodHandleNatives.resolve(Native
Method)
zeppelin | at
java.lang.invoke.MemberName$Factory.resolve(MemberName.java:975)
zeppelin | at
java.lang.invoke.MemberName$Factory.resolveOrFail(MemberName.java:1000)
zeppelin | at
java.lang.invoke.MethodHandles$Lookup.resolveOrFail(MethodHandles.java:1394)
zeppelin | at
java.lang.invoke.MethodHandles$Lookup.linkMethodHandleConstant(MethodHandles.java:1750)
zeppelin | at
java.lang.invoke.MethodHandleNatives.linkMethodHandleConstant(MethodHandleNatives.java:477)
zeppelin | ... 34 more
zeppelin |
zeppelin | INFO [2021-07-01 08:35:03,829] ({SchedulerFactory60}
VFSNotebookRepo.java[save]:144) - Saving note 2GAM4MGRX to T3_A_2GAM4MGRX.zpln
zeppelin | INFO [2021-07-01 08:35:03,840] ({SchedulerFactory60}
AbstractScheduler.java[runJob]:154) - Job paragraph_1625079989375_1248319040
finished by scheduler RemoteInterpreter-spark-shared_process-shared_session
with status ERROR
