[ https://issues.apache.org/jira/browse/SPARK-34452?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17489517#comment-17489517 ]
zhengruifeng commented on SPARK-34452: -------------------------------------- I can not reproduce this issue in 3.1.2, could you please provide a simple example? {code:java} scala> import org.apache.spark.ml.classification._ import org.apache.spark.ml.classification._ scala> scala> val df = spark.read.format("libsvm").load("/d0/Dev/Opensource/spark/data/mllib/sample_multiclass_classification_data.txt") 22/02/09 20:38:21 WARN LibSVMFileFormat: 'numFeatures' option not specified, determining the number of features by going though the input. If you know the number in advance, please specify it via 'numFeatures' option to avoid the extra scan. df: org.apache.spark.sql.DataFrame = [label: double, features: vector] scala> scala> val classifier = new GBTClassifier().setMaxIter(2) classifier: org.apache.spark.ml.classification.GBTClassifier = gbtc_e9ae5159908e scala> val ovr = new OneVsRest().setClassifier(classifier) ovr: org.apache.spark.ml.classification.OneVsRest = oneVsRest_466495ea9392 scala> val ovrm = ovr.fit(df) ovrm: org.apache.spark.ml.classification.OneVsRestModel = OneVsRestModel: uid=oneVsRest_466495ea9392, classifier=gbtc_e9ae5159908e, numClasses=3, numFeatures=4 scala> ovrm.transform(df).show 22/02/09 20:38:27 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS 22/02/09 20:38:27 WARN BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS +-----+--------------------+--------------------+----------+ |label| features| rawPrediction|prediction| +-----+--------------------+--------------------+----------+ | 1.0|(4,[0,1,2,3],[-0....|[-1.0476811688088...| 1.0| | 1.0|(4,[0,1,2,3],[-0....|[-1.0476811688088...| 1.0| | 1.0|(4,[0,1,2,3],[-0....|[-1.0476811688088...| 1.0| | 1.0|(4,[0,1,2,3],[-0....|[-1.0476811688088...| 1.0| | 0.0|(4,[0,1,2,3],[0.1...|[1.04768116880884...| 0.0| | 1.0|(4,[0,2,3],[-0.83...|[-1.0476811688088...| 1.0| | 2.0|(4,[0,1,2,3],[-1....|[-1.0476811688088...| 2.0| | 2.0|(4,[0,1,2,3],[-1....|[-1.0476811688088...| 2.0| | 1.0|(4,[0,1,2,3],[-0....|[-1.0476811688088...| 1.0| | 0.0|(4,[0,2,3],[0.611...|[1.04768116880884...| 0.0| | 0.0|(4,[0,1,2,3],[0.2...|[1.04768116880884...| 0.0| | 1.0|(4,[0,1,2,3],[-0....|[-1.0476811688088...| 1.0| | 1.0|(4,[0,1,2,3],[-0....|[-1.0476811688088...| 1.0| | 2.0|(4,[0,1,2,3],[-0....|[-1.0476811688088...| 2.0| | 2.0|(4,[0,1,2,3],[-0....|[-1.0476811688088...| 2.0| | 2.0|(4,[0,1,2,3],[-0....|[-1.0476811688088...| 2.0| | 1.0|(4,[0,2,3],[-0.94...|[-1.0476811688088...| 1.0| | 2.0|(4,[0,1,2,3],[-0....|[-1.0476811688088...| 2.0| | 0.0|(4,[0,1,2,3],[0.1...|[1.04768116880884...| 0.0| | 2.0|(4,[0,1,2,3],[-0....|[-1.0476811688088...| 2.0| +-----+--------------------+--------------------+----------+ only showing top 20 rows{code} > OneVsRest with GBTClassifier throws InternalCompilerException in 3.1.0 > ---------------------------------------------------------------------- > > Key: SPARK-34452 > URL: https://issues.apache.org/jira/browse/SPARK-34452 > Project: Spark > Issue Type: Bug > Components: ML > Affects Versions: 3.1.0 > Reporter: Danijel Zečević > Priority: Major > > It looks like a problem with user defined function from OneVsRestModel. > Log: > {code} > 2021-02-17 13:24:17.517 ERROR 711498 --- [818.0 (TID 818)] > o.a.s.s.c.e.codegen.CodeGenerator : failed to compile: > org.codehaus.janino.InternalCompilerException: Compiling "GeneratedClass": > Two non-abstract methods "public int scala.collection.TraversableOnce.size()" > have the same parameter types, declaring type and return type > org.codehaus.janino.InternalCompilerException: Compiling "GeneratedClass": > Two non-abstract methods "public int scala.collection.TraversableOnce.size()" > have the same parameter types, declaring type and return type > at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:361) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:234) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:446) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:313) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:235) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:204) > ~[janino-3.0.8.jar:na] > at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:82) > ~[commons-compiler-3.1.2.jar:na] > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1403) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1500) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1497) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.sparkproject.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at > org.sparkproject.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at > org.sparkproject.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at > org.sparkproject.guava.cache.LocalCache$Segment.get(LocalCache.java:2257) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at org.sparkproject.guava.cache.LocalCache.get(LocalCache.java:4000) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at > org.sparkproject.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at > org.sparkproject.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:1351) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:378) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:331) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:34) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:1277) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer.apply(ExpressionEncoder.scala:207) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer.apply(ExpressionEncoder.scala:196) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.ScalaUDF.$anonfun$catalystConverter$3(ScalaUDF.scala:128) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.project_subExpr_0$(Unknown > Source) ~[na:na] > at > org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown > Source) ~[na:na] > at > org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) > ~[spark-sql_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755) > ~[spark-sql_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118) > ~[spark-sql_2.12-3.1.0.jar:3.1.0] > at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) > ~[scala-library-2.12.10.jar:na] > at > org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1418) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1345) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1409) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1230) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.execution.SQLExecutionRDD.$anonfun$compute$1(SQLExecutionRDD.scala:52) > ~[spark-sql_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.internal.SQLConf$.withExistingConf(SQLConf.scala:122) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.execution.SQLExecutionRDD.compute(SQLExecutionRDD.scala:52) > ~[spark-sql_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.scheduler.Task.run(Task.scala:131) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) > ~[na:na] > at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) > ~[na:na] > at java.base/java.lang.Thread.run(Thread.java:834) ~[na:na] > Caused by: org.codehaus.janino.InternalCompilerException: Two non-abstract > methods "public int scala.collection.TraversableOnce.size()" have the same > parameter types, declaring type and return type > at > org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:9112) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:8888) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8770) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8672) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:4737) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.access$8300(UnitCompiler.java:212) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:4097) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:4070) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:4902) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4070) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5253) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2559) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.access$2700(UnitCompiler.java:212) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1482) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1466) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.Java$LocalVariableDeclarationStatement.accept(Java.java:3351) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1532) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:212) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1472) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1466) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.Java$Block.accept(Java.java:2756) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2455) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.access$1900(UnitCompiler.java:212) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1474) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1466) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.Java$IfStatement.accept(Java.java:2926) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3075) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1336) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1309) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:799) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:958) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.access$700(UnitCompiler.java:212) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:393) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:385) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1286) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:385) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1285) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:825) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:411) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:212) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:390) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:385) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1405) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:385) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:357) > ~[janino-3.0.8.jar:na] > ... 77 common frames omitted > 2021-02-17 13:24:17.522 INFO 711498 --- [818.0 (TID 818)] > o.a.s.s.c.e.codegen.CodeGenerator : > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificUnsafeProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificUnsafeProjection extends > org.apache.spark.sql.catalyst.expressions.UnsafeProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private int value_ExternalMapToCatalyst_key_lambda_variable_1; > /* 009 */ private double > value_ExternalMapToCatalyst_value_lambda_variable_2; > /* 010 */ private boolean globalIsNull_0; > /* 011 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] > mutableStateArray_0 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1]; > /* 012 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[] > mutableStateArray_1 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter[2]; > /* 013 */ > /* 014 */ public SpecificUnsafeProjection(Object[] references) { > /* 015 */ this.references = references; > /* 016 */ > /* 017 */ mutableStateArray_0[0] = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32); > /* 018 */ mutableStateArray_1[0] = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_0[0], > 4); > /* 019 */ mutableStateArray_1[1] = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(mutableStateArray_0[0], > 8); > /* 020 */ > /* 021 */ } > /* 022 */ > /* 023 */ public void initialize(int partitionIndex) { > /* 024 */ > /* 025 */ } > /* 026 */ > /* 027 */ // Scala.Function1 need this > /* 028 */ public java.lang.Object apply(java.lang.Object row) { > /* 029 */ return apply((InternalRow) row); > /* 030 */ } > /* 031 */ > /* 032 */ public UnsafeRow apply(InternalRow i) { > /* 033 */ mutableStateArray_0[0].reset(); > /* 034 */ > /* 035 */ > /* 036 */ mutableStateArray_0[0].zeroOutNullBytes(); > /* 037 */ > /* 038 */ MapData value_2 = ExternalMapToCatalyst_0(i); > /* 039 */ if (globalIsNull_0) { > /* 040 */ mutableStateArray_0[0].setNullAt(0); > /* 041 */ } else { > /* 042 */ final MapData tmpInput_0 = value_2; > /* 043 */ if (tmpInput_0 instanceof UnsafeMapData) { > /* 044 */ mutableStateArray_0[0].write(0, (UnsafeMapData) tmpInput_0); > /* 045 */ } else { > /* 046 */ // Remember the current cursor so that we can calculate how > many bytes are > /* 047 */ // written later. > /* 048 */ final int previousCursor_0 = > mutableStateArray_0[0].cursor(); > /* 049 */ > /* 050 */ // preserve 8 bytes to write the key array numBytes later. > /* 051 */ mutableStateArray_0[0].grow(8); > /* 052 */ mutableStateArray_0[0].increaseCursor(8); > /* 053 */ > /* 054 */ // Remember the current cursor so that we can write > numBytes of key array later. > /* 055 */ final int tmpCursor_0 = mutableStateArray_0[0].cursor(); > /* 056 */ > /* 057 */ > /* 058 */ final ArrayData tmpInput_1 = tmpInput_0.keyArray(); > /* 059 */ if (tmpInput_1 instanceof UnsafeArrayData) { > /* 060 */ mutableStateArray_0[0].write((UnsafeArrayData) > tmpInput_1); > /* 061 */ } else { > /* 062 */ final int numElements_0 = tmpInput_1.numElements(); > /* 063 */ mutableStateArray_1[0].initialize(numElements_0); > /* 064 */ > /* 065 */ for (int index_1 = 0; index_1 < numElements_0; index_1++) > { > /* 066 */ mutableStateArray_1[0].write(index_1, > tmpInput_1.getInt(index_1)); > /* 067 */ } > /* 068 */ } > /* 069 */ > /* 070 */ > /* 071 */ // Write the numBytes of key array into the first 8 bytes. > /* 072 */ Platform.putLong( > /* 073 */ mutableStateArray_0[0].getBuffer(), > /* 074 */ tmpCursor_0 - 8, > /* 075 */ mutableStateArray_0[0].cursor() - tmpCursor_0); > /* 076 */ > /* 077 */ > /* 078 */ final ArrayData tmpInput_2 = tmpInput_0.valueArray(); > /* 079 */ if (tmpInput_2 instanceof UnsafeArrayData) { > /* 080 */ mutableStateArray_0[0].write((UnsafeArrayData) > tmpInput_2); > /* 081 */ } else { > /* 082 */ final int numElements_1 = tmpInput_2.numElements(); > /* 083 */ mutableStateArray_1[1].initialize(numElements_1); > /* 084 */ > /* 085 */ for (int index_2 = 0; index_2 < numElements_1; index_2++) > { > /* 086 */ mutableStateArray_1[1].write(index_2, > tmpInput_2.getDouble(index_2)); > /* 087 */ } > /* 088 */ } > /* 089 */ > /* 090 */ > mutableStateArray_0[0].setOffsetAndSizeFromPreviousCursor(0, > previousCursor_0); > /* 091 */ } > /* 092 */ } > /* 093 */ return (mutableStateArray_0[0].getRow()); > /* 094 */ } > /* 095 */ > /* 096 */ > /* 097 */ private MapData ExternalMapToCatalyst_0(InternalRow i) { > /* 098 */ boolean isNull_1 = i.isNullAt(0); > /* 099 */ scala.collection.immutable.Map value_1 = isNull_1 ? > /* 100 */ null : ((scala.collection.immutable.Map)i.get(0, null)); > /* 101 */ MapData value_0 = null; > /* 102 */ if (!isNull_1) { > /* 103 */ final int length_0 = value_1.size(); > /* 104 */ final Object[] convertedKeys_0 = new Object[length_0]; > /* 105 */ final Object[] convertedValues_0 = new Object[length_0]; > /* 106 */ int index_0 = 0; > /* 107 */ final scala.collection.Iterator entries_0 = > value_1.iterator(); > /* 108 */ while(entries_0.hasNext()) { > /* 109 */ > /* 110 */ final scala.Tuple2 entry_0 = (scala.Tuple2) > entries_0.next(); > /* 111 */ value_ExternalMapToCatalyst_key_lambda_variable_1 = > (Integer) entry_0._1(); > /* 112 */ value_ExternalMapToCatalyst_value_lambda_variable_2 = > (Double) entry_0._2(); > /* 113 */ > /* 114 */ > /* 115 */ > /* 116 */ > /* 117 */ > /* 118 */ if (false) { > /* 119 */ throw new RuntimeException("Cannot use null as map key!"); > /* 120 */ } else { > /* 121 */ convertedKeys_0[index_0] = (Integer) > value_ExternalMapToCatalyst_key_lambda_variable_1; > /* 122 */ } > /* 123 */ > /* 124 */ > /* 125 */ if (false) { > /* 126 */ convertedValues_0[index_0] = null; > /* 127 */ } else { > /* 128 */ convertedValues_0[index_0] = (Double) > value_ExternalMapToCatalyst_value_lambda_variable_2; > /* 129 */ } > /* 130 */ > /* 131 */ index_0++; > /* 132 */ } > /* 133 */ > /* 134 */ value_0 = new > org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new > org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys_0), new > org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues_0)); > /* 135 */ } > /* 136 */ globalIsNull_0 = isNull_1; > /* 137 */ return value_0; > /* 138 */ } > /* 139 */ > /* 140 */ } > 2021-02-17 13:24:17.523 WARN 711498 --- [818.0 (TID 818)] > org.apache.spark.storage.BlockManager : Putting block rdd_1866_0 failed > due to exception org.apache.spark.SparkException: Failed to execute user > defined function(OneVsRestModel$$Lambda$5255/0x0000000801a89440: () => > map<int,double>). > 2021-02-17 13:24:17.524 WARN 711498 --- [818.0 (TID 818)] > org.apache.spark.storage.BlockManager : Block rdd_1866_0 could not be > removed as it was not found on disk or in memory > 2021-02-17 13:24:17.527 ERROR 711498 --- [818.0 (TID 818)] > org.apache.spark.executor.Executor : Exception in task 0.0 in stage > 818.0 (TID 818) > org.apache.spark.SparkException: Failed to execute user defined > function(OneVsRestModel$$Lambda$5255/0x0000000801a89440: () => > map<int,double>) > at > org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.project_subExpr_0$(Unknown > Source) ~[na:na] > at > org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown > Source) ~[na:na] > at > org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) > ~[spark-sql_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755) > ~[spark-sql_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118) > ~[spark-sql_2.12-3.1.0.jar:3.1.0] > at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) > ~[scala-library-2.12.10.jar:na] > at > org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1418) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1345) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1409) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1230) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.execution.SQLExecutionRDD.$anonfun$compute$1(SQLExecutionRDD.scala:52) > ~[spark-sql_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.internal.SQLConf$.withExistingConf(SQLConf.scala:122) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.execution.SQLExecutionRDD.compute(SQLExecutionRDD.scala:52) > ~[spark-sql_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.scheduler.Task.run(Task.scala:131) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500) > ~[spark-core_2.12-3.1.0.jar:3.1.0] > at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) > ~[na:na] > at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) > ~[na:na] > at java.base/java.lang.Thread.run(Thread.java:834) ~[na:na] > Caused by: java.lang.RuntimeException: Error while encoding: > org.codehaus.janino.InternalCompilerException: failed to compile: > org.codehaus.janino.InternalCompilerException: Compiling "GeneratedClass": > Two non-abstract methods "public int scala.collection.TraversableOnce.size()" > have the same parameter types, declaring type and return type > externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, IntegerType, > false, -1), lambdavariable(ExternalMapToCatalyst_key, IntegerType, false, > -1), lambdavariable(ExternalMapToCatalyst_value, DoubleType, false, -2), > lambdavariable(ExternalMapToCatalyst_value, DoubleType, false, -2), input[0, > scala.collection.immutable.Map, true]) AS value#497 > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer.apply(ExpressionEncoder.scala:214) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer.apply(ExpressionEncoder.scala:196) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.ScalaUDF.$anonfun$catalystConverter$3(ScalaUDF.scala:128) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > ... 53 common frames omitted > Caused by: org.codehaus.janino.InternalCompilerException: failed to compile: > org.codehaus.janino.InternalCompilerException: Compiling "GeneratedClass": > Two non-abstract methods "public int scala.collection.TraversableOnce.size()" > have the same parameter types, declaring type and return type > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1410) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1500) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1497) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.sparkproject.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at > org.sparkproject.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at > org.sparkproject.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at > org.sparkproject.guava.cache.LocalCache$Segment.get(LocalCache.java:2257) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at org.sparkproject.guava.cache.LocalCache.get(LocalCache.java:4000) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at > org.sparkproject.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at > org.sparkproject.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874) > ~[spark-network-common_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:1351) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:378) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:331) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:34) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:1277) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer.apply(ExpressionEncoder.scala:207) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > ... 55 common frames omitted > Caused by: org.codehaus.janino.InternalCompilerException: Compiling > "GeneratedClass": Two non-abstract methods "public int > scala.collection.TraversableOnce.size()" have the same parameter types, > declaring type and return type > at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:361) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:234) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:446) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:313) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:235) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:204) > ~[janino-3.0.8.jar:na] > at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:82) > ~[commons-compiler-3.1.2.jar:na] > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1403) > ~[spark-catalyst_2.12-3.1.0.jar:3.1.0] > ... 70 common frames omitted > Caused by: org.codehaus.janino.InternalCompilerException: Two non-abstract > methods "public int scala.collection.TraversableOnce.size()" have the same > parameter types, declaring type and return type > at > org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:9112) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:8888) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8770) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8672) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:4737) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.access$8300(UnitCompiler.java:212) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:4097) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:4070) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:4902) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4070) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5253) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2559) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.access$2700(UnitCompiler.java:212) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1482) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1466) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.Java$LocalVariableDeclarationStatement.accept(Java.java:3351) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1532) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:212) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1472) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1466) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.Java$Block.accept(Java.java:2756) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2455) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.access$1900(UnitCompiler.java:212) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1474) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1466) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.Java$IfStatement.accept(Java.java:2926) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3075) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1336) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1309) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:799) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:958) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.access$700(UnitCompiler.java:212) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:393) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:385) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1286) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:385) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1285) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:825) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:411) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:212) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:390) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:385) > ~[janino-3.0.8.jar:na] > at > org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1405) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:385) > ~[janino-3.0.8.jar:na] > at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:357) > ~[janino-3.0.8.jar:na] > ... 77 common frames omitted > 2021-02-17 13:24:17.540 WARN 711498 --- [result-getter-2] > o.apache.spark.scheduler.TaskSetManager : Lost task 0.0 in stage 818.0 (TID > 818) (10.99.10.158 executor driver): org.apache.spark.SparkException: Failed > to execute user defined > function(OneVsRestModel$$Lambda$5255/0x0000000801a89440: () => > map<int,double>) > at > org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.project_subExpr_0$(Unknown > Source) > at > org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown > Source) > at > org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) > at > org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755) > at > org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118) > at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) > at > org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221) > at > org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299) > at > org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1418) > at > org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1345) > at > org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1409) > at > org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1230) > at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.sql.execution.SQLExecutionRDD.$anonfun$compute$1(SQLExecutionRDD.scala:52) > at > org.apache.spark.sql.internal.SQLConf$.withExistingConf(SQLConf.scala:122) > at > org.apache.spark.sql.execution.SQLExecutionRDD.compute(SQLExecutionRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) > at > org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) > at > org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) > at org.apache.spark.scheduler.Task.run(Task.scala:131) > at > org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497) > at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439) > at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500) > at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) > at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) > at java.base/java.lang.Thread.run(Thread.java:834) > Caused by: java.lang.RuntimeException: Error while encoding: > org.codehaus.janino.InternalCompilerException: failed to compile: > org.codehaus.janino.InternalCompilerException: Compiling "GeneratedClass": > Two non-abstract methods "public int scala.collection.TraversableOnce.size()" > have the same parameter types, declaring type and return type > externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, IntegerType, > false, -1), lambdavariable(ExternalMapToCatalyst_key, IntegerType, false, > -1), lambdavariable(ExternalMapToCatalyst_value, DoubleType, false, -2), > lambdavariable(ExternalMapToCatalyst_value, DoubleType, false, -2), input[0, > scala.collection.immutable.Map, true]) AS value#497 > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer.apply(ExpressionEncoder.scala:214) > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer.apply(ExpressionEncoder.scala:196) > at > org.apache.spark.sql.catalyst.expressions.ScalaUDF.$anonfun$catalystConverter$3(ScalaUDF.scala:128) > ... 53 more > Caused by: org.codehaus.janino.InternalCompilerException: failed to compile: > org.codehaus.janino.InternalCompilerException: Compiling "GeneratedClass": > Two non-abstract methods "public int scala.collection.TraversableOnce.size()" > have the same parameter types, declaring type and return type > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1410) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1500) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1497) > at > org.sparkproject.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599) > at > org.sparkproject.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379) > at > org.sparkproject.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342) > at > org.sparkproject.guava.cache.LocalCache$Segment.get(LocalCache.java:2257) > at org.sparkproject.guava.cache.LocalCache.get(LocalCache.java:4000) > at > org.sparkproject.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004) > at > org.sparkproject.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:1351) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:378) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:331) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:34) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:1277) > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer.apply(ExpressionEncoder.scala:207) > ... 55 more > Caused by: org.codehaus.janino.InternalCompilerException: Compiling > "GeneratedClass": Two non-abstract methods "public int > scala.collection.TraversableOnce.size()" have the same parameter types, > declaring type and return type > at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:361) > at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:234) > at > org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:446) > at > org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:313) > at > org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:235) > at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:204) > at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:82) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1403) > ... 70 more > Caused by: org.codehaus.janino.InternalCompilerException: Two non-abstract > methods "public int scala.collection.TraversableOnce.size()" have the same > parameter types, declaring type and return type > at > org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:9112) > at > org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:8888) > at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8770) > at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8672) > at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:4737) > at org.codehaus.janino.UnitCompiler.access$8300(UnitCompiler.java:212) > at > org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:4097) > at > org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:4070) > at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:4902) > at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4070) > at > org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5253) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2559) > at org.codehaus.janino.UnitCompiler.access$2700(UnitCompiler.java:212) > at > org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1482) > at > org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1466) > at > org.codehaus.janino.Java$LocalVariableDeclarationStatement.accept(Java.java:3351) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466) > at > org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1532) > at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:212) > at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1472) > at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1466) > at org.codehaus.janino.Java$Block.accept(Java.java:2756) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2455) > at org.codehaus.janino.UnitCompiler.access$1900(UnitCompiler.java:212) > at > org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1474) > at > org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1466) > at org.codehaus.janino.Java$IfStatement.accept(Java.java:2926) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466) > at > org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3075) > at > org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1336) > at > org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1309) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:799) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:958) > at org.codehaus.janino.UnitCompiler.access$700(UnitCompiler.java:212) > at > org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:393) > at > org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:385) > at > org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1286) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:385) > at > org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1285) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:825) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:411) > at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:212) > at > org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:390) > at > org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:385) > at > org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1405) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:385) > at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:357) > ... 77 more > 2021-02-17 13:24:17.541 ERROR 711498 --- [result-getter-2] > o.apache.spark.scheduler.TaskSetManager : Task 0 in stage 818.0 failed 1 > times; aborting job > 2021-02-17 13:24:17.541 INFO 711498 --- [result-getter-2] > o.a.spark.scheduler.TaskSchedulerImpl : Removed TaskSet 818.0, whose tasks > have all completed, from pool > 2021-02-17 13:24:17.542 INFO 711498 --- [uler-event-loop] > o.a.spark.scheduler.TaskSchedulerImpl : Cancelling stage 818 > 2021-02-17 13:24:17.542 INFO 711498 --- [uler-event-loop] > o.a.spark.scheduler.TaskSchedulerImpl : Killing all running tasks in stage > 818: Stage cancelled > 2021-02-17 13:24:17.543 INFO 711498 --- [uler-event-loop] > org.apache.spark.scheduler.DAGScheduler : ShuffleMapStage 818 (map at > MulticlassMetrics.scala:52) failed in 0.160 s due to Job aborted due to stage > failure: Task 0 in stage 818.0 failed 1 times, most recent failure: Lost task > 0.0 in stage 818.0 (TID 818) (10.99.10.158 executor driver): > org.apache.spark.SparkException: Failed to execute user defined > function(OneVsRestModel$$Lambda$5255/0x0000000801a89440: () => > map<int,double>) > at > org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.project_subExpr_0$(Unknown > Source) > at > org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown > Source) > at > org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) > at > org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755) > at > org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118) > at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458) > at > org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221) > at > org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299) > at > org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1418) > at > org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1345) > at > org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1409) > at > org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1230) > at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:335) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.sql.execution.SQLExecutionRDD.$anonfun$compute$1(SQLExecutionRDD.scala:52) > at > org.apache.spark.sql.internal.SQLConf$.withExistingConf(SQLConf.scala:122) > at > org.apache.spark.sql.execution.SQLExecutionRDD.compute(SQLExecutionRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) > at > org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) > at > org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99) > at > org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52) > at org.apache.spark.scheduler.Task.run(Task.scala:131) > at > org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497) > at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439) > at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500) > at > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) > at > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) > at java.base/java.lang.Thread.run(Thread.java:834) > Caused by: java.lang.RuntimeException: Error while encoding: > org.codehaus.janino.InternalCompilerException: failed to compile: > org.codehaus.janino.InternalCompilerException: Compiling "GeneratedClass": > Two non-abstract methods "public int scala.collection.TraversableOnce.size()" > have the same parameter types, declaring type and return type > externalmaptocatalyst(lambdavariable(ExternalMapToCatalyst_key, IntegerType, > false, -1), lambdavariable(ExternalMapToCatalyst_key, IntegerType, false, > -1), lambdavariable(ExternalMapToCatalyst_value, DoubleType, false, -2), > lambdavariable(ExternalMapToCatalyst_value, DoubleType, false, -2), input[0, > scala.collection.immutable.Map, true]) AS value#497 > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer.apply(ExpressionEncoder.scala:214) > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer.apply(ExpressionEncoder.scala:196) > at > org.apache.spark.sql.catalyst.expressions.ScalaUDF.$anonfun$catalystConverter$3(ScalaUDF.scala:128) > ... 53 more > Caused by: org.codehaus.janino.InternalCompilerException: failed to compile: > org.codehaus.janino.InternalCompilerException: Compiling "GeneratedClass": > Two non-abstract methods "public int scala.collection.TraversableOnce.size()" > have the same parameter types, declaring type and return type > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1410) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1500) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1497) > at > org.sparkproject.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599) > at > org.sparkproject.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379) > at > org.sparkproject.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342) > at > org.sparkproject.guava.cache.LocalCache$Segment.get(LocalCache.java:2257) > at org.sparkproject.guava.cache.LocalCache.get(LocalCache.java:4000) > at > org.sparkproject.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004) > at > org.sparkproject.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:1351) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:378) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:331) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:34) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:1277) > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer.apply(ExpressionEncoder.scala:207) > ... 55 more > Caused by: org.codehaus.janino.InternalCompilerException: Compiling > "GeneratedClass": Two non-abstract methods "public int > scala.collection.TraversableOnce.size()" have the same parameter types, > declaring type and return type > at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:361) > at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:234) > at > org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:446) > at > org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:313) > at > org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:235) > at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:204) > at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:82) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1403) > ... 70 more > Caused by: org.codehaus.janino.InternalCompilerException: Two non-abstract > methods "public int scala.collection.TraversableOnce.size()" have the same > parameter types, declaring type and return type > at > org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:9112) > at > org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:8888) > at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8770) > at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:8672) > at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:4737) > at org.codehaus.janino.UnitCompiler.access$8300(UnitCompiler.java:212) > at > org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:4097) > at > org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:4070) > at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:4902) > at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4070) > at > org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5253) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2559) > at org.codehaus.janino.UnitCompiler.access$2700(UnitCompiler.java:212) > at > org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1482) > at > org.codehaus.janino.UnitCompiler$6.visitLocalVariableDeclarationStatement(UnitCompiler.java:1466) > at > org.codehaus.janino.Java$LocalVariableDeclarationStatement.accept(Java.java:3351) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466) > at > org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1532) > at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:212) > at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1472) > at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1466) > at org.codehaus.janino.Java$Block.accept(Java.java:2756) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2455) > at org.codehaus.janino.UnitCompiler.access$1900(UnitCompiler.java:212) > at > org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1474) > at > org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1466) > at org.codehaus.janino.Java$IfStatement.accept(Java.java:2926) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1466) > at > org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1546) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3075) > at > org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1336) > at > org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1309) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:799) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:958) > at org.codehaus.janino.UnitCompiler.access$700(UnitCompiler.java:212) > at > org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:393) > at > org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:385) > at > org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1286) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:385) > at > org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1285) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:825) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:411) > at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:212) > at > org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:390) > at > org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:385) > at > org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1405) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:385) > at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:357) > ... 77 more > {code} -- This message was sent by Atlassian Jira (v8.20.1#820001) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org