[ https://issues.apache.org/jira/browse/SPARK-44706?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Juan Carlos Blanco Martinez updated SPARK-44706: ------------------------------------------------ Description: Hi, When running the following code in Databricks' notebook: {code:java} import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} case class DataRow(field1: String) val sparkSession = SparkSession.builder.getOrCreate() import sparkSession.implicits._ val udf1 = udf((x: String) => { val testData = Seq(DataRow("test1"), DataRow("test2")).toDF("test") // This is failing at runtime 3 }) val df1 = Seq(DataRow("test1"), DataRow("test2")).toDF("test").withColumn("udf", udf1($"test")) // This is working display(df1) {code} I get a ScalaReflectionException at runtime: {code:java} ScalaReflectionException: class $linedb3da7b2933d4b63a62b3d2a21c675f2141.$read in JavaMirror with com.databricks.backend.daemon.driver.DriverLocal$DriverLocalClassLoader@717115ad of type class com.databricks.backend.daemon.driver.DriverLocal$DriverLocalClassLoader with classpath [] and parent being com.databricks.backend.daemon.driver.ClassLoaders$ReplWrappingClassLoader@1670897 of type class com.databricks.backend.daemon.driver.ClassLoaders$ReplWrappingClassLoader with classpath [<unknown>] and parent being com.databricks.backend.daemon.driver.ClassLoaders$LibraryClassLoader@1a42da0a of type class com.databricks.backend.daemon.driver.ClassLoaders$LibraryClassLoader with classpath [file:/local_disk0/tmp/repl/spark-4071811259476162981-8e526ae9-25fb-4545-8d3f-963a8661cd2b/] and parent being sun.misc.Launcher$AppClassLoader@43ee72e6 of type class sun.misc.Launcher$AppClassLoader with classpath [...] not found. at scala.reflect.internal.Mirrors$RootsBase.staticClass(Mirrors.scala:141) at scala.reflect.internal.Mirrors$RootsBase.staticClass(Mirrors.scala:29) at $linedb3da7b2933d4b63a62b3d2a21c675f2196.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$$$c35395a9233a7197629c985e87dce75$$$$w$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$typecreator6$1.apply(command-2013905963200886:11) at scala.reflect.api.TypeTags$WeakTypeTagImpl.tpe$lzycompute(TypeTags.scala:237) at scala.reflect.api.TypeTags$WeakTypeTagImpl.tpe(TypeTags.scala:237) at org.apache.spark.sql.catalyst.ScalaReflection$.encoderFor(ScalaReflection.scala:848) at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:55) at org.apache.spark.sql.Encoders$.product(Encoders.scala:312) at org.apache.spark.sql.LowPrioritySQLImplicits.newProductEncoder(SQLImplicits.scala:302) at org.apache.spark.sql.LowPrioritySQLImplicits.newProductEncoder$(SQLImplicits.scala:302) at org.apache.spark.sql.SQLImplicits.newProductEncoder(SQLImplicits.scala:34) at $linedb3da7b2933d4b63a62b3d2a21c675f2196.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$$$c35395a9233a7197629c985e87dce75$$$$w$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.$anonfun$udf1$1(command-2013905963200886:11) {code} Declaring the case class within the UDF: {code:java} import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} val sparkSession = SparkSession.builder.getOrCreate() import sparkSession.implicits._ val udf1 = udf((x: String) => { case class DataRow(field1: String) val testData = Seq(DataRow("test1"), DataRow("test2")).toDF("test") // This is failing at runtime 3 }) val df1 = Seq(DataRow("test1"), DataRow("test2")).toDF("test").withColumn("udf", udf1($"test")) // This is working display(df1) {code} I get a different error {code:java} error: value toDF is not a member of Seq[DataRow] val testData = Seq(DataRow("test1"), DataRow("test2")).toDF("test"){code} Thanks. was: Hi, When running the following code in Databricks' notebook: {code:java} import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} case class DataRow(field1: String) val sparkSession = SparkSession.builder.getOrCreate() import sparkSession.implicits._ val udf1 = udf((x: String) => { val testData = Seq(DataRow("test1"), DataRow("test2")).toDF("test") // This is failing at runtime 3 }) val df1 = Seq(DataRow("test1"), DataRow("test2")).toDF("test").withColumn("udf", udf1($"test")) // This is working display(df1) {code} I get a ScalaReflectionException at runtime: {code:java} ScalaReflectionException: class $linedb3da7b2933d4b63a62b3d2a21c675f2141.$read in JavaMirror with com.databricks.backend.daemon.driver.DriverLocal$DriverLocalClassLoader@717115ad of type class com.databricks.backend.daemon.driver.DriverLocal$DriverLocalClassLoader with classpath [] and parent being com.databricks.backend.daemon.driver.ClassLoaders$ReplWrappingClassLoader@1670897 of type class com.databricks.backend.daemon.driver.ClassLoaders$ReplWrappingClassLoader with classpath [<unknown>] and parent being com.databricks.backend.daemon.driver.ClassLoaders$LibraryClassLoader@1a42da0a of type class com.databricks.backend.daemon.driver.ClassLoaders$LibraryClassLoader with classpath [file:/local_disk0/tmp/repl/spark-4071811259476162981-8e526ae9-25fb-4545-8d3f-963a8661cd2b/] and parent being sun.misc.Launcher$AppClassLoader@43ee72e6 of type class sun.misc.Launcher$AppClassLoader with classpath [...] not found. at scala.reflect.internal.Mirrors$RootsBase.staticClass(Mirrors.scala:141) at scala.reflect.internal.Mirrors$RootsBase.staticClass(Mirrors.scala:29) at $linedb3da7b2933d4b63a62b3d2a21c675f2196.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$$$c35395a9233a7197629c985e87dce75$$$$w$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$typecreator6$1.apply(command-2013905963200886:11) at scala.reflect.api.TypeTags$WeakTypeTagImpl.tpe$lzycompute(TypeTags.scala:237) at scala.reflect.api.TypeTags$WeakTypeTagImpl.tpe(TypeTags.scala:237) at org.apache.spark.sql.catalyst.ScalaReflection$.encoderFor(ScalaReflection.scala:848) at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:55) at org.apache.spark.sql.Encoders$.product(Encoders.scala:312) at org.apache.spark.sql.LowPrioritySQLImplicits.newProductEncoder(SQLImplicits.scala:302) at org.apache.spark.sql.LowPrioritySQLImplicits.newProductEncoder$(SQLImplicits.scala:302) at org.apache.spark.sql.SQLImplicits.newProductEncoder(SQLImplicits.scala:34) at $linedb3da7b2933d4b63a62b3d2a21c675f2196.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$$$c35395a9233a7197629c985e87dce75$$$$w$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.$anonfun$udf1$1(command-2013905963200886:11) {code} Declaring the case class within the UDF: {code:java} import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} case class DataRow(field1: String) val sparkSession = SparkSession.builder.getOrCreate() import sparkSession.implicits._ val udf1 = udf((x: String) => { val testData = Seq(DataRow("test1"), DataRow("test2")).toDF("test") // This is failing at runtime 3 }) val df1 = Seq(DataRow("test1"), DataRow("test2")).toDF("test").withColumn("udf", udf1($"test")) // This is working display(df1) {code} I get a different exception at runtime: {code:java} at $linedb3da7b2933d4b63a62b3d2a21c675f2196.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$$$c35395a9233a7197629c985e87dce75$$$$w$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.$anonfun$udf1$1$adapted(command-2013905963200886:10) at org.apache.spark.sql.catalyst.expressions.ScalaUDF.$anonfun$f$2(ScalaUDF.scala:223) at org.apache.spark.sql.catalyst.expressions.ScalaUDF.eval(ScalaUDF.scala:1205) ... 111 more{code} Thanks. > ProductEncoder not working as expected within User Defined Fuction (UDF) > ------------------------------------------------------------------------ > > Key: SPARK-44706 > URL: https://issues.apache.org/jira/browse/SPARK-44706 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 3.4.0 > Environment: DBR (Databricks) 13.2, Spark 3.4.0 and Scala 2.12. > Reporter: Juan Carlos Blanco Martinez > Priority: Major > > Hi, > When running the following code in Databricks' notebook: > {code:java} > import org.apache.spark.{SparkConf, SparkContext} > import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} > case class DataRow(field1: String) > val sparkSession = SparkSession.builder.getOrCreate() > import sparkSession.implicits._ > val udf1 = udf((x: String) => { > val testData = Seq(DataRow("test1"), DataRow("test2")).toDF("test") // > This is failing at runtime > 3 > }) > val df1 = Seq(DataRow("test1"), > DataRow("test2")).toDF("test").withColumn("udf", udf1($"test")) // This is > working > display(df1) > {code} > I get a ScalaReflectionException at runtime: > {code:java} > ScalaReflectionException: class > $linedb3da7b2933d4b63a62b3d2a21c675f2141.$read in JavaMirror with > com.databricks.backend.daemon.driver.DriverLocal$DriverLocalClassLoader@717115ad > of type class > com.databricks.backend.daemon.driver.DriverLocal$DriverLocalClassLoader with > classpath [] and parent being > com.databricks.backend.daemon.driver.ClassLoaders$ReplWrappingClassLoader@1670897 > of type class > com.databricks.backend.daemon.driver.ClassLoaders$ReplWrappingClassLoader > with classpath [<unknown>] and parent being > com.databricks.backend.daemon.driver.ClassLoaders$LibraryClassLoader@1a42da0a > of type class > com.databricks.backend.daemon.driver.ClassLoaders$LibraryClassLoader with > classpath > [file:/local_disk0/tmp/repl/spark-4071811259476162981-8e526ae9-25fb-4545-8d3f-963a8661cd2b/] > and parent being sun.misc.Launcher$AppClassLoader@43ee72e6 of type class > sun.misc.Launcher$AppClassLoader with classpath [...] not found. > at scala.reflect.internal.Mirrors$RootsBase.staticClass(Mirrors.scala:141) > at scala.reflect.internal.Mirrors$RootsBase.staticClass(Mirrors.scala:29) > at > $linedb3da7b2933d4b63a62b3d2a21c675f2196.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$$$c35395a9233a7197629c985e87dce75$$$$w$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$typecreator6$1.apply(command-2013905963200886:11) > > at > scala.reflect.api.TypeTags$WeakTypeTagImpl.tpe$lzycompute(TypeTags.scala:237) > at scala.reflect.api.TypeTags$WeakTypeTagImpl.tpe(TypeTags.scala:237) > at > org.apache.spark.sql.catalyst.ScalaReflection$.encoderFor(ScalaReflection.scala:848) > > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:55) > > at org.apache.spark.sql.Encoders$.product(Encoders.scala:312) > at > org.apache.spark.sql.LowPrioritySQLImplicits.newProductEncoder(SQLImplicits.scala:302) > > at > org.apache.spark.sql.LowPrioritySQLImplicits.newProductEncoder$(SQLImplicits.scala:302) > > at org.apache.spark.sql.SQLImplicits.newProductEncoder(SQLImplicits.scala:34) > at > $linedb3da7b2933d4b63a62b3d2a21c675f2196.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$$$c35395a9233a7197629c985e87dce75$$$$w$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.$anonfun$udf1$1(command-2013905963200886:11) > {code} > Declaring the case class within the UDF: > {code:java} > import org.apache.spark.{SparkConf, SparkContext} > import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} val > sparkSession = SparkSession.builder.getOrCreate() > import sparkSession.implicits._ val udf1 = udf((x: String) => { > case class DataRow(field1: String) > val testData = Seq(DataRow("test1"), DataRow("test2")).toDF("test") // > This is failing at runtime > 3 > }) val df1 = Seq(DataRow("test1"), > DataRow("test2")).toDF("test").withColumn("udf", udf1($"test")) // This is > working > display(df1) {code} > I get a different error > {code:java} > error: value toDF is not a member of Seq[DataRow] val testData = > Seq(DataRow("test1"), DataRow("test2")).toDF("test"){code} > Thanks. -- This message was sent by Atlassian Jira (v8.20.10#820010) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org