[
https://issues.apache.org/jira/browse/SPARK-53635?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Jiri Humpolicek updated SPARK-53635:
------------------------------------
Description:
Hi, what am I doing wrong? Following is working in spark-3.x but doesn't work
in spark-4.x
{code:scala}
case class N(navs: Int)
case class I(first: String, second: Seq[N])
val h = Seq(I("a", Seq(N(10), N(15)))).toDF
h.printSchema
// root
// |-- first: string (nullable = true)
// |-- second: array (nullable = true)
// | |-- element: struct (containsNull = true)
// | | |-- navs: integer (nullable = false)
import org.apache.spark.sql.Row
import org.apache.spark.sql.expressions.UserDefinedFunction
val reduceItems = (items: Seq[Row]) => {
items.map(_.getAs[Int]("navs")).reduce(_ + _)
}
val reduceItemsUdf = udf(reduceItems)
h.select(reduceItemsUdf($"second").as("r")).show()
{code}
```
in spark-4.x it throws:
{code:java}
scala.MatchError:
org.apache.spark.sql.catalyst.encoders.AgnosticEncoders$UnboundRowEncoder$@4aaf547c
(of class
org.apache.spark.sql.catalyst.encoders.AgnosticEncoders$UnboundRowEncoder$)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializer(SerializerBuildHelper.scala:308)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.$anonfun$validateAndSerializeElement$1(SerializerBuildHelper.scala:500)
at
org.apache.spark.sql.catalyst.expressions.objects.MapObjects$.apply(objects.scala:890)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializerForMapObjects(SerializerBuildHelper.scala:242)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.serializerForArray(SerializerBuildHelper.scala:449)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializer(SerializerBuildHelper.scala:356)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializer(SerializerBuildHelper.scala:300)
at
org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:59)
at
org.apache.spark.sql.catalyst.encoders.package$.encoderFor(package.scala:34)
at
org.apache.spark.sql.classic.UserDefinedFunctionUtils$$anonfun$$nestedInanonfun$toScalaUDF$1$1.applyOrElse(UserDefinedFunctionUtils.scala:37)
at
org.apache.spark.sql.classic.UserDefinedFunctionUtils$$anonfun$$nestedInanonfun$toScalaUDF$1$1.applyOrElse(UserDefinedFunctionUtils.scala:35)
at scala.PartialFunction$Lifted.apply(PartialFunction.scala:338)
at scala.PartialFunction$Lifted.apply(PartialFunction.scala:334)
at scala.Option.collect(Option.scala:462)
at
org.apache.spark.sql.classic.UserDefinedFunctionUtils$.$anonfun$toScalaUDF$1(UserDefinedFunctionUtils.scala:35)
at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:75)
at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:35)
at
org.apache.spark.sql.classic.UserDefinedFunctionUtils$.toScalaUDF(UserDefinedFunctionUtils.scala:35)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$2(columnNodeSupport.scala:181)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$1(columnNodeSupport.scala:48)
at org.apache.spark.sql.internal.SQLConf$.withExistingConf(SQLConf.scala:161)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply(columnNodeSupport.scala:47)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply$(columnNodeSupport.scala:46)
at
org.apache.spark.sql.classic.SparkSession$$anon$3.apply(SparkSession.scala:713)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$2(columnNodeSupport.scala:91)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$1(columnNodeSupport.scala:48)
at org.apache.spark.sql.internal.SQLConf$.withExistingConf(SQLConf.scala:161)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply(columnNodeSupport.scala:47)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply$(columnNodeSupport.scala:46)
at
org.apache.spark.sql.classic.SparkSession$$anon$3.apply(SparkSession.scala:713)
at org.apache.spark.sql.classic.RichColumn.expr(conversions.scala:106)
at org.apache.spark.sql.classic.RichColumn.named(conversions.scala:110)
at org.apache.spark.sql.classic.Dataset.$anonfun$select$3(Dataset.scala:894)
at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:75)
at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:35)
at org.apache.spark.sql.classic.Dataset.select(Dataset.scala:894)
at org.apache.spark.sql.classic.Dataset.select(Dataset.scala:232)
{code}
was:
Hi, what am I doing wrong? Following is working in spark-3.x but doesn't work
in spark-4.x
{code:scala}
case class N(navs: Int)
case class I(first: String, second: Seq[N])
val h = Seq(I("a", Seq(N(10), N(15)))).toDF
h.printSchema
// root
// |-- first: string (nullable = true)
// |-- second: array (nullable = true)
// | |-- element: struct (containsNull = true)
// | | |-- navs: integer (nullable = false)
import org.apache.spark.sql.Row
import org.apache.spark.sql.expressions.UserDefinedFunction
val reduceItems = (items: Seq[Row]) => {
items.map(_.getAs[Int]("navs")).reduce(_ + _)
/// 10
}
val reduceItemsUdf = udf(reduceItems)
h.select(reduceItemsUdf($"second").as("r")).show()
{code}
```
in spark-4.x it throws:
{code:java}
scala.MatchError:
org.apache.spark.sql.catalyst.encoders.AgnosticEncoders$UnboundRowEncoder$@4aaf547c
(of class
org.apache.spark.sql.catalyst.encoders.AgnosticEncoders$UnboundRowEncoder$)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializer(SerializerBuildHelper.scala:308)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.$anonfun$validateAndSerializeElement$1(SerializerBuildHelper.scala:500)
at
org.apache.spark.sql.catalyst.expressions.objects.MapObjects$.apply(objects.scala:890)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializerForMapObjects(SerializerBuildHelper.scala:242)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.serializerForArray(SerializerBuildHelper.scala:449)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializer(SerializerBuildHelper.scala:356)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializer(SerializerBuildHelper.scala:300)
at
org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:59)
at
org.apache.spark.sql.catalyst.encoders.package$.encoderFor(package.scala:34)
at
org.apache.spark.sql.classic.UserDefinedFunctionUtils$$anonfun$$nestedInanonfun$toScalaUDF$1$1.applyOrElse(UserDefinedFunctionUtils.scala:37)
at
org.apache.spark.sql.classic.UserDefinedFunctionUtils$$anonfun$$nestedInanonfun$toScalaUDF$1$1.applyOrElse(UserDefinedFunctionUtils.scala:35)
at scala.PartialFunction$Lifted.apply(PartialFunction.scala:338)
at scala.PartialFunction$Lifted.apply(PartialFunction.scala:334)
at scala.Option.collect(Option.scala:462)
at
org.apache.spark.sql.classic.UserDefinedFunctionUtils$.$anonfun$toScalaUDF$1(UserDefinedFunctionUtils.scala:35)
at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:75)
at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:35)
at
org.apache.spark.sql.classic.UserDefinedFunctionUtils$.toScalaUDF(UserDefinedFunctionUtils.scala:35)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$2(columnNodeSupport.scala:181)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$1(columnNodeSupport.scala:48)
at org.apache.spark.sql.internal.SQLConf$.withExistingConf(SQLConf.scala:161)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply(columnNodeSupport.scala:47)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply$(columnNodeSupport.scala:46)
at
org.apache.spark.sql.classic.SparkSession$$anon$3.apply(SparkSession.scala:713)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$2(columnNodeSupport.scala:91)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$1(columnNodeSupport.scala:48)
at org.apache.spark.sql.internal.SQLConf$.withExistingConf(SQLConf.scala:161)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply(columnNodeSupport.scala:47)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply$(columnNodeSupport.scala:46)
at
org.apache.spark.sql.classic.SparkSession$$anon$3.apply(SparkSession.scala:713)
at org.apache.spark.sql.classic.RichColumn.expr(conversions.scala:106)
at org.apache.spark.sql.classic.RichColumn.named(conversions.scala:110)
at org.apache.spark.sql.classic.Dataset.$anonfun$select$3(Dataset.scala:894)
at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:75)
at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:35)
at org.apache.spark.sql.classic.Dataset.select(Dataset.scala:894)
at org.apache.spark.sql.classic.Dataset.select(Dataset.scala:232)
{code}
> udf with array of structure in spark 4
> --------------------------------------
>
> Key: SPARK-53635
> URL: https://issues.apache.org/jira/browse/SPARK-53635
> Project: Spark
> Issue Type: Question
> Components: SQL
> Affects Versions: 4.0.0, 4.0.1
> Reporter: Jiri Humpolicek
> Priority: Major
>
> Hi, what am I doing wrong? Following is working in spark-3.x but doesn't work
> in spark-4.x
> {code:scala}
> case class N(navs: Int)
> case class I(first: String, second: Seq[N])
> val h = Seq(I("a", Seq(N(10), N(15)))).toDF
> h.printSchema
> // root
> // |-- first: string (nullable = true)
> // |-- second: array (nullable = true)
> // | |-- element: struct (containsNull = true)
> // | | |-- navs: integer (nullable = false)
> import org.apache.spark.sql.Row
> import org.apache.spark.sql.expressions.UserDefinedFunction
> val reduceItems = (items: Seq[Row]) => {
> items.map(_.getAs[Int]("navs")).reduce(_ + _)
> }
> val reduceItemsUdf = udf(reduceItems)
> h.select(reduceItemsUdf($"second").as("r")).show()
> {code}
> ```
> in spark-4.x it throws:
> {code:java}
> scala.MatchError:
> org.apache.spark.sql.catalyst.encoders.AgnosticEncoders$UnboundRowEncoder$@4aaf547c
> (of class
> org.apache.spark.sql.catalyst.encoders.AgnosticEncoders$UnboundRowEncoder$)
> at
> org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializer(SerializerBuildHelper.scala:308)
> at
> org.apache.spark.sql.catalyst.SerializerBuildHelper$.$anonfun$validateAndSerializeElement$1(SerializerBuildHelper.scala:500)
> at
> org.apache.spark.sql.catalyst.expressions.objects.MapObjects$.apply(objects.scala:890)
> at
> org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializerForMapObjects(SerializerBuildHelper.scala:242)
> at
> org.apache.spark.sql.catalyst.SerializerBuildHelper$.serializerForArray(SerializerBuildHelper.scala:449)
> at
> org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializer(SerializerBuildHelper.scala:356)
> at
> org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializer(SerializerBuildHelper.scala:300)
> at
> org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:59)
> at
> org.apache.spark.sql.catalyst.encoders.package$.encoderFor(package.scala:34)
> at
> org.apache.spark.sql.classic.UserDefinedFunctionUtils$$anonfun$$nestedInanonfun$toScalaUDF$1$1.applyOrElse(UserDefinedFunctionUtils.scala:37)
> at
> org.apache.spark.sql.classic.UserDefinedFunctionUtils$$anonfun$$nestedInanonfun$toScalaUDF$1$1.applyOrElse(UserDefinedFunctionUtils.scala:35)
> at scala.PartialFunction$Lifted.apply(PartialFunction.scala:338)
> at scala.PartialFunction$Lifted.apply(PartialFunction.scala:334)
> at scala.Option.collect(Option.scala:462)
> at
> org.apache.spark.sql.classic.UserDefinedFunctionUtils$.$anonfun$toScalaUDF$1(UserDefinedFunctionUtils.scala:35)
> at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:75)
> at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:35)
> at
> org.apache.spark.sql.classic.UserDefinedFunctionUtils$.toScalaUDF(UserDefinedFunctionUtils.scala:35)
> at
> org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$2(columnNodeSupport.scala:181)
> at
> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
> at
> org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$1(columnNodeSupport.scala:48)
> at
> org.apache.spark.sql.internal.SQLConf$.withExistingConf(SQLConf.scala:161)
> at
> org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply(columnNodeSupport.scala:47)
> at
> org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply$(columnNodeSupport.scala:46)
> at
> org.apache.spark.sql.classic.SparkSession$$anon$3.apply(SparkSession.scala:713)
> at
> org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$2(columnNodeSupport.scala:91)
> at
> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
> at
> org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$1(columnNodeSupport.scala:48)
> at
> org.apache.spark.sql.internal.SQLConf$.withExistingConf(SQLConf.scala:161)
> at
> org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply(columnNodeSupport.scala:47)
> at
> org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply$(columnNodeSupport.scala:46)
> at
> org.apache.spark.sql.classic.SparkSession$$anon$3.apply(SparkSession.scala:713)
> at org.apache.spark.sql.classic.RichColumn.expr(conversions.scala:106)
> at org.apache.spark.sql.classic.RichColumn.named(conversions.scala:110)
> at org.apache.spark.sql.classic.Dataset.$anonfun$select$3(Dataset.scala:894)
> at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:75)
> at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:35)
> at org.apache.spark.sql.classic.Dataset.select(Dataset.scala:894)
> at org.apache.spark.sql.classic.Dataset.select(Dataset.scala:232)
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
