Jiri Humpolicek created SPARK-53635:
---------------------------------------
Summary: udf with array of structure in spark 4
Key: SPARK-53635
URL: https://issues.apache.org/jira/browse/SPARK-53635
Project: Spark
Issue Type: Question
Components: SQL
Affects Versions: 4.0.1, 4.0.0
Reporter: Jiri Humpolicek
Hi, what am I doing wrong? Following is working in spark-3.x but doesn't work
in spark-4.x
{code:scala}
case class N(navs: Int)
case class I(first: String, second: Seq[N])
val h = Seq(I("a", Seq(N(10), N(15)))).toDF
h.printSchema
// root
// |-- first: string (nullable = true)
// |-- second: array (nullable = true)
// | |-- element: struct (containsNull = true)
// | | |-- navs: integer (nullable = false)
import org.apache.spark.sql.Row
import org.apache.spark.sql.expressions.UserDefinedFunction
val reduceItems = (items: Seq[Row]) => {
items.map(_.getAs[Int]("navs")).reduce(_ + _)
/// 10
}
val reduceItemsUdf = udf(reduceItems)
h.select(reduceItemsUdf($"second").as("r")).show()
{code}
```
in spark-4.x it throws:
{code:java}
scala.MatchError:
org.apache.spark.sql.catalyst.encoders.AgnosticEncoders$UnboundRowEncoder$@4aaf547c
(of class
org.apache.spark.sql.catalyst.encoders.AgnosticEncoders$UnboundRowEncoder$)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializer(SerializerBuildHelper.scala:308)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.$anonfun$validateAndSerializeElement$1(SerializerBuildHelper.scala:500)
at
org.apache.spark.sql.catalyst.expressions.objects.MapObjects$.apply(objects.scala:890)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializerForMapObjects(SerializerBuildHelper.scala:242)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.serializerForArray(SerializerBuildHelper.scala:449)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializer(SerializerBuildHelper.scala:356)
at
org.apache.spark.sql.catalyst.SerializerBuildHelper$.createSerializer(SerializerBuildHelper.scala:300)
at
org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:59)
at
org.apache.spark.sql.catalyst.encoders.package$.encoderFor(package.scala:34)
at
org.apache.spark.sql.classic.UserDefinedFunctionUtils$$anonfun$$nestedInanonfun$toScalaUDF$1$1.applyOrElse(UserDefinedFunctionUtils.scala:37)
at
org.apache.spark.sql.classic.UserDefinedFunctionUtils$$anonfun$$nestedInanonfun$toScalaUDF$1$1.applyOrElse(UserDefinedFunctionUtils.scala:35)
at scala.PartialFunction$Lifted.apply(PartialFunction.scala:338)
at scala.PartialFunction$Lifted.apply(PartialFunction.scala:334)
at scala.Option.collect(Option.scala:462)
at
org.apache.spark.sql.classic.UserDefinedFunctionUtils$.$anonfun$toScalaUDF$1(UserDefinedFunctionUtils.scala:35)
at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:75)
at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:35)
at
org.apache.spark.sql.classic.UserDefinedFunctionUtils$.toScalaUDF(UserDefinedFunctionUtils.scala:35)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$2(columnNodeSupport.scala:181)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$1(columnNodeSupport.scala:48)
at org.apache.spark.sql.internal.SQLConf$.withExistingConf(SQLConf.scala:161)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply(columnNodeSupport.scala:47)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply$(columnNodeSupport.scala:46)
at
org.apache.spark.sql.classic.SparkSession$$anon$3.apply(SparkSession.scala:713)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$2(columnNodeSupport.scala:91)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:86)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.$anonfun$apply$1(columnNodeSupport.scala:48)
at org.apache.spark.sql.internal.SQLConf$.withExistingConf(SQLConf.scala:161)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply(columnNodeSupport.scala:47)
at
org.apache.spark.sql.classic.ColumnNodeToExpressionConverter.apply$(columnNodeSupport.scala:46)
at
org.apache.spark.sql.classic.SparkSession$$anon$3.apply(SparkSession.scala:713)
at org.apache.spark.sql.classic.RichColumn.expr(conversions.scala:106)
at org.apache.spark.sql.classic.RichColumn.named(conversions.scala:110)
at org.apache.spark.sql.classic.Dataset.$anonfun$select$3(Dataset.scala:894)
at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:75)
at scala.collection.immutable.ArraySeq.map(ArraySeq.scala:35)
at org.apache.spark.sql.classic.Dataset.select(Dataset.scala:894)
at org.apache.spark.sql.classic.Dataset.select(Dataset.scala:232)
{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
