Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/22814#discussion_r228380639
--- Diff:
external/avro/src/test/scala/org/apache/spark/sql/avro/AvroFunctionsSuite.scala
---
@@ -61,6 +59,24 @@ class AvroFunctionsSuite extends QueryTest with
SharedSQLContext {
checkAnswer(avroStructDF.select(from_avro('avro, avroTypeStruct)), df)
}
+ test("handle invalid input in from_avro") {
+ val count = 10
+ val df = spark.range(count).select(struct('id,
'id.as("id2")).as("struct"))
+ val avroStructDF = df.select(to_avro('struct).as("avro"))
+ val avroTypeStruct = s"""
+ |{
+ | "type": "record",
+ | "name": "struct",
+ | "fields": [
+ | {"name": "col1", "type": "long"},
+ | {"name": "col2", "type": "double"}
+ | ]
+ |}
+ """.stripMargin
+ val expected = (0 until count).map(_ => Row(Row(null, null)))
+ checkAnswer(avroStructDF.select(from_avro('avro, avroTypeStruct)),
expected)
--- End diff --
Thanks. Makes sense to me
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
