Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/21732#discussion_r232563132
--- Diff: sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
---
@@ -1556,6 +1547,69 @@ class DatasetSuite extends QueryTest with
SharedSQLContext {
df.where($"city".contains(new java.lang.Character('A'))),
Seq(Row("Amsterdam")))
}
+
+ test("SPARK-24762: Enable top-level Option of Product encoders") {
+ val data = Seq(Some((1, "a")), Some((2, "b")), None)
+ val ds = data.toDS()
+
+ checkDataset(
+ ds,
+ data: _*)
+
+ val schema = new StructType().add(
+ "value",
+ new StructType()
+ .add("_1", IntegerType, nullable = false)
+ .add("_2", StringType, nullable = true),
+ nullable = true)
+
+ assert(ds.schema == schema)
+
+ val nestedOptData = Seq(Some((Some((1, "a")), 2.0)), Some((Some((2,
"b")), 3.0)))
+ val nestedDs = nestedOptData.toDS()
+
+ checkDataset(
+ nestedDs,
+ nestedOptData: _*)
+
+ val nestedSchema = StructType(Seq(
+ StructField("value", StructType(Seq(
+ StructField("_1", StructType(Seq(
+ StructField("_1", IntegerType, nullable = false),
+ StructField("_2", StringType, nullable = true)))),
+ StructField("_2", DoubleType, nullable = false)
+ )), nullable = true)
+ ))
+ assert(nestedDs.schema == nestedSchema)
+ }
+
+ test("SPARK-24762: Resolving Option[Product] field") {
+ val ds = Seq((1, ("a", 1.0)), (2, ("b", 2.0))).toDS().as[(Int,
Option[(String, Double)])]
+ checkDataset(ds,
+ (1, Some(("a", 1.0))), (2, Some(("b", 2.0))))
+ }
+
+ test("SPARK-24762: select Option[Product] field") {
+ val ds = Seq(("a", 1), ("b", 2), ("c", 3)).toDS()
+ .select(expr("struct(_2, _2 + 1)").as[Option[(Int, Int)]])
+ checkDataset(ds,
+ Some((1, 2)), Some((2, 3)), Some((3, 4)))
+ }
+
+ test("SPARK-24762: joinWith on Option[Product]") {
+ val ds1 = Seq(Some((1, 2)), Some((2, 3))).toDS().as("a")
--- End diff --
ditto, let's test None
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
