dongjoon-hyun commented on code in PR #52305:
URL: https://github.com/apache/spark/pull/52305#discussion_r2341175503
##########
sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala:
##########
@@ -1617,6 +1617,32 @@ class DataFrameSetOperationsSuite extends QueryTest
}
}
}
+
+ test("SPARK-53550: union partitioning should compare canonicalized
attributes") {
+ withSQLConf(
+ "spark.sql.autoBroadcastJoinThreshold" -> "-1") {
+ withTempView("person", "person_a", "person_b", "person_c") {
+ // scalastyle:off line.size.limit
+ sql(
+ """create or replace temp view person(id, name, age) as values
+ |(0, "mike", 30),
+ |(1, "jim", 20);
+ |""".stripMargin)
+ sql(
+ """create or replace temp view person_a as
+ |SELECT name, avg(age) as avg_age FROM person GROUP BY
name;""".stripMargin)
+ sql(
+ """create or replace temp view person_b as
+ |SELECT p1.name, p2.avg_age FROM person p1 JOIN person_a p2 ON
p1.name = p2.name;""".stripMargin)
+ sql(
+ """create or replace temp view person_c as
+ |SELECT * FROM person_a UNION SELECT * FROM
person_b;""".stripMargin)
+ val df = sql("SELECT p1.name, p2.avg_age FROM person_c p1 JOIN
person_c p2 ON p1.name = p2.name")
+ // scalastyle:on line.size.limit
+ checkAnswer(df, Row("jim", 20) :: Row("mike", 30) :: Nil)
Review Comment:
So, without the normalization, the following (and very confusing) error hits
the users?
```
[info] java.lang.IllegalArgumentException: Can't zip RDDs with unequal
numbers of partitions: List(5, 10)
[info] at
org.apache.spark.rdd.ZippedPartitionsBaseRDD.getPartitions(ZippedPartitionsRDD.scala:58)
...
[info] at
org.apache.spark.sql.DataFrameSetOperationsSuite.$anonfun$new$248(DataFrameSetOperationsSuite.scala:1642)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
