AnishMahto commented on code in PR #59192:
URL: https://github.com/apache/spark/pull/59192#discussion_r4186770371
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala:
##########
@@ -333,6 +333,45 @@ class NestedColumnAliasingSuite extends SchemaPruningTest {
comparePlans(optimized, expected)
}
+ test("Redundant nested fields are detected through cosmetic variations of
their parents") {
+ val a = $"a".struct(StructType.fromDDL("b struct<c: int, d: int>, e int"))
+ // `a.b`, referenced many times below.
+ val ab = GetStructField(a, 0, Some("b"))
+ // `a.b` without the field name.
+ val unnamedAb = GetStructField(a, 0, None)
+ // `a.b.c` over the unnamed `a.b`.
+ val abc = GetStructField(unnamedAb, 0, Some("c"))
+ // `a.b.d` over a qualified `a`.
+ val abd = GetStructField(
+ GetStructField(a.withQualifier(Seq("t")), 0, Some("b")), 1, Some("d"))
+
+ val attrToExtractValues = NestedColumnAliasing.getAttributeToExtractValues(
+ Seq.fill(100)(ab) ++ Seq(unnamedAb, abc, abd), Seq.empty)
+
+ assert(attrToExtractValues.keys.map(_.exprId).toSeq == Seq(a.exprId))
+ assert(attrToExtractValues.values.toSeq == Seq(Seq(ab, unnamedAb)))
+ }
+
+ test("Nested fields are not redundant with non-deterministic parents") {
+ val a = $"a".array(StructType.fromDDL("b struct<c: int, d: int>, e int, f
int"))
+ // `a[ordinal].b` and `a[ordinal].b.c`.
+ def parentAndChild(ordinal: Expression): Seq[ExtractValue] = {
+ val parent = GetStructField(GetArrayItem(a, ordinal), 0, Some("b"))
+ Seq(parent, GetStructField(parent, 0, Some("c")))
+ }
+
+ val deterministic = parentAndChild(Literal(0))
+ assert(NestedColumnAliasing.getAttributeToExtractValues(deterministic,
Seq.empty)
+ .values.toSeq == Seq(deterministic.take(1)))
+
+ // Each evaluation of `a[rand].b` may read a different array element, so
`a[rand].b.c` is
+ // not redundant with it.
+ val nondeterministic =
+ parentAndChild(Cast(Multiply(Rand(Literal(0L)), Literal(2.0)),
IntegerType))
Review Comment:
Replaced the independent assertions with one that mixes both deterministic
and nondeterministic read references in the same invocation.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
