cloud-fan commented on code in PR #55582:
URL: https://github.com/apache/spark/pull/55582#discussion_r3154164832
##########
python/pyspark/sql/tests/test_column.py:
##########
@@ -558,6 +558,26 @@ def test_select_join_keys(self):
self.assertTrue(df1.join(df2, "id", how).select(df1["id"]).count()
>= 0, how)
self.assertTrue(df1.join(df2, "id", how).select(df2["id"]).count()
>= 0, how)
+ def
test_select_regular_column_with_reused_dataframe_hidden_in_natural_join(self):
+ # A DataFrame appears both as a direct join side and inside a
natural/USING
+ # join that hides one of its columns into `metadataOutput`. When
resolving
+ # `dim["dim_id"]`, two candidates match the plan id: one from
`p.output`
+ # (the direct join side) and one only visible via `p.metadataOutput`
(the
+ # reused `dim` nested under the USING-join wrapper). We should prefer
the
+ # regular candidate and not throw AMBIGUOUS_COLUMN_REFERENCE.
+ fact = self.spark.createDataFrame([(1, 10, "T1"), (2, 20, "T2")],
["id", "fk", "txn_id"])
+ dim = self.spark.createDataFrame([(10, "X"), (20, "Y"), (30, "Z")],
["dim_id", "dim_name"])
+ events = self.spark.createDataFrame(
+ [(10, "T1", 100), (20, "T2", 200)], ["dim_id", "txn_id", "amount"]
+ )
+ enriched = events.join(dim, "dim_id", "left")
+ result = (
+ fact.join(dim, fact["fk"] == dim["dim_id"], "left")
+ .join(enriched, "txn_id", "full_outer")
+ .select(dim["dim_id"])
+ )
+ self.assertEqual(result.count(), 2)
Review Comment:
`count() == 2` only proves the query didn't throw
`AMBIGUOUS_COLUMN_REFERENCE`. A wrong resolution that picks the hidden
candidate could still yield 2 rows (or fail later in execution rather than
analysis). Consider asserting on the actual resolved values to pin the
resolution to the direct-side `dim_id`, e.g.:
```suggestion
self.assertEqual(
[r["dim_id"] for r in result.sort("txn_id").collect()],
[10, 20],
)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
