szehon-ho commented on code in PR #55329:
URL: https://github.com/apache/spark/pull/55329#discussion_r3327457737
##########
sql/core/src/test/scala/org/apache/spark/sql/connector/MergeIntoSchemaEvolutionBasicTests.scala:
##########
@@ -1265,4 +1265,71 @@ trait MergeIntoSchemaEvolutionBasicTests extends
MergeIntoSchemaEvolutionSuiteBa
expected = Seq((1, "hr")).toDF("pk", "dep"),
expectedWithoutEvolution = Seq((1, "hr")).toDF("pk", "dep")
)
+
+ // SPARK-56462: UPDATE * / INSERT * schema evolution must work when the
source has a column
Review Comment:
comment is too much detail, can we remove?
##########
sql/core/src/test/scala/org/apache/spark/sql/connector/MergeIntoSchemaEvolutionBasicTests.scala:
##########
@@ -1265,4 +1265,71 @@ trait MergeIntoSchemaEvolutionBasicTests extends
MergeIntoSchemaEvolutionSuiteBa
expected = Seq((1, "hr")).toDF("pk", "dep"),
expectedWithoutEvolution = Seq((1, "hr")).toDF("pk", "dep")
)
+
+ // SPARK-56462: UPDATE * / INSERT * schema evolution must work when the
source has a column
+ // whose name contains a dot (e.g. `job.title`). Previously, constructing
the assignment key
+ // with UnresolvedAttribute(sourceAttr.name) called apply(), which parsed
the dot as a
+ // name-part separator and produced nameParts = Seq("job", "title") instead
of
+ // Seq("job.title"). That made isSameColumnAssignment return false,
blocking schema evolution
+ // and causing an UNRESOLVED_COLUMN analysis error rather than adding the
new column.
+ testEvolution("SPARK-56462: source has extra column with dot in name")(
+ targetData = Seq(
+ (1, 100, "hr"),
+ (2, 200, "software"),
+ (3, 300, "hr")
+ ).toDF("pk", "salary", "dep"),
+ // "job.title" is a single column name containing a dot (not a qualified
reference)
+ sourceData = Seq(
+ (2, 150, "finance", "engineer"),
+ (4, 400, "finance", "manager")
+ ).toDF("pk", "salary", "dep", "job.title"),
+ clauses = Seq(
+ updateAll(),
+ insertAll()
+ ),
+ // With schema evolution the dot-named column is added to target and
populated
+ expected = Seq[(Int, Int, String, String)](
+ (1, 100, "hr", null),
+ (2, 150, "finance", "engineer"),
+ (3, 300, "hr", null),
+ (4, 400, "finance", "manager")
+ ).toDF("pk", "salary", "dep", "job.title"),
+ // Without schema evolution the dot-named extra column is simply ignored
+ expectedWithoutEvolution = Seq(
+ (1, 100, "hr"),
+ (2, 150, "finance"),
+ (3, 300, "hr"),
+ (4, 400, "finance")
+ ).toDF("pk", "salary", "dep")
+ )
+
+ // SPARK-56462: UPDATE * / INSERT * schema evolution must work when the
source has columns
Review Comment:
same
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
