srielau commented on code in PR #58581:
URL: https://github.com/apache/spark/pull/58581#discussion_r4027707088
##########
python/pyspark/sql/tests/connect/test_connect_basic.py:
##########
@@ -491,17 +491,28 @@ def test_schema(self):
self._check_print_schema(query)
def test_char_varchar_result_schema(self):
- # SPARK-58794: Python Connect maps first-class CHAR/VARCHAR the same
as classic.
- query = "SELECT CAST('ab' AS CHAR(4)) AS c, CAST('cd' AS VARCHAR(6))
AS v"
+ # SPARK-59276: Python Connect maps first-class CHAR/VARCHAR the same
as classic.
Review Comment:
Done — 56788ccc44c. Python Connect now converts CHAR/VARCHAR leaves to
collation-preserving STRING only for local Arrow, keeps the original requested
schema on the LocalRelation, and covers populated and empty createDataFrame
under standard, legacy-as-string, and default server policies.
##########
sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala:
##########
@@ -137,6 +137,8 @@ case class StructField(
}
private def isCollatedString(dt: DataType): Boolean = dt match {
+ case c: CharType => c.collation.isDefined
Review Comment:
Done — 3ac37023436. CHAR/VARCHAR collations now use
__CHAR_VARCHAR_COLLATIONS, STRING stays on __COLLATIONS, and older readers that
ignore the new key parse uncollated char(n)/varchar(n). Dual encodings and
__COLLATIONS on CHAR/VARCHAR are rejected.
##########
sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala:
##########
@@ -1641,9 +1647,35 @@ class SparkConnectPlanner(
.asInstanceOf[Project]
val proj = UnsafeProjection.create(project.projectList,
project.child.output)
- logical.LocalRelation(
- DataTypeUtils.toAttributes(schema),
- data.map(proj).map(_.copy()).toSeq)
+ def restoreFieldNames(actual: DataType, requested: DataType): DataType
=
+ (actual, requested) match {
+ case (_, requestedUdt: UserDefinedType[_]) => requestedUdt
+ case (StructType(actualFields), StructType(requestedFields)) =>
+ StructType(
+ actualFields.zip(requestedFields).map { case (actualField,
requestedField) =>
+ actualField.copy(
+ name = requestedField.name,
+ dataType = restoreFieldNames(actualField.dataType,
requestedField.dataType))
+ })
+ case (ArrayType(actualElement, containsNull),
ArrayType(requestedElement, _)) =>
+ ArrayType(restoreFieldNames(actualElement, requestedElement),
containsNull)
+ case (
+ MapType(actualKey, actualValue, valueContainsNull),
+ MapType(requestedKey, requestedValue, _)) =>
+ MapType(
+ restoreFieldNames(actualKey, requestedKey),
+ restoreFieldNames(actualValue, requestedValue),
+ valueContainsNull)
+ case _ => actual
+ }
+ val output = project.output.zip(schema.fields).map { case (attribute,
field) =>
+ AttributeReference(
+ field.name,
+ restoreFieldNames(attribute.dataType, field.dataType),
Review Comment:
Done — 3ac37023436. restoreFieldNames now keeps the requested explicit
UTF8_BINARY StringType when Dataset.to has not selected a different leaf, and
still keeps the analyzed type when CHAR/VARCHAR policy changes it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
