srielau commented on code in PR #58581:
URL: https://github.com/apache/spark/pull/58581#discussion_r3994026777
##########
sql/connect/common/src/main/scala/org/apache/spark/sql/connect/SparkSession.scala:
##########
@@ -223,7 +223,9 @@ class SparkSession private[sql] (
/** @inheritdoc */
def createDataFrame(rows: java.util.List[Row], schema: StructType):
DataFrame = {
- createDataset(RowEncoder.encoderFor(schema),
rows.iterator().asScala).toDF()
+ // The client cannot observe the server's CHAR/VARCHAR configuration.
Encode an explicitly
Review Comment:
Fixed in 46f0f8dea49. The comment now specifically describes the
client-local CHAR/VARCHAR semantics applied by RowEncoder and leaves other
local encoder configuration out of scope.
##########
python/pyspark/sql/types.py:
##########
@@ -330,19 +330,31 @@ class CharType(AtomicType):
----------
length : int
the length limitation.
+ collation : str, optional
Review Comment:
Fixed in 46f0f8dea49. Both parameter docs now state that None means no
explicitly declared collation and is distinct from explicit UTF8_BINARY.
##########
python/pyspark/sql/types.py:
##########
@@ -330,19 +330,31 @@ class CharType(AtomicType):
----------
length : int
the length limitation.
+ collation : str, optional
+ name of the collation.
"""
- def __init__(self, length: int):
+ def __init__(self, length: int, collation: Optional[str] = None):
self.length = length
+ self.collation = collation
def simpleString(self) -> str:
- return "char(%d)" % (self.length)
+ if self.collation is None:
+ return "char(%d)" % (self.length)
+
+ return "char(%d) collate %s" % (self.length, self.collation)
def jsonValue(self) -> str:
- return "char(%d)" % (self.length)
+ return self.simpleString()
def __repr__(self) -> str:
Review Comment:
Fixed in 46f0f8dea49. The repr round-trip test now includes explicit
UTF8_BINARY and non-binary collations for both CharType and VarcharType.
##########
python/pyspark/sql/types.py:
##########
@@ -2671,7 +2708,12 @@ def _parse_datatype_json_value( # type: ignore[return]
def _assert_valid_type_for_collation(
fieldPath: str, fieldType: Any, collationMap: Dict[str, str]
) -> None:
- if fieldPath in collationMap and fieldType != "string":
+ is_string_type = (
+ fieldType == "string"
+ or (isinstance(fieldType, str) and _LENGTH_CHAR.fullmatch(fieldType)
is not None)
Review Comment:
Fixed in 46f0f8dea49. Collation restoration metadata now accepts only
uncollated char(n)/varchar(n) type text. Matching and conflicting
inline-plus-metadata encodings are both covered as negative cases.
##########
sql/connect/common/src/main/scala/org/apache/spark/sql/connect/SparkSession.scala:
##########
@@ -223,7 +223,10 @@ class SparkSession private[sql] (
/** @inheritdoc */
def createDataFrame(rows: java.util.List[Row], schema: StructType):
DataFrame = {
- createDataset(RowEncoder.encoderFor(schema),
rows.iterator().asScala).toDF()
+ // RowEncoder consults the client process's local SqlApiConf, which can
differ from the
+ // server-side configuration visible through SparkSession.conf. Encode an
explicitly provided
+ // schema independently of that local configuration, as for result schemas.
+ createDataset(RowEncoder.encoderForResultSchema(schema),
rows.iterator().asScala).toDF()
Review Comment:
Fixed in 46f0f8dea49. Explicit Row input now uses a collation-preserving
physical STRING encoder while carrying the requested logical schema separately.
Server-side reconciliation selects standard, legacy-as-string, or rejection
behavior and builds the final attributes from that result. The E2E test covers
empty and populated nested values, collations, standard padding/overflow,
legacy unpadded STRING output, and default rejection.
##########
sql/connect/common/src/main/scala/org/apache/spark/sql/connect/SparkSession.scala:
##########
@@ -223,7 +223,9 @@ class SparkSession private[sql] (
/** @inheritdoc */
def createDataFrame(rows: java.util.List[Row], schema: StructType):
DataFrame = {
- createDataset(RowEncoder.encoderFor(schema),
rows.iterator().asScala).toDF()
+ // The client cannot observe the server's CHAR/VARCHAR configuration.
Encode an explicitly
+ // provided schema independently of the client's local configuration, as
for result schemas.
+ createDataset(RowEncoder.encoderForResultSchema(schema),
rows.iterator().asScala).toDF()
Review Comment:
Fixed in 46f0f8dea49. Empty and populated input now share server-side policy
reconciliation, including legacy charVarcharAsString output with unpadded
physical values. The expanded E2E matrix covers standard, legacy, and default
modes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
