cloud-fan commented on code in PR #59188:
URL: https://github.com/apache/spark/pull/59188#discussion_r4185992489
##########
sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala:
##########
@@ -2444,42 +2444,47 @@ class BasicCharVarcharTestSuite extends
SharedSparkSession {
test("SPARK-59274: from_json/csv/xml honor CHAR/VARCHAR under
standardSemantics") {
withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
- def checkTextInput(query: String, inputType: DataType): DataFrame = {
- val df = sql(query)
- val inputTypes =
df.queryExecution.analyzed.expressions.flatMap(_.collect {
- case e: SupportTrimmedCharInput => e.child.dataType
- })
- assert(inputTypes === Seq(inputType), df.queryExecution.analyzed)
- df
- }
-
+ // CHAR padding is part of the STRING value after CAST. Parsers do not
rtrim it.
+ // from_json on padded CHAR must match the same document as STRING with
trailing spaces.
checkAnswer(
- checkTextInput(
- """SELECT from_json(CAST('{"a":1}' AS CHAR(12)), 'a INT')""",
- CharType(12)),
+ sql("""SELECT from_json(CAST('{"a":1}' AS CHAR(12)), 'a INT')"""),
+ sql("""SELECT from_json('{"a":1} ', 'a INT')"""))
+ // XML ignores trailing whitespace, so a padded CHAR document still
parses.
+ checkAnswer(
+ sql(
+ """SELECT from_xml(
Review Comment:
**Nit (P3):** This well-formed XML case has the same result with or without
the removed CHAR trim because XML ignores trailing whitespace. As written, it
stays green if from_xml starts stripping the padded value again, so the new
no-rtrim behavior has no regression-sensitive XML assertion.
See **Shared repair plan 1** in the review body.
##########
sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala:
##########
@@ -2444,42 +2444,47 @@ class BasicCharVarcharTestSuite extends
SharedSparkSession {
test("SPARK-59274: from_json/csv/xml honor CHAR/VARCHAR under
standardSemantics") {
withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
- def checkTextInput(query: String, inputType: DataType): DataFrame = {
- val df = sql(query)
- val inputTypes =
df.queryExecution.analyzed.expressions.flatMap(_.collect {
- case e: SupportTrimmedCharInput => e.child.dataType
- })
- assert(inputTypes === Seq(inputType), df.queryExecution.analyzed)
- df
- }
-
+ // CHAR padding is part of the STRING value after CAST. Parsers do not
rtrim it.
+ // from_json on padded CHAR must match the same document as STRING with
trailing spaces.
checkAnswer(
- checkTextInput(
- """SELECT from_json(CAST('{"a":1}' AS CHAR(12)), 'a INT')""",
- CharType(12)),
+ sql("""SELECT from_json(CAST('{"a":1}' AS CHAR(12)), 'a INT')"""),
Review Comment:
**Nit (P3):** This assertion does not distinguish the new no-rtrim behavior:
JSON accepts trailing whitespace, so restoring CHAR right trimming would still
make both queries return the same Row(1). The complete padded-value behavior
for from_json can therefore regress without any focused test failure.
See **Shared repair plan 1** in the review body.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
