cloud-fan commented on code in PR #58255:
URL: https://github.com/apache/spark/pull/58255#discussion_r3863781540
##########
sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala:
##########
@@ -2157,6 +2157,43 @@ class BasicCharVarcharTestSuite extends
SharedSparkSession {
}
}
}
+
+ test("SPARK-59001: empty CHAR/VARCHAR partition values become null like
STRING") {
+ withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+ // CHAR(n>0) pads '' to spaces; CHAR(0) is the empty CHAR that
empty2null should treat
+ // the same as VARCHAR/STRING.
+ Seq("CHAR(0)", "VARCHAR(5)").foreach { typ =>
+ withTempPath { path =>
+ sql(s"SELECT 0 AS id, CAST('' AS $typ) AS p UNION ALL SELECT 1,
CAST(NULL AS $typ)")
+
.write.mode("overwrite").partitionBy("p").parquet(path.getCanonicalPath)
+ val df = spark.read.parquet(path.getCanonicalPath)
+ checkAnswer(df.where("p IS NULL").select("id"), Seq(Row(0), Row(1)))
+ val dirs = path.listFiles().filterNot(
+ f => f.getName.startsWith(".") || f.getName.startsWith("_"))
+ assert(dirs.length === 1, dirs.map(_.getName).mkString(","))
+ }
+ }
+ }
+ }
+
+ test("SPARK-59001: text datasource accepts CHAR/VARCHAR as a string family
type") {
Review Comment:
**Non-blocking:**
Run this test with `SQLConf.USE_V1_SOURCE_LIST.key -> ""` (or exercise both
source modes). The default V1 list includes `text`, so this currently covers
only `TextFileFormat` and cannot regress the `TextTable.supportsDataType`
change.
##########
sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala:
##########
@@ -408,9 +408,30 @@ private[client] class Shim_v2_0 extends Shim with Logging {
}
}
- if (!SQLConf.get.metastorePartitionPruningFastFallback ||
- predicates.isEmpty ||
- predicates.exists(hasTimeZoneAwareExpression)) {
+ // CHAR/VARCHAR partition keys are excluded from the metastore filter (see
+ // SupportedAttribute), because Hive compares them with its own
trailing-blank rules. When
+ // a predicate actually mentions such a key, prune on the client instead.
Other empty-filter
+ // or MetaException fallbacks still honor
metastorePartitionPruningFastFallback.
+ def referencesCharVarcharPartitionKey: Boolean = {
+ SQLConf.get.charVarcharStandardSemantics && {
+ val charVarcharPartNames = catalogTable.partitionSchema.fields.collect
{
+ case f if CharVarcharUtils.hasCharVarchar(f.dataType) => f.name
+ }
+ charVarcharPartNames.nonEmpty && {
+ val resolver = SQLConf.get.resolver
+ predicates.exists(_.exists {
+ case a: Attribute => charVarcharPartNames.exists(n => resolver(n,
a.name))
+ case _ => false
+ })
+ }
+ }
+ }
+ val useClientSidePrune = SQLConf.get.metastorePartitionPruningFastFallback
||
Review Comment:
Thanks, the new branch order and mixed-key fetch-count assertion address
this concern.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
