uros-b commented on code in PR #59640:
URL: https://github.com/apache/spark/pull/59640#discussion_r4346501959
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/OperatorWithUncomparableTypeValidator.scala:
##########
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.analysis.resolver
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.{Distinct, LogicalPlan,
SetOperation}
import org.apache.spark.sql.errors.QueryCompilationErrors
-import org.apache.spark.sql.types.{DataType, MapType, VariantType}
+import org.apache.spark.sql.types.{DataType, FileType, MapType, VariantType}
/**
Review Comment:
The class scaladoc still omits FileType. The method comment lists Union as a
SetOperation; Union is UnionBase. UNION DISTINCT is rejected via Distinct (the
suite case works). UNION ALL is still allowed, same as VARIANT. Please fix the
comments rather than adding Union to the match.
##########
common/utils/src/main/resources/error/error-conditions.json:
##########
@@ -3008,6 +3013,48 @@
],
"sqlState" : "42704"
},
+ "FILE_TYPE_IN_ROUTINE_SIGNATURE_UNSUPPORTED_MAP_KEY" : {
+ "message" : [
+ "The FILE type cannot be a map key in the return type of <routineType>
<routineName>. Use FILE only as a map value, array element, or struct field."
+ ],
+ "sqlState" : "42601"
+ },
+ "FILE_TYPE_IN_ROUTINE_SIGNATURE_UNSUPPORTED_QUALIFIER" : {
+ "message" : [
+ "The FILE type in <routineType> <routineName> does not support the
EXTERNAL qualifier. Use FILE without a qualifier."
+ ],
+ "sqlState" : "42601"
+ },
+ "FILE_TYPE_MISSING_QUALIFIER" : {
+ "message" : [
+ "FILE type requires the EXTERNAL qualifier in a column definition. Use
FILE EXTERNAL."
+ ],
+ "sqlState" : "42601"
+ },
+ "FILE_TYPE_UNKNOWN_WRITE" : {
+ "message" : [
+ "Column(s) <colNames> have an ambiguous FILE type. Declare the columns
as FILE EXTERNAL in the target table before writing."
Review Comment:
`FILE_TYPE_UNKNOWN_WRITE` says "Declare the columns as FILE EXTERNAL in the
target table before writing." The only caller is CTAS
(`checkNoUnknownFileTypeWrite` on V2CreateTableAsSelectPlan and V1
PreprocessTableCreation). CTAS takes its schema from the query, so the useful
hint is CAST(... AS FILE EXTERNAL). INSERT is not covered; UpCastRule still
rejects FILE vs FILE EXTERNAL with a generic mismatch because mode is part of
equals. Please either use this error on writes too and fix the CTAS wording, or
split the messages accordingly.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala:
##########
@@ -5173,6 +5184,37 @@ private[sql] object QueryCompilationErrors extends
QueryErrorsBase with Compilat
messageParameters = Map.empty)
}
+ def fileTypeUnknownWriteError(colNames: Seq[String]): Throwable = {
+ new AnalysisException(
+ errorClass = "FILE_TYPE_UNKNOWN_WRITE",
+ messageParameters = Map(
+ "colNames" -> colNames.map(quoteIdentifier).mkString("[", ", ", "]")))
+ }
+
+ def fileTypeUnsupportedColumnUsageError(field: StructField, usage: String):
Throwable = {
+ new AnalysisException(
+ errorClass = s"FILE_TYPE_UNSUPPORTED_COLUMN_USAGE.$usage",
+ messageParameters = Map(
+ "colName" -> toSQLId(field.name),
+ "dataType" -> toSQLType(field.dataType)))
+ }
+
+ def fileTypeInRoutineSignatureUnsupportedQualifierError(
+ routineType: String,
+ routineName: String): Throwable = {
+ new AnalysisException(
+ errorClass = "FILE_TYPE_IN_ROUTINE_SIGNATURE_UNSUPPORTED_QUALIFIER",
+ messageParameters = Map("routineType" -> routineType, "routineName" ->
routineName))
+ }
+
Review Comment:
Please quote routine names in
`fileTypeInRoutineSignatureUnsupportedQualifierError` / `…MapKeyError`
(toSQLId). CREATE FUNCTION already passes a quoted id; spark.udf.register /
registerPython pass the raw name, so the tests expect `fn` vs file_udf.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
