andygrove commented on code in PR #5079:
URL: https://github.com/apache/datafusion-comet/pull/5079#discussion_r3668124328
##########
spark/src/main/scala/org/apache/comet/expressions/CometCast.scala:
##########
@@ -25,30 +25,41 @@ import org.apache.spark.sql.types.{ArrayType, DataType,
DataTypes, DecimalType,
import org.apache.comet.CometConf
import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus,
withFallbackReason}
-import org.apache.comet.serde.{CometExpressionSerde, Compatible,
ExprOuterClass, Incompatible, SupportLevel, Unsupported}
+import org.apache.comet.DataTypeSupport.isComplexType
+import org.apache.comet.serde.{CodegenDispatchFallback, CometExpressionSerde,
Compatible, ExprOuterClass, Incompatible, SupportLevel, Unsupported}
import org.apache.comet.serde.ExprOuterClass.Expr
import org.apache.comet.serde.QueryPlanSerde.{evalModeToProto,
exprToProtoInternal, serializeDataType}
-import org.apache.comet.shims.CometExprShim
+import org.apache.comet.shims.{CometExprShim, CometTypeShim}
-object CometCast extends CometExpressionSerde[Cast] with CometExprShim {
+object CometCast
+ extends CometExpressionSerde[Cast]
+ with CometExprShim
+ with CometTypeShim
+ with CodegenDispatchFallback {
// Shared with CometCastSuite so the asserted reason cannot drift from
production.
private[comet] val negativeScaleDecimalToStringReason: String =
"Negative-scale decimal requires
spark.sql.legacy.allowNegativeScaleOfDecimal=true"
- // When `spark.sql.legacy.castComplexTypesToString.enabled` is true, Spark
wraps maps and
- // structs with `[]` (instead of `{}`) when casting to string, and omits
NULL elements of
- // structs/maps/arrays (instead of rendering them as the literal "null").
Comet only
- // implements the default formatting, so fall back to Spark for any
array/map/struct to-string
- // cast when the flag is enabled. The flag is internal in Spark 4.0 and
defaults to false.
private[comet] val legacyCastComplexTypesToStringReason: String =
- "spark.sql.legacy.castComplexTypesToString.enabled=true is not supported"
+ "spark.sql.legacy.castComplexTypesToString.enabled=true is not supported
natively"
+
+ private[comet] val nonDefaultTimeParserPolicyReason: String =
+ "spark.sql.legacy.timeParserPolicy is set to a non-CORRECTED value; the
native " +
+ "string-to-datetime parser only implements CORRECTED semantics"
private def legacyCastComplexTypesToString: Boolean =
SQLConf.get
.getConfString("spark.sql.legacy.castComplexTypesToString.enabled",
"false")
.toBoolean
+ // Non-CORRECTED policies (LEGACY, EXCEPTION) change
string-to-date/timestamp parsing behavior in
+ // ways the native cast kernel does not replicate.
+ private def isNonDefaultTimeParserPolicy: Boolean =
+ !SQLConf.get
+ .getConfString("spark.sql.legacy.timeParserPolicy", "CORRECTED")
+ .equalsIgnoreCase("CORRECTED")
Review Comment:
Claude tells me that `timeParserPolicy` has no impact on cast operations.
Could you confirm this is really an issue?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
