andygrove commented on code in PR #1385:
URL: https://github.com/apache/datafusion-comet/pull/1385#discussion_r1985765346
##########
spark/src/test/scala/org/apache/comet/CometCastSuite.scala:
##########
@@ -1126,27 +1129,33 @@ class CometCastSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
val cometMessage =
if (cometException.getCause != null)
cometException.getCause.getMessage
else cometException.getMessage
- if (CometSparkSessionExtensions.isSpark40Plus) {
- // for Spark 4 we expect to sparkException carries the message
- assert(
- sparkException.getMessage
- .replace(".WITH_SUGGESTION] ", "]")
- .startsWith(cometMessage))
- } else if (CometSparkSessionExtensions.isSpark34Plus) {
- // for Spark 3.4 we expect to reproduce the error message
exactly
- assert(cometMessage == sparkMessage)
+ // for comet decimal conversion throws ArrowError(string) from
arrow - across spark versions the message dont match.
+ if (sparkMessage.contains("cannot be represented as")) {
+ cometMessage.contains("cannot be represented as") ||
cometMessage.contains(
+ "too large to store")
} else {
- // for Spark 3.3 we just need to strip the prefix from the
Comet message
- // before comparing
- val cometMessageModified = cometMessage
- .replace("[CAST_INVALID_INPUT] ", "")
- .replace("[CAST_OVERFLOW] ", "")
- .replace("[NUMERIC_VALUE_OUT_OF_RANGE] ", "")
-
- if (sparkMessage.contains("cannot be represented as")) {
- assert(cometMessage.contains("cannot be represented as"))
+ if (CometSparkSessionExtensions.isSpark40Plus) {
+ // for Spark 4 we expect to sparkException carries the
message
+ assert(
+ sparkException.getMessage
+ .replace(".WITH_SUGGESTION] ", "]")
+ .startsWith(cometMessage))
+ } else if (CometSparkSessionExtensions.isSpark34Plus) {
+ // for Spark 3.4 we expect to reproduce the error message
exactly
+ assert(cometMessage == sparkMessage)
} else {
- assert(cometMessageModified == sparkMessage)
+ // for Spark 3.3 we just need to strip the prefix from the
Comet message
Review Comment:
note that we plan on dropping 3.x support soon so can simplify the code here
once we do that
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
