felipepessoto commented on code in PR #12444:
URL: https://github.com/apache/gluten/pull/12444#discussion_r3549894999
##########
backends-velox/src-delta40/main/scala/org/apache/spark/sql/delta/GlutenOptimisticTransaction.scala:
##########
@@ -48,6 +48,14 @@ class GlutenOptimisticTransaction(delegate:
OptimisticTransaction)
writeOptions: Option[DeltaOptions],
isOptimize: Boolean,
additionalConstraints: Seq[Constraint]): Seq[FileAction] = {
+ // Velox cannot write every Spark type: the native write path inserts a
RowToVeloxColumnarExec
+ // transition whose SparkArrowUtil.toArrowSchema call throws (e.g.
`Unsupported data type:
+ // variant`) for any type it has no Arrow mapping for. Reuse the backend's
schema validator so
+ // such writes fall back to the vanilla Delta write path instead of
failing at runtime.
+ if (VeloxValidatorApi.validateSchema(inputData.schema).isDefined) {
+ logInfo("Schema is not supported by Velox; falling back to the vanilla
Delta write path.")
+ return super.writeFiles(inputData, writeOptions, isOptimize,
additionalConstraints)
+ }
Review Comment:
Accepted
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
