voonhous commented on code in PR #18065:
URL: https://github.com/apache/hudi/pull/18065#discussion_r2840967886
##########
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieInternalRowUtils.scala:
##########
@@ -398,11 +399,47 @@ object HoodieInternalRowUtils {
(fieldUpdater, ordinal, value) =>
fieldUpdater.set(ordinal,
CatalystTypeConverters.convertToCatalyst(java.sql.Date.valueOf(value.toString)))
+ // Handle conversion from VariantType to variant struct representation
+ case (newStructType: StructType, _) if
sparkAdapter.isVariantType(prevDataType) &&
looksLikeVariantStruct(newStructType) =>
+ (fieldUpdater, ordinal, value) => {
+ if (value == null) {
+ fieldUpdater.setNullAt(ordinal)
+ } else {
+ val row = sparkAdapter.convertVariantToStruct(value, newStructType)
+ fieldUpdater.set(ordinal, row)
+ }
+ }
+
+ // Handle conversion from variant struct representation to VariantType
+ case (_, prevStructType: StructType) if
sparkAdapter.isVariantType(newDataType) &&
looksLikeVariantStruct(prevStructType) =>
+ (fieldUpdater, ordinal, value) => {
+ if (value == null) {
+ fieldUpdater.setNullAt(ordinal)
+ } else {
+ val row = value.asInstanceOf[InternalRow]
+ val variant = sparkAdapter.convertStructToVariant(row,
prevStructType)
+ fieldUpdater.set(ordinal, variant)
+ }
+ }
+
case (_, _) =>
throw new IllegalArgumentException(s"$prevDataType and $newDataType
are incompatible")
}
}
+ /**
+ * Checks if a StructType looks like a variant representation (has value and
metadata binary fields).
+ * This is a structural check that doesn't rely on metadata, useful during
schema reconciliation
Review Comment:
https://github.com/apache/hudi/commit/bbbb8e777f2d6b19f70c7302b4d778abafeaf0ec#diff-05005766a1bf96c0f5246f8c1703e8bf19a5e2697bbf9d23b1a33c205e3ea3b5R431-R441
Leaving this code here as reference, i rebased and this code seemed to be
shifted somehwere.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
