voonhous commented on code in PR #19558:
URL: https://github.com/apache/hudi/pull/19558#discussion_r3755268506
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestVariantDataType.scala:
##########
@@ -253,6 +253,201 @@ class TestVariantDataType extends HoodieSparkSqlTestBase {
})
}
+ test("Test COW clustering preserves VARIANT values") {
+ // Same Spark 4.1 gate as the compaction test above: clustering reads the
shredded
+ // base files back through the native reader, which rejects the 3-field
shredded
+ // layout before SPARK-54410 (Spark 4.1+).
+ assume(HoodieSparkUtils.gteqSpark4_1, "Shredded variant base-file read
requires Spark 4.1 or higher")
+
+ withRecordType()(withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = tmp.getCanonicalPath
+ // Clustering rewrites ALL rows of the clustered file groups through the
internal
+ // write-side reader context (SparkReaderContextFactory ->
+ // SparkFileFormatInternalRowReaderContext), the stack whose blob
handling silently
+ // lost bytes in #19232. Nothing pinned its VARIANT behavior: this is
the first
+ // clustering coverage for the type. Shredding is forced so the rewrite
reads and
+ // rewrites the shredded layout, the default in production.
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | v variant,
+ | ts long
+ |) using hudi
+ | location '$tablePath'
+ | tblproperties (
+ | primaryKey = 'id',
+ | type = 'cow',
+ | preCombineField = 'ts',
+ | hoodie.parquet.variant.write.shredding.enabled = 'true',
+ | hoodie.parquet.variant.force.shredding.schema.for.test = 'key
string',
Review Comment:
Addressed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
