voonhous commented on code in PR #18961:
URL: https://github.com/apache/hudi/pull/18961#discussion_r3828459043
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestVariantDataType.scala:
##########
@@ -1192,6 +1377,38 @@ class TestVariantDataType extends HoodieSparkSqlTestBase
{
}
}
+ /**
+ * Pins an INFERRED shredding layout of `column` across every data parquet
file of the table:
+ * see [[assertInferredTypedValueIn]].
+ */
+ private def assertInferredTypedValue(tablePath: String, column: String, leg:
String,
+ present: Seq[String], absent:
Seq[String] = Seq.empty): Unit = {
+ assertInferredTypedValueIn(listDataParquetFiles(tablePath), column, leg,
present, absent)
+ }
+
+ /**
+ * Pins an INFERRED shredding layout of `column` in the given parquet files:
the variant group is
+ * shredded, carries the VARIANT logical type (the inferrer only exists on
Spark 4.1+, whose
+ * parquet ships the annotation) and its typed_value has every `present`
member and none of
+ * the `absent` ones (e.g. an avro-illegal key the inferrer dropped).
+ */
+ private def assertInferredTypedValueIn(files: Seq[String], column: String,
leg: String,
+ present: Seq[String], absent:
Seq[String] = Seq.empty): Unit = {
+ assert(files.nonEmpty, s"[$leg] should have at least one data parquet
file")
+ files.foreach { filePath =>
+ val variantGroup = getFieldAsGroup(readParquetSchema(filePath), column)
+ assert(variantGroup.containsField("typed_value"),
+ s"[$leg] $column should be shredded with an inferred typed_value.
File: $filePath Schema:\n$variantGroup")
+
assert(Option(variantGroup.getLogicalTypeAnnotation).exists(_.toString.contains("VARIANT")),
Review Comment:
Added: every pinned file must now show a non-null count on a typed_value
leaf in the block column statistics.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
