voonhous commented on code in PR #19687:
URL: https://github.com/apache/hudi/pull/19687#discussion_r3877862792
##########
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHiveHoodieReaderContext.java:
##########
@@ -160,4 +180,179 @@ private static HoodieSchema getBaseSchema() {
private ArrayWritable createBaseRecord(Writable[] values) {
return new ArrayWritable(Writable.class, values);
}
+
+ @Test
+ void getFileRecordIteratorFailsFastOnShreddedVariantColumn(@TempDir
java.nio.file.Path tempDir) throws Exception {
+ // The Hive reader hands base files to a plain parquet-avro read at the
requested
+ // {metadata, value} projection; a shredded file would come back with
silent nulls (the
+ // payload of typed rows lives in typed_value, which the projection
drops). The footer is
+ // already read for schema pruning, so the shredded shape must fail fast
instead.
+ StoragePath filePath =
InputFormatTestUtil.writeVariantParquetFile(tempDir, "shredded.parquet", true);
+ HoodieSchema tableSchema = tableSchemaWithVariant();
+ HiveHoodieReaderContext readerContext = newReaderContext();
+ HoodieStorage storage = HoodieStorageUtils.getStorage(filePath,
storageConfiguration);
+ requestColumns("id", "v");
+
+ HoodieException failure = assertThrows(HoodieException.class, () ->
+ readerContext.getFileRecordIterator(filePath, 0, Long.MAX_VALUE,
tableSchema, tableSchema, storage));
+ assertTrue(failure.getMessage().contains("shredded variant") &&
failure.getMessage().contains("'v'"),
+ "The error must name the shredded variant column, got: " +
failure.getMessage());
+
+ // A query that does not project the variant column (`select id`) stays
readable.
+ HoodieSchema withoutVariant = HoodieSchema.createRecord("TestRecord",
null, null, Collections.singletonList(
+ HoodieSchemaField.of("id",
HoodieSchema.create(HoodieSchemaType.INT))));
+ requestColumns("id");
+ when(readerCreator.getRecordReader(any(), any(), any()))
+ .thenReturn((RecordReader<NullWritable, ArrayWritable>)
mock(RecordReader.class));
+ assertDoesNotThrow(() ->
+ readerContext.getFileRecordIterator(filePath, 0, Long.MAX_VALUE,
tableSchema, withoutVariant, storage));
+ }
+
+ @Test
+ void getFileRecordIteratorFailsOnShreddedVariantReadForMerging(@TempDir
java.nio.file.Path tempDir) throws Exception {
+ // The required schema can be wider than the query: a CUSTOM merge whose
merger is not
+ // projection compatible reads the whole table schema for merging, so
`select id` reaches the
+ // context asking for the variant column too. Hive's read column names
split the flagged
+ // columns: the ones Hive selected fail as Hive-visible nulls, the ones
only merging needs fail
+ // as well, since setSchemas materializes them at {metadata, value} for
the merger. count(*)
+ // names no column and its requested schema is empty, so nothing is
flagged and it reads.
+ StoragePath filePath =
InputFormatTestUtil.writeVariantParquetFile(tempDir, "shredded.parquet", true);
+ HoodieSchema tableSchema = tableSchemaWithVariant();
+ HiveHoodieReaderContext readerContext = newReaderContext();
+ HoodieStorage storage = HoodieStorageUtils.getStorage(filePath,
storageConfiguration);
+ when(readerCreator.getRecordReader(any(), any(), any()))
+ .thenReturn((RecordReader<NullWritable, ArrayWritable>)
mock(RecordReader.class));
+
+ requestColumns("id", "v");
+ HoodieException hiveVisible = assertThrows(HoodieException.class, () ->
+ readerContext.getFileRecordIterator(filePath, 0, Long.MAX_VALUE,
tableSchema, tableSchema, storage));
+ assertTrue(hiveVisible.getMessage().contains("'v'") &&
!hiveVisible.getMessage().contains("for merging"),
+ "select * must fail as a Hive-visible read of the shredded variant
column, got: " + hiveVisible.getMessage());
+
+ requestColumns("id");
+ HoodieException mergeOnly = assertThrows(HoodieException.class, () ->
+ readerContext.getFileRecordIterator(filePath, 0, Long.MAX_VALUE,
tableSchema, tableSchema, storage));
+ assertTrue(mergeOnly.getMessage().contains("'v'") &&
mergeOnly.getMessage().contains("for merging"),
+ "A variant only the merge reads must fail as a merge read, got: " +
mergeOnly.getMessage());
+ verify(readerCreator, never()).getRecordReader(any(), any(), any());
+
+ // count(*): Hive names no column, and createRequestedSchema turns that
into an empty record.
+ requestColumns();
Review Comment:
Replaced. The leg now passes `tableSchema` as the required schema with an
empty Hive name list, so `v` lands in the merge-only bucket and fails `for
merging`; the same read passes under skip-merge (see the sibling thread). The
empty-schema short-circuit stays pinned by
`getFileRecordIteratorFailsFastOnShreddedVariantColumn`, and the test's comment
says so. Fixed in 725f0269ac0d.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
