pvary commented on code in PR #15675:
URL: https://github.com/apache/iceberg/pull/15675#discussion_r2975675637
##########
data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java:
##########
@@ -628,6 +638,390 @@ void
testReaderBuilderRecordsPerBatchNotSupported(FileFormat fileFormat) throws
.isInstanceOf(UnsupportedOperationException.class);
}
+ @ParameterizedTest
+ @FieldSource("FILE_FORMATS")
+ void testReadMetadataColumnsFilePathAndSpecId(FileFormat fileFormat) throws
IOException {
+
+ DataGenerator dataGenerator = new DataGenerators.DefaultSchema();
+ Schema schema = dataGenerator.schema();
+ List<Record> genericRecords = dataGenerator.generateRecords();
+ writeGenericRecords(fileFormat, schema, genericRecords);
+
+ String filePath = "test-data-file.parquet";
+ int specId = 0;
+ Schema projectionSchema = new Schema(MetadataColumns.FILE_PATH,
MetadataColumns.SPEC_ID);
+
+ Map<Integer, Object> idToConstant =
+ ImmutableMap.of(
+ MetadataColumns.FILE_PATH.fieldId(), filePath,
+ MetadataColumns.SPEC_ID.fieldId(), specId);
+
+ InputFile inputFile = encryptedFile.encryptingOutputFile().toInputFile();
+ List<T> readRecords;
+ try (CloseableIterable<T> reader =
+ FormatModelRegistry.readBuilder(fileFormat, engineType(), inputFile)
+ .project(projectionSchema)
+ .engineProjection(engineSchema(projectionSchema))
+ .idToConstant(convertConstantsToEngine(projectionSchema,
idToConstant))
+ .build()) {
+ readRecords = ImmutableList.copyOf(reader);
+ }
+
+ List<Record> expected =
+ IntStream.range(0, genericRecords.size())
+ .mapToObj(
+ i ->
+ GenericRecord.create(projectionSchema)
+ .copy(
+ MetadataColumns.FILE_PATH.name(), filePath,
+ MetadataColumns.SPEC_ID.name(), specId))
+ .toList();
+
+ assertThat(readRecords).hasSize(genericRecords.size());
+ assertEquals(projectionSchema, convertToEngineRecords(expected,
projectionSchema), readRecords);
+ }
+
+ @ParameterizedTest
+ @FieldSource("FILE_FORMATS")
+ void testReadMetadataColumnRowPosition(FileFormat fileFormat) throws
IOException {
+
+ DataGenerator dataGenerator = new DataGenerators.DefaultSchema();
+ Schema schema = dataGenerator.schema();
+ List<Record> genericRecords = dataGenerator.generateRecords();
+ writeGenericRecords(fileFormat, schema, genericRecords);
+
+ Schema projectionSchema = new Schema(MetadataColumns.ROW_POSITION);
+
+ InputFile inputFile = encryptedFile.encryptingOutputFile().toInputFile();
+ List<T> readRecords;
+ try (CloseableIterable<T> reader =
+ FormatModelRegistry.readBuilder(fileFormat, engineType(), inputFile)
+ .project(projectionSchema)
+ .engineProjection(engineSchema(projectionSchema))
+ .build()) {
+ readRecords = ImmutableList.copyOf(reader);
+ }
+
+ List<Record> expected =
+ IntStream.range(0, genericRecords.size())
+ .mapToObj(
+ i ->
+ GenericRecord.create(projectionSchema)
+ .copy(MetadataColumns.ROW_POSITION.name(), (long) i))
+ .toList();
+
+ assertThat(readRecords).hasSize(genericRecords.size());
+ assertEquals(projectionSchema, convertToEngineRecords(expected,
projectionSchema), readRecords);
+ }
+
+ @ParameterizedTest
+ @FieldSource("FILE_FORMATS")
+ void testReadMetadataColumnIsDeleted(FileFormat fileFormat) throws
IOException {
+
+ DataGenerator dataGenerator = new DataGenerators.DefaultSchema();
+ Schema schema = dataGenerator.schema();
+ List<Record> genericRecords = dataGenerator.generateRecords();
+ writeGenericRecords(fileFormat, schema, genericRecords);
+
+ Schema projectionSchema = new Schema(MetadataColumns.IS_DELETED);
+
+ InputFile inputFile = encryptedFile.encryptingOutputFile().toInputFile();
+ List<T> readRecords;
+ try (CloseableIterable<T> reader =
+ FormatModelRegistry.readBuilder(fileFormat, engineType(), inputFile)
+ .project(projectionSchema)
+ .engineProjection(engineSchema(projectionSchema))
+ .build()) {
+ readRecords = ImmutableList.copyOf(reader);
+ }
+
+ List<Record> expected =
+ IntStream.range(0, genericRecords.size())
+ .mapToObj(
+ i ->
+ GenericRecord.create(projectionSchema)
+ .copy(MetadataColumns.IS_DELETED.name(), false))
+ .toList();
+
+ assertThat(readRecords).hasSize(genericRecords.size());
+ assertEquals(projectionSchema, convertToEngineRecords(expected,
projectionSchema), readRecords);
+ }
+
+ @ParameterizedTest
+ @FieldSource("FILE_FORMATS")
+ void testReadMetadataColumnRowLinage(FileFormat fileFormat) throws
IOException {
+ assumeSupports(fileFormat, FEATURE_META_ROW_LINEAGE);
+
+ DataGenerator dataGenerator = new DataGenerators.DefaultSchema();
+ Schema schema = dataGenerator.schema();
+ List<Record> genericRecords = dataGenerator.generateRecords();
+ writeGenericRecords(fileFormat, schema, genericRecords);
+
+ long baseRowId = 100L;
+ long fileSeqNumber = 5L;
+ Schema projectionSchema =
+ new Schema(MetadataColumns.ROW_ID,
MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER);
+
+ Map<Integer, Object> idToConstant =
+ ImmutableMap.of(
+ MetadataColumns.ROW_ID.fieldId(), baseRowId,
+ MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(),
fileSeqNumber);
+
+ InputFile inputFile = encryptedFile.encryptingOutputFile().toInputFile();
+ List<T> readRecords;
+ try (CloseableIterable<T> reader =
+ FormatModelRegistry.readBuilder(fileFormat, engineType(), inputFile)
+ .project(projectionSchema)
+ .engineProjection(engineSchema(projectionSchema))
+ .idToConstant(convertConstantsToEngine(projectionSchema,
idToConstant))
+ .build()) {
+ readRecords = ImmutableList.copyOf(reader);
+ }
+
+ List<Record> expected =
+ IntStream.range(0, genericRecords.size())
+ .mapToObj(
+ i ->
+ GenericRecord.create(projectionSchema)
+ .copy(
+ MetadataColumns.ROW_ID.name(),
+ baseRowId + i,
+
MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.name(),
+ fileSeqNumber))
+ .toList();
+
+ assertThat(readRecords).hasSize(genericRecords.size());
+ assertEquals(projectionSchema, convertToEngineRecords(expected,
projectionSchema), readRecords);
+ }
+
+ @ParameterizedTest
+ @FieldSource("FILE_FORMATS")
+ void testReadMetadataColumnPartitionIdentity(FileFormat fileFormat) throws
IOException {
+
+ DataGenerator dataGenerator = new DataGenerators.DefaultSchema();
+ PartitionSpec spec =
PartitionSpec.builderFor(dataGenerator.schema()).identity("col_a").build();
+
+ Types.StructType partitionType = spec.partitionType();
+ PartitionData partitionData = new PartitionData(partitionType);
+ partitionData.set(0, "test_col_a");
+
+ DataWriter<Record> writer =
+ FormatModelRegistry.dataWriteBuilder(fileFormat, Record.class,
encryptedFile)
Review Comment:
Does the writer remove the partition columns? If so, then we need these
tests, but this is more like a writer test
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
