This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 3ac06dde1 [test] Add test case for format read nested type with
purning (#4210)
3ac06dde1 is described below
commit 3ac06dde1663b48e25eaf1df5fffc915e32e33de
Author: Zouxxyy <[email protected]>
AuthorDate: Thu Sep 19 08:50:28 2024 +0800
[test] Add test case for format read nested type with purning (#4210)
---
.../apache/paimon/format/FormatReadWriteTest.java | 46 ++++++++++++++++++++--
1 file changed, 42 insertions(+), 4 deletions(-)
diff --git
a/paimon-common/src/test/java/org/apache/paimon/format/FormatReadWriteTest.java
b/paimon-common/src/test/java/org/apache/paimon/format/FormatReadWriteTest.java
index 062a75c4d..d393a9192 100644
---
a/paimon-common/src/test/java/org/apache/paimon/format/FormatReadWriteTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/format/FormatReadWriteTest.java
@@ -106,10 +106,6 @@ public abstract class FormatReadWriteTest {
GenericRow.of(1, 1L), GenericRow.of(2, 2L),
GenericRow.of(3, null));
}
- /**
- * Currently, Parquet format doesn't support nested row in array, so this
test handles Parquet
- * specially.
- */
@Test
public void testFullTypes() throws IOException {
RowType rowType = rowTypeForFullTypesTest();
@@ -133,6 +129,48 @@ public abstract class FormatReadWriteTest {
validateFullTypesResult(result.get(0), expected);
}
+ @Test
+ public void testNestedReadPruning() throws Exception {
+ FileFormat format = fileFormat();
+
+ RowType writeType =
+ DataTypes.ROW(
+ DataTypes.FIELD(0, "f0", DataTypes.INT()),
+ DataTypes.FIELD(
+ 1,
+ "f1",
+ DataTypes.ROW(
+ DataTypes.FIELD(2, "f0",
DataTypes.INT()),
+ DataTypes.FIELD(3, "f1",
DataTypes.INT()),
+ DataTypes.FIELD(4, "f2",
DataTypes.INT()))));
+
+ try (PositionOutputStream out = fileIO.newOutputStream(file, false);
+ FormatWriter writer =
format.createWriterFactory(writeType).create(out, "zstd")) {
+ writer.addElement(GenericRow.of(0, GenericRow.of(10, 11, 12)));
+ }
+
+ // skip read f0, f1.f1
+ RowType readType =
+ DataTypes.ROW(
+ DataTypes.FIELD(
+ 1,
+ "f1",
+ DataTypes.ROW(
+ DataTypes.FIELD(2, "f0",
DataTypes.INT()),
+ DataTypes.FIELD(4, "f2",
DataTypes.INT()))));
+
+ List<InternalRow> result = new ArrayList<>();
+ try (RecordReader<InternalRow> reader =
+ format.createReaderFactory(readType)
+ .createReader(
+ new FormatReaderContext(fileIO, file,
fileIO.getFileSize(file)))) {
+ InternalRowSerializer serializer = new
InternalRowSerializer(readType);
+ reader.forEachRemaining(row -> result.add(serializer.copy(row)));
+ }
+
+ assertThat(result).containsExactly(GenericRow.of(GenericRow.of(10,
12)));
+ }
+
private RowType rowTypeForFullTypesTest() {
RowType.Builder builder =
RowType.builder()
