This is an automated email from the ASF dual-hosted git repository. lzljs3620320 pushed a commit to branch release-1.0 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit e1b70da24ccbaeb202f83fde05d51aac0b9d63af Author: JingsongLi <[email protected]> AuthorDate: Fri Jan 24 11:05:58 2025 +0800 [parquet] Parquet ColumnarBatch should return ColumnarRowIterator for nested schema --- .../format/parquet/newreader/ColumnarBatch.java | 20 ++++++++++++++++++-- .../format/parquet/ParquetColumnVectorTest.java | 7 ------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/ColumnarBatch.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/ColumnarBatch.java index 5d8ac09789..6bf00c106c 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/ColumnarBatch.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/newreader/ColumnarBatch.java @@ -18,24 +18,40 @@ package org.apache.paimon.format.parquet.newreader; +import org.apache.paimon.data.columnar.ArrayColumnVector; import org.apache.paimon.data.columnar.ColumnVector; import org.apache.paimon.data.columnar.ColumnarRow; +import org.apache.paimon.data.columnar.ColumnarRowIterator; +import org.apache.paimon.data.columnar.MapColumnVector; +import org.apache.paimon.data.columnar.RowColumnVector; import org.apache.paimon.data.columnar.VectorizedColumnBatch; import org.apache.paimon.data.columnar.VectorizedRowIterator; import org.apache.paimon.fs.Path; +import java.util.Arrays; + /** A batch of rows in columnar format. */ public class ColumnarBatch { protected final ColumnVector[] columns; protected final VectorizedColumnBatch vectorizedColumnBatch; - protected final VectorizedRowIterator vectorizedRowIterator; + protected final ColumnarRowIterator vectorizedRowIterator; public ColumnarBatch(Path filePath, ColumnVector[] columns) { this.columns = columns; this.vectorizedColumnBatch = new VectorizedColumnBatch(columns); + boolean containsNestedColumn = + Arrays.stream(columns) + .anyMatch( + vector -> + vector instanceof MapColumnVector + || vector instanceof RowColumnVector + || vector instanceof ArrayColumnVector); + ColumnarRow row = new ColumnarRow(vectorizedColumnBatch); this.vectorizedRowIterator = - new VectorizedRowIterator(filePath, new ColumnarRow(vectorizedColumnBatch), null); + containsNestedColumn + ? new ColumnarRowIterator(filePath, row, null) + : new VectorizedRowIterator(filePath, row, null); } /** Reset next record position and return self. */ diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetColumnVectorTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetColumnVectorTest.java index 82714e4785..ee20945d64 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetColumnVectorTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetColumnVectorTest.java @@ -25,7 +25,6 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalMap; import org.apache.paimon.data.InternalRow; -import org.apache.paimon.data.columnar.BytesColumnVector; import org.apache.paimon.data.columnar.ColumnVector; import org.apache.paimon.data.columnar.ColumnarRowIterator; import org.apache.paimon.format.FormatReaderContext; @@ -54,7 +53,6 @@ import java.util.Map; import java.util.Random; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; -import java.util.function.BiFunction; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -67,11 +65,6 @@ public class ParquetColumnVectorTest { private @TempDir java.nio.file.Path tempDir; private static final Random RND = ThreadLocalRandom.current(); - private static final BiFunction<ColumnVector, Integer, String> BYTES_COLUMN_VECTOR_STRING_FUNC = - (cv, i) -> - cv.isNullAt(i) - ? "null" - : new String(((BytesColumnVector) cv).getBytes(i).getBytes()); @Test public void testNormalStrings() throws IOException {
