This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/dev/pr-12978-8622bbdc883704faecb14e94bd73313291647438
in repository https://gitbox.apache.org/repos/asf/seatunnel.git

commit e388eaba95109d5af28fb8f5016b3f2728a3fa9b
Author: Goutam Adwant <[email protected]>
AuthorDate: Sat Nov 7 06:39:53 2026 +0000

    [Improve][Connector-V2] Prune unselected Parquet columns (#12978)
    
    Signed-off-by: Goutam Adwant <[email protected]>
---
 docs/en/connectors/source/LocalFile.md             |   4 +
 docs/zh/connectors/source/LocalFile.md             |   3 +
 .../file/source/reader/ParquetReadStrategy.java    | 101 +++-
 .../source/reader/ParquetReadStrategyTest.java     | 532 +++++++++++++++++++++
 .../e2e/connector/file/local/LocalFileIT.java      |   7 +
 .../local_file_parquet_projection_to_assert.conf   |   9 +-
 6 files changed, 639 insertions(+), 17 deletions(-)

diff --git a/docs/en/connectors/source/LocalFile.md 
b/docs/en/connectors/source/LocalFile.md
index 59f6a179810..8f6de728ca2 100644
--- a/docs/en/connectors/source/LocalFile.md
+++ b/docs/en/connectors/source/LocalFile.md
@@ -253,6 +253,10 @@ Note: Only single-column (top-to-bottom) PDF layouts are 
supported. Multi-column
 
 The read column list of the data source, user can use it to implement field 
projection.
 
+For Parquet files, the reader also skips unselected top-level column data. 
Selected nested fields
+are read in full, and output column order is unchanged. Explicit Hadoop 
Parquet projections, reader
+schemas and record filters retain their existing behavior and disable 
automatic column pruning.
+
 ### delimiter/field_delimiter [string]
 
 **delimiter** parameter will deprecate after version 2.3.5, please use 
**field_delimiter** instead.
diff --git a/docs/zh/connectors/source/LocalFile.md 
b/docs/zh/connectors/source/LocalFile.md
index 84c2c02d22c..7a8683e6012 100644
--- a/docs/zh/connectors/source/LocalFile.md
+++ b/docs/zh/connectors/source/LocalFile.md
@@ -253,6 +253,9 @@ PDF 特有的解析行为如下:
 
 数据源的读取列列表,用户可以使用它来实现字段投影。
 
+对于 Parquet 文件,读取器还会跳过未选择的顶层列数据。选中的嵌套字段仍完整读取,输出列顺序保持不变。
+显式配置 Hadoop Parquet 投影、读取模式或记录过滤器时,保持原有行为,不启用自动列裁剪。
+
 ### delimiter/field_delimiter [string]
 
 **delimiter** 参数将在 2.3.5 版本后弃用,请使用 **field_delimiter** 代替。
diff --git 
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategy.java
 
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategy.java
index 79246a14b5b..d21bdb157fe 100644
--- 
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategy.java
+++ 
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategy.java
@@ -42,20 +42,25 @@ import org.apache.avro.data.TimeConversions;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.avro.util.Utf8;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.parquet.HadoopReadOptions;
-import org.apache.parquet.avro.AvroParquetReader;
+import org.apache.parquet.avro.AvroReadSupport;
+import org.apache.parquet.avro.AvroSchemaConverter;
 import org.apache.parquet.example.data.Group;
 import org.apache.parquet.example.data.simple.NanoTime;
 import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.ParquetInputFormat;
 import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.hadoop.api.InitContext;
 import org.apache.parquet.hadoop.api.ReadSupport;
 import org.apache.parquet.hadoop.example.GroupReadSupport;
 import org.apache.parquet.hadoop.metadata.FileMetaData;
 import org.apache.parquet.hadoop.metadata.ParquetMetadata;
 import org.apache.parquet.io.InputFile;
 import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.RecordMaterializer;
 import org.apache.parquet.schema.GroupType;
 import org.apache.parquet.schema.LogicalTypeAnnotation;
 import org.apache.parquet.schema.MessageType;
@@ -78,8 +83,10 @@ import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.IntStream;
 
@@ -144,13 +151,19 @@ public class ParquetReadStrategy extends 
AbstractReadStrategy {
         final boolean useSplitRange =
                 enableSplitFile && split.getStart() >= 0 && split.getLength() 
> 0;
         GenericRecord record;
-        AvroParquetReader.Builder<GenericData.Record> builder =
-                AvroParquetReader.<GenericData.Record>builder(hadoopInputFile)
-                        .withDataModel(dataModel);
+        ParquetReader.Builder<GenericData.Record> builder =
+                new ProjectedParquetReaderBuilder<>(
+                        hadoopInputFile,
+                        new AvroReadSupport<>(dataModel),
+                        seaTunnelRowType.getFieldNames());
         // HadoopConf carries READ_INT96_AS_FIXED / ADD_LIST_ELEMENT_RECORDS. 
ParquetReader lifts
         // those automatically only from a HadoopInputFile, so set them 
explicitly here. withConf()
         // rebuilds the read options, so it must run before withFileRange() 
below.
-        builder.withConf(hadoopFileSystemProxy.getConfiguration());
+        Configuration readConfiguration =
+                new Configuration(hadoopFileSystemProxy.getConfiguration());
+        // AvroParquetReader's GenericData builder uses the non-compatible 
materializer.
+        readConfiguration.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, 
false);
+        builder.withConf(readConfiguration);
         if (useSplitRange) {
             long start = split.getStart();
             long end = start + split.getLength();
@@ -197,10 +210,12 @@ public class ParquetReadStrategy extends 
AbstractReadStrategy {
                             // HadoopInputFile.fromStatus), which leaks with 
the FS cache
                             // disabled. Bind to the connector's FileSystem 
instead.
                             ParquetReader.Builder<Group> builder =
-                                    new GroupParquetReaderBuilder(
+                                    new ProjectedParquetReaderBuilder<>(
                                             FileSystemInputFile.fromPath(
                                                     
hadoopFileSystemProxy.getFileSystem(),
-                                                    filePath));
+                                                    filePath),
+                                            new GroupReadSupport(),
+                                            seaTunnelRowType.getFieldNames());
                             // withConf() rebuilds the read options, so it 
must precede
                             // withFileRange() below.
                             
builder.withConf(hadoopFileSystemProxy.getConfiguration());
@@ -1017,20 +1032,76 @@ public class ParquetReadStrategy extends 
AbstractReadStrategy {
     }
 
     /**
-     * Binds the native (Group) Parquet reader to an already-open {@link
-     * org.apache.hadoop.fs.FileSystem}. {@code 
ParquetReader.Builder(InputFile)} is protected and
-     * there is no public {@code builder(ReadSupport, InputFile)}, so this 
mirrors what {@code
-     * AvroParquetReader.Builder} does for the Avro path.
+     * Binds a Parquet reader to an already-open {@link 
org.apache.hadoop.fs.FileSystem}. {@code
+     * ParquetReader.Builder(InputFile)} is protected and there is no public 
{@code
+     * builder(ReadSupport, InputFile)}, so this mirrors what {@code 
AvroParquetReader.Builder} does
+     * for the Avro path.
      */
-    private static final class GroupParquetReaderBuilder extends 
ParquetReader.Builder<Group> {
+    private static final class ProjectedParquetReaderBuilder<T> extends 
ParquetReader.Builder<T> {
+        private final ReadSupport<T> readSupport;
 
-        private GroupParquetReaderBuilder(InputFile file) {
+        private ProjectedParquetReaderBuilder(
+                InputFile file, ReadSupport<T> delegate, String[] fields) {
             super(file);
+            this.readSupport = new ProjectedReadSupport<>(delegate, fields);
         }
 
         @Override
-        protected ReadSupport<Group> getReadSupport() {
-            return new GroupReadSupport();
+        protected ReadSupport<T> getReadSupport() {
+            return readSupport;
+        }
+    }
+
+    /** Prunes whole top-level fields while retaining the library's decoding 
and schema metadata. */
+    private static final class ProjectedReadSupport<T> extends ReadSupport<T> {
+        private final ReadSupport<T> delegate;
+        private final Set<String> fields;
+
+        private ProjectedReadSupport(ReadSupport<T> delegate, String[] fields) 
{
+            this.delegate = delegate;
+            this.fields = new HashSet<>(Arrays.asList(fields));
+        }
+
+        @Override
+        public ReadContext init(InitContext context) {
+            ReadContext original = delegate.init(context);
+            Configuration configuration = context.getConfiguration();
+            // Explicit schemas and filters may require columns outside the 
output schema.
+            if (configuration.get(ReadSupport.PARQUET_READ_SCHEMA) != null
+                    || 
configuration.get(AvroReadSupport.AVRO_REQUESTED_PROJECTION) != null
+                    || configuration.get("parquet.avro.read.schema") != null
+                    || configuration.get(ParquetInputFormat.FILTER_PREDICATE) 
!= null
+                    || 
configuration.get(ParquetInputFormat.UNBOUND_RECORD_FILTER) != null) {
+                return original;
+            }
+            MessageType schema = original.getRequestedSchema();
+            List<Type> selected = new ArrayList<>();
+            for (Type field : schema.getFields()) {
+                if (fields.contains(field.getName())) {
+                    selected.add(field);
+                }
+            }
+            // Preserve cardinality/decoding for zero-column and all-missing 
output schemas.
+            if (selected.isEmpty() || selected.size() == 
schema.getFieldCount()) {
+                return original;
+            }
+            if (delegate instanceof AvroReadSupport
+                    && 
!context.getKeyValueMetadata().containsKey("parquet.avro.schema")
+                    && 
!context.getKeyValueMetadata().containsKey("avro.schema")) {
+                // Keep the native fallback for illegal names even in 
unselected file fields.
+                new 
AvroSchemaConverter(configuration).convert(context.getFileSchema());
+            }
+            return new ReadContext(
+                    new MessageType(schema.getName(), selected), 
original.getReadSupportMetadata());
+        }
+
+        @Override
+        public RecordMaterializer<T> prepareForRead(
+                Configuration configuration,
+                Map<String, String> metadata,
+                MessageType fileSchema,
+                ReadContext readContext) {
+            return delegate.prepareForRead(configuration, metadata, 
fileSchema, readContext);
         }
     }
 }
diff --git 
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategyTest.java
 
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategyTest.java
index 70ac9f92400..35a96492dae 100644
--- 
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategyTest.java
+++ 
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategyTest.java
@@ -31,6 +31,8 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
 import org.apache.seatunnel.api.table.type.SeaTunnelRow;
 import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
 import 
org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSourceOptions;
+import 
org.apache.seatunnel.connectors.seatunnel.file.source.split.FileSourceSplit;
+import 
org.apache.seatunnel.connectors.seatunnel.file.source.split.ParquetFileSplitStrategy;
 import org.apache.seatunnel.connectors.seatunnel.file.util.LocalFileSystemConf;
 
 import org.apache.avro.Conversions;
@@ -41,24 +43,38 @@ import org.apache.avro.generic.GenericFixed;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.avro.util.Utf8;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSInputStream;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
 import org.apache.parquet.avro.AvroParquetWriter;
+import org.apache.parquet.avro.AvroReadSupport;
 import org.apache.parquet.example.data.Group;
 import org.apache.parquet.example.data.simple.SimpleGroup;
+import org.apache.parquet.filter2.predicate.FilterApi;
 import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.ParquetInputFormat;
 import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.api.ReadSupport;
 import org.apache.parquet.hadoop.example.ExampleParquetWriter;
 import org.apache.parquet.hadoop.example.GroupWriteSupport;
 import org.apache.parquet.hadoop.metadata.CompressionCodecName;
 import org.apache.parquet.hadoop.util.HadoopInputFile;
+import org.apache.parquet.io.DelegatingPositionOutputStream;
+import org.apache.parquet.io.OutputFile;
+import org.apache.parquet.io.PositionOutputStream;
 import org.apache.parquet.schema.LogicalTypeAnnotation;
 import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.MessageTypeParser;
 import org.apache.parquet.schema.Types;
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.DisabledOnOs;
 import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 
 import lombok.extern.slf4j.Slf4j;
 
@@ -67,15 +83,22 @@ import java.io.IOException;
 import java.math.BigDecimal;
 import java.net.URL;
 import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.TimeZone;
+import java.util.concurrent.atomic.AtomicLong;
 
 import static 
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
 import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
@@ -83,6 +106,515 @@ import static 
org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
 
 @Slf4j
 public class ParquetReadStrategyTest {
+    @TempDir java.nio.file.Path projectionDirectory;
+
+    @Test
+    void testProjectionFixtureOutputPreservesPositionAndCreateMode() throws 
Exception {
+        Path path = new 
Path(projectionDirectory.resolve("output.parquet").toUri());
+        OutputFile file = new ProjectionOutputFile(path);
+        try (PositionOutputStream output = file.create(0)) {
+            output.write(1);
+            output.write(new byte[] {2, 3, 4}, 0, 2);
+            Assertions.assertEquals(3, output.getPos());
+        }
+        Assertions.assertThrows(FileAlreadyExistsException.class, () -> 
file.create(0));
+        try (PositionOutputStream output = file.createOrOverwrite(0)) {
+            Assertions.assertEquals(0, output.getPos());
+            output.write(new byte[] {5, 6});
+            Assertions.assertEquals(2, output.getPos());
+        }
+        Assertions.assertArrayEquals(
+                new byte[] {5, 6}, 
Files.readAllBytes(Paths.get(path.toUri())));
+    }
+
+    /** Writes projection fixtures without Hadoop's platform-specific local 
permission helpers. */
+    private static class ProjectionOutputFile implements OutputFile {
+        private final java.nio.file.Path path;
+
+        private ProjectionOutputFile(Path path) {
+            this.path = Paths.get(path.toUri());
+        }
+
+        @Override
+        public PositionOutputStream create(long blockSizeHint) throws 
IOException {
+            return open(StandardOpenOption.WRITE, 
StandardOpenOption.CREATE_NEW);
+        }
+
+        @Override
+        public PositionOutputStream createOrOverwrite(long blockSizeHint) 
throws IOException {
+            return open(
+                    StandardOpenOption.WRITE,
+                    StandardOpenOption.CREATE,
+                    StandardOpenOption.TRUNCATE_EXISTING);
+        }
+
+        private PositionOutputStream open(StandardOpenOption... options) 
throws IOException {
+            FileChannel channel = FileChannel.open(path, options);
+            return new 
DelegatingPositionOutputStream(Channels.newOutputStream(channel)) {
+                @Override
+                public long getPos() throws IOException {
+                    return channel.position();
+                }
+            };
+        }
+
+        @Override
+        public boolean supportsBlockSize() {
+            return false;
+        }
+
+        @Override
+        public long defaultBlockSize() {
+            return 0;
+        }
+    }
+
+    @Test
+    void testProjectionRetainsAvroSchemaMetadataAndEmptyInput() throws 
Exception {
+        Schema schema =
+                new Schema.Parser()
+                        .parse(
+                                "{\"type\":\"record\",\"name\":\"Named\","
+                                        + 
"\"namespace\":\"example\",\"fields\":[{\"name\":\"id\",\"type\":\"int\"},"
+                                        + 
"{\"name\":\"text\",\"type\":\"string\",\"aliases\":[\"old_text\"]},"
+                                        + 
"{\"name\":\"unused\",\"type\":[\"null\",\"string\"],\"default\":null}]}");
+        for (boolean empty : new boolean[] {false, true}) {
+            Path file = new Path(projectionDirectory.resolve("avro-" + empty + 
".parquet").toUri());
+            try (ParquetWriter<GenericData.Record> writer =
+                    AvroParquetWriter.<GenericData.Record>builder(new 
ProjectionOutputFile(file))
+                            .withSchema(schema)
+                            
.withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+                            .build()) {
+                if (!empty) {
+                    GenericData.Record record = new GenericData.Record(schema);
+                    record.put("id", 7);
+                    record.put("text", "value");
+                    record.put("unused", null);
+                    writer.write(record);
+                }
+            }
+            ParquetReadStrategy strategy = new ParquetReadStrategy();
+            try {
+                strategy.init(new LocalFileSystemConf.LocalConf("file:///"));
+                strategy.setPluginConfig(
+                        ConfigFactory.parseString(
+                                
"read_columns=[text,id]\nparse_partition_from_path=false"));
+                strategy.getSeaTunnelRowTypeInfo(file.toString());
+                TestCollector collector = new TestCollector();
+                strategy.read(file.toString(), "", collector);
+                Assertions.assertEquals(empty ? 0 : 1, 
collector.getRows().size());
+                if (!empty) {
+                    Assertions.assertArrayEquals(
+                            new Object[] {"value", 7}, 
collector.getRows().get(0).getFields());
+                }
+            } finally {
+                strategy.close();
+            }
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(booleans = {false, true})
+    void testProjectionRetainsLogicalSplitBoundaries(boolean nativeReader) 
throws Exception {
+        String id = nativeReader ? "id-value" : "id";
+        MessageType schema =
+                Types.buildMessage()
+                        .required(INT32)
+                        .named(id)
+                        .required(BINARY)
+                        .named("unused")
+                        .named("split");
+        Path file = new 
Path(projectionDirectory.resolve("split.parquet").toUri());
+        try (ParquetWriter<Group> writer =
+                ExampleParquetWriter.builder(new ProjectionOutputFile(file))
+                        .withType(schema)
+                        
.withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+                        .withDictionaryEncoding(false)
+                        .withRowGroupSize(1024)
+                        .withMinRowCountForPageSizeCheck(1)
+                        .withMaxRowCountForPageSizeCheck(1)
+                        .build()) {
+            for (int i = 0; i < 40; i++) {
+                writer.write(
+                        new SimpleGroup(schema).append(id, i).append("unused", 
"x".repeat(1024)));
+            }
+        }
+        List<FileSourceSplit> splits;
+        try (ParquetFileReader fileReader =
+                ParquetFileReader.open(HadoopInputFile.fromPath(file, new 
Configuration()))) {
+            Assertions.assertTrue(fileReader.getFooter().getBlocks().size() > 
1);
+            splits = new ParquetFileSplitStrategy(1).split("table", 
file.toString());
+        }
+        ParquetReadStrategy strategy = new ParquetReadStrategy();
+        try {
+            strategy.init(new LocalFileSystemConf.LocalConf("file:///"));
+            strategy.setPluginConfig(
+                    ConfigFactory.parseString(
+                            "read_columns=[\""
+                                    + id
+                                    + 
"\"]\nparse_partition_from_path=false\nenable_file_split=true"));
+            strategy.getSeaTunnelRowTypeInfo(file.toString());
+            TestCollector collector = new TestCollector();
+            for (FileSourceSplit split : splits) {
+                strategy.read(split, collector);
+            }
+            Assertions.assertEquals(40, collector.getRows().size());
+            for (int i = 0; i < 40; i++) {
+                Assertions.assertEquals(i, 
collector.getRows().get(i).getField(0));
+                Assertions.assertEquals("table", 
collector.getRows().get(i).getTableId());
+            }
+        } finally {
+            strategy.close();
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(
+            strings = {
+                "parquet.avro.projection",
+                "parquet.avro.read.schema",
+                "parquet.read.schema"
+            })
+    void testProjectionPreservesExplicitReaderSchemas(String option) throws 
Exception {
+        boolean nativeReader = option.equals(ReadSupport.PARQUET_READ_SCHEMA);
+        String id = nativeReader ? "id-value" : "id";
+        MessageType schema =
+                Types.buildMessage()
+                        .required(INT32)
+                        .named(id)
+                        .required(INT32)
+                        .named("unused")
+                        .named("explicit");
+        Path file = new 
Path(projectionDirectory.resolve("explicit.parquet").toUri());
+        try (ParquetWriter<Group> writer =
+                ExampleParquetWriter.builder(new ProjectionOutputFile(file))
+                        .withType(schema)
+                        
.withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+                        .build()) {
+            writer.write(new SimpleGroup(schema).append(id, 
1).append("unused", 2));
+        }
+        String explicit =
+                nativeReader
+                        ? "message explicit { required int32 unused; }"
+                        : 
"{\"type\":\"record\",\"name\":\"explicit\",\"fields\":[{\"name\":\"unused\",\"type\":\"int\"}]}";
+        boolean readSchema = option.equals("parquet.avro.read.schema");
+        if (readSchema) {
+            explicit =
+                    
"{\"type\":\"record\",\"name\":\"explicit\",\"fields\":[{\"name\":\"id\",\"type\":\"int\"},{\"name\":\"unused\",\"type\":\"int\"}]}";
+        }
+        LocalFileSystemConf.LocalConf conf = new 
LocalFileSystemConf.LocalConf("file:///");
+        conf.getExtraOptions().put(option, explicit);
+        ParquetReadStrategy strategy = new ParquetReadStrategy();
+        try {
+            strategy.init(conf);
+            strategy.setPluginConfig(
+                    ConfigFactory.parseString(
+                            "read_columns=[\"" + id + 
"\"]\nparse_partition_from_path=false"));
+            strategy.getSeaTunnelRowTypeInfo(file.toString());
+            TestCollector collector = new TestCollector();
+            strategy.read(file.toString(), "", collector);
+            Assertions.assertEquals(1, collector.getRows().size());
+            Assertions.assertEquals(readSchema ? 1 : null, 
collector.getRows().get(0).getField(0));
+            Assertions.assertEquals(
+                    explicit, 
strategy.hadoopFileSystemProxy.getConfiguration().get(option));
+        } finally {
+            strategy.close();
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(booleans = {false, true})
+    void testProjectionPreservesFiltersOnUnselectedColumns(boolean 
nativeReader) throws Exception {
+        String id = nativeReader ? "id-value" : "id";
+        MessageType schema =
+                Types.buildMessage()
+                        .required(INT32)
+                        .named(id)
+                        .required(INT32)
+                        .named("filter_column")
+                        .named("filtered");
+        Path file = new 
Path(projectionDirectory.resolve("filter.parquet").toUri());
+        try (ParquetWriter<Group> writer =
+                ExampleParquetWriter.builder(new ProjectionOutputFile(file))
+                        .withType(schema)
+                        
.withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+                        .build()) {
+            writer.write(new SimpleGroup(schema).append(id, 
1).append("filter_column", 0));
+            writer.write(new SimpleGroup(schema).append(id, 
2).append("filter_column", 1));
+        }
+        Configuration filterConfiguration = new Configuration(false);
+        ParquetInputFormat.setFilterPredicate(
+                filterConfiguration, 
FilterApi.eq(FilterApi.intColumn("filter_column"), 1));
+        LocalFileSystemConf.LocalConf conf = new 
LocalFileSystemConf.LocalConf("file:///");
+        conf.getExtraOptions()
+                .put(
+                        ParquetInputFormat.FILTER_PREDICATE,
+                        
filterConfiguration.get(ParquetInputFormat.FILTER_PREDICATE));
+        ParquetReadStrategy strategy = new ParquetReadStrategy();
+        try {
+            strategy.init(conf);
+            strategy.setPluginConfig(
+                    ConfigFactory.parseString(
+                            "read_columns=[\"" + id + 
"\"]\nparse_partition_from_path=false"));
+            strategy.getSeaTunnelRowTypeInfo(file.toString());
+            TestCollector collector = new TestCollector();
+            strategy.read(file.toString(), "", collector);
+            Assertions.assertEquals(1, collector.getRows().size());
+            Assertions.assertEquals(2, collector.getRows().get(0).getField(0));
+        } finally {
+            strategy.close();
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(booleans = {false, true})
+    void testProjectionRetainsNestedLogicalNullAndPartitionValues(boolean 
nativeReader)
+            throws Exception {
+        String unused = nativeReader ? "unused-value" : "unused";
+        MessageType schema =
+                MessageTypeParser.parseMessageType(
+                        "message row { required int32 id; "
+                                + "optional binary nullable (UTF8); required 
int32 day (DATE); "
+                                + "required group nested { required int32 
number; required binary text (UTF8); } "
+                                + "required binary "
+                                + unused
+                                + "; }");
+        java.nio.file.Path directory =
+                
Files.createDirectories(projectionDirectory.resolve("region=east"));
+        Path file = new Path(directory.resolve("nested.parquet").toUri());
+        try (ParquetWriter<Group> writer =
+                ExampleParquetWriter.builder(new ProjectionOutputFile(file))
+                        .withType(schema)
+                        
.withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+                        .build()) {
+            SimpleGroup record = new SimpleGroup(schema);
+            record.append("id", 7).append("day", 1).append(unused, "discard");
+            record.addGroup("nested").append("number", 3).append("text", 
"value");
+            writer.write(record);
+        }
+        ParquetReadStrategy strategy = new ParquetReadStrategy();
+        try {
+            strategy.init(new LocalFileSystemConf.LocalConf("file:///"));
+            strategy.setPluginConfig(
+                    ConfigFactory.parseString("read_columns = [nested, day, 
nullable, id]"));
+            strategy.getSeaTunnelRowTypeInfo(file.toString());
+            TestCollector collector = new TestCollector();
+            strategy.read(new FileSourceSplit("table", file.toString()), 
collector);
+            Assertions.assertEquals(1, collector.getRows().size());
+            SeaTunnelRow row = collector.getRows().get(0);
+            Assertions.assertEquals("table", row.getTableId());
+            Assertions.assertEquals(5, row.getArity());
+            Assertions.assertArrayEquals(
+                    new Object[] {3, "value"}, ((SeaTunnelRow) 
row.getField(0)).getFields());
+            Assertions.assertEquals(LocalDate.ofEpochDay(1), row.getField(1));
+            Assertions.assertNull(row.getField(2));
+            Assertions.assertEquals(7, row.getField(3));
+            Assertions.assertEquals("east", row.getField(4));
+        } finally {
+            strategy.close();
+        }
+    }
+
+    @Test
+    void testProjectionIsPerFileAndPreservesMissingColumns() throws Exception {
+        MessageType firstSchema =
+                MessageTypeParser.parseMessageType(
+                        "message first { required int32 id; required int32 
first_only; }");
+        MessageType secondSchema =
+                MessageTypeParser.parseMessageType(
+                        "message second { required int32 second_only; required 
int32 id; }");
+        Path first = new 
Path(projectionDirectory.resolve("first.parquet").toUri());
+        Path second = new 
Path(projectionDirectory.resolve("second.parquet").toUri());
+        try (ParquetWriter<Group> writer =
+                ExampleParquetWriter.builder(new ProjectionOutputFile(first))
+                        .withType(firstSchema)
+                        .build()) {
+            writer.write(new SimpleGroup(firstSchema).append("id", 
1).append("first_only", 9));
+        }
+        try (ParquetWriter<Group> writer =
+                ExampleParquetWriter.builder(new ProjectionOutputFile(second))
+                        .withType(secondSchema)
+                        .build()) {
+            writer.write(new SimpleGroup(secondSchema).append("id", 
2).append("second_only", 8));
+        }
+        ParquetReadStrategy strategy = new ParquetReadStrategy();
+        try {
+            strategy.init(new LocalFileSystemConf.LocalConf("file:///"));
+            
strategy.setPluginConfig(ConfigFactory.parseString("parse_partition_from_path=false"));
+            SeaTunnelRowType rowType =
+                    new SeaTunnelRowType(
+                            new String[] {"first_only", "id", "absent"},
+                            new SeaTunnelDataType[] {
+                                BasicType.INT_TYPE, BasicType.INT_TYPE, 
BasicType.INT_TYPE
+                            });
+            strategy.setCatalogTable(CatalogTableUtil.getCatalogTable("table", 
rowType));
+            Configuration shared = 
strategy.hadoopFileSystemProxy.getConfiguration();
+            TestCollector collector = new TestCollector();
+            strategy.read(first.toString(), "", collector);
+            strategy.read(second.toString(), "", collector);
+            Assertions.assertArrayEquals(
+                    new Object[] {9, 1, null}, 
collector.getRows().get(0).getFields());
+            Assertions.assertArrayEquals(
+                    new Object[] {null, 2, null}, 
collector.getRows().get(1).getFields());
+            strategy.setCatalogTable(
+                    CatalogTableUtil.getCatalogTable(
+                            "table",
+                            new SeaTunnelRowType(
+                                    new String[] {"absent"},
+                                    new SeaTunnelDataType[] 
{BasicType.INT_TYPE})));
+            strategy.read(second.toString(), "", collector);
+            Assertions.assertArrayEquals(
+                    new Object[] {null}, 
collector.getRows().get(2).getFields());
+            Assertions.assertNull(shared.get(ReadSupport.PARQUET_READ_SCHEMA));
+            
Assertions.assertNull(shared.get(AvroReadSupport.AVRO_REQUESTED_PROJECTION));
+            
Assertions.assertNull(shared.get(AvroReadSupport.AVRO_COMPATIBILITY));
+        } finally {
+            strategy.close();
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(booleans = {false, true})
+    void testReadColumnsPrunesPhysicalColumns(boolean nativeReader) throws 
Exception {
+        String id = nativeReader ? "id-value" : "id";
+        Path file = new 
Path(projectionDirectory.resolve("wide.parquet").toUri());
+        MessageType schema =
+                Types.buildMessage()
+                        .required(INT32)
+                        .named(id)
+                        .required(BINARY)
+                        .named("unused")
+                        .named("wide");
+        Random random = new Random(1234);
+        try (ParquetWriter<Group> writer =
+                ExampleParquetWriter.builder(new ProjectionOutputFile(file))
+                        .withType(schema)
+                        
.withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+                        .withDictionaryEncoding(false)
+                        .build()) {
+            for (int i = 0; i < 128; i++) {
+                byte[] bytes = new byte[32768];
+                random.nextBytes(bytes);
+                writer.write(
+                        new SimpleGroup(schema)
+                                .append(id, i)
+                                .append(
+                                        "unused",
+                                        
org.apache.parquet.io.api.Binary.fromConstantByteArray(
+                                                bytes)));
+            }
+        }
+        LocalFileSystemConf.LocalConf conf =
+                new LocalFileSystemConf.LocalConf("file:///") {
+                    @Override
+                    public String getFsHdfsImpl() {
+                        return CountingFileSystem.class.getName();
+                    }
+                };
+        ParquetReadStrategy strategy = new ParquetReadStrategy();
+        try {
+            strategy.init(conf);
+            strategy.setPluginConfig(
+                    ConfigFactory.parseString(
+                            "read_columns = [\"" + id + 
"\"]\nparse_partition_from_path = false"));
+            strategy.getSeaTunnelRowTypeInfo(file.toString());
+            CountingFileSystem.BYTES.set(0);
+            TestCollector collector = new TestCollector();
+            strategy.read(file.toString(), "", collector);
+            long bytes = CountingFileSystem.BYTES.get();
+            Assertions.assertEquals(128, collector.getRows().size());
+            for (int i = 0; i < 128; i++) {
+                Assertions.assertEquals(1, 
collector.getRows().get(i).getArity());
+                Assertions.assertEquals(i, 
collector.getRows().get(i).getField(0));
+            }
+            long fileBytes = 
Files.size(projectionDirectory.resolve("wide.parquet"));
+            System.out.printf(
+                    "PARQUET_PROJECTION native=%s rows=128 checksum=8128 
file_bytes=%d read_bytes=%d%n",
+                    nativeReader, fileBytes, bytes);
+            ParquetReadStrategy allColumns = new ParquetReadStrategy();
+            try {
+                allColumns.init(conf);
+                allColumns.setPluginConfig(
+                        
ConfigFactory.parseString("parse_partition_from_path=false"));
+                allColumns.getSeaTunnelRowTypeInfo(file.toString());
+                CountingFileSystem.BYTES.set(0);
+                TestCollector allCollector = new TestCollector();
+                allColumns.read(file.toString(), "", allCollector);
+                long fullBytes = CountingFileSystem.BYTES.get();
+                Assertions.assertEquals(128, allCollector.getRows().size());
+                Random expected = new Random(1234);
+                for (int i = 0; i < 128; i++) {
+                    SeaTunnelRow row = allCollector.getRows().get(i);
+                    byte[] payload = new byte[32768];
+                    expected.nextBytes(payload);
+                    Assertions.assertEquals(2, row.getArity());
+                    Assertions.assertEquals(i, row.getField(0));
+                    Assertions.assertArrayEquals(payload, (byte[]) 
row.getField(1));
+                }
+                Assertions.assertTrue(fullBytes > fileBytes / 2);
+                System.out.printf(
+                        "PARQUET_ALL_COLUMNS native=%s rows=128 checksum=8128 
read_bytes=%d%n",
+                        nativeReader, fullBytes);
+            } finally {
+                allColumns.close();
+            }
+            Assertions.assertTrue(
+                    bytes < fileBytes / 4,
+                    "Unselected Parquet column was read: " + bytes + " of " + 
fileBytes);
+        } finally {
+            strategy.close();
+        }
+    }
+
+    public static class CountingFileSystem extends RawLocalFileSystem {
+        static final AtomicLong BYTES = new AtomicLong();
+
+        @Override
+        public FSDataInputStream open(Path path, int bufferSize) throws 
IOException {
+            FSDataInputStream input = super.open(path, bufferSize);
+            return new FSDataInputStream(
+                    new FSInputStream() {
+                        @Override
+                        public void seek(long position) throws IOException {
+                            input.seek(position);
+                        }
+
+                        @Override
+                        public long getPos() throws IOException {
+                            return input.getPos();
+                        }
+
+                        @Override
+                        public boolean seekToNewSource(long position) throws 
IOException {
+                            return input.seekToNewSource(position);
+                        }
+
+                        @Override
+                        public int read() throws IOException {
+                            int value = input.read();
+                            if (value >= 0) {
+                                BYTES.incrementAndGet();
+                            }
+                            return value;
+                        }
+
+                        @Override
+                        public int read(byte[] bytes, int offset, int length) 
throws IOException {
+                            int read = input.read(bytes, offset, length);
+                            if (read > 0) {
+                                BYTES.addAndGet(read);
+                            }
+                            return read;
+                        }
+
+                        @Override
+                        public void close() throws IOException {
+                            input.close();
+                        }
+                    });
+        }
+    }
+
     @Test
     public void testParquetRead1() throws Exception {
         URL resource = 
ParquetReadStrategyTest.class.getResource("/timestamp_as_int64.parquet");
diff --git 
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java
 
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java
index 97a7462fa0b..a8aa98c614d 100644
--- 
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java
+++ 
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java
@@ -441,6 +441,13 @@ public class LocalFileIT extends TestSuiteBase {
         helper.execute("/json/local_file_to_console.conf");
     }
 
+    @TestTemplate
+    public void testLocalFileParquetProjection(TestContainer container)
+            throws IOException, InterruptedException {
+        TestHelper helper = new TestHelper(container);
+        
helper.execute("/parquet/local_file_parquet_projection_to_assert.conf");
+    }
+
     @TestTemplate
     public void testLocalFileOrcParquetBinaryXml(TestContainer container)
             throws IOException, InterruptedException {
diff --git 
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/parquet/local_file_parquet_projection_to_assert.conf
 
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/parquet/local_file_parquet_projection_to_assert.conf
index 871b3c02576..de3d2add530 100644
--- 
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/parquet/local_file_parquet_projection_to_assert.conf
+++ 
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/parquet/local_file_parquet_projection_to_assert.conf
@@ -31,7 +31,8 @@ source {
   LocalFile {
     path = "/seatunnel/read/parquet"
     file_format_type = "parquet"
-    read_columns = [c_string, c_boolean, c_double]
+    # Keep output order independent of the physical Parquet column order.
+    read_columns = [c_double, c_boolean, c_string]
     plugin_output = "fake"
   }
 }
@@ -40,6 +41,10 @@ sink {
   Assert {
     rules {
       row_rules = [
+        {
+          rule_type = MIN_ROW
+          rule_value = 5
+        },
         {
           rule_type = MAX_ROW
           rule_value = 5
@@ -76,4 +81,4 @@ sink {
       ]
     }
   }
-}
\ No newline at end of file
+}

Reply via email to