This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new d020350d0d06 fix(flink): filter pushdown predicates by physical
Parquet schema (#20654)
d020350d0d06 is described below
commit d020350d0d0605b7e30d2bcde42de39067529b13
Author: Shuo Cheng <[email protected]>
AuthorDate: Mon Nov 16 04:14:29 2026 +0000
fix(flink): filter pushdown predicates by physical Parquet schema (#20654)
* fix(flink): skip predicate pushdown for native delete logs
* fix(flink): validate predicates against physical Parquet schema
* fix(flink): validate configured Parquet predicates
Check configured predicate columns against each file's physical schema.
Remove unsupported predicates from a per-file configuration copy so
ParquetFileReader cannot reload them, while preserving the shared configuration
and valid filters for other files.
Add native-log and snapshot regressions for configured filters, compound
predicates, and configuration isolation.
---
.../format/ParquetPredicateColumnValidator.java | 113 ++++++++++
.../apache/hudi/table/format/RecordIterators.java | 48 ++--
.../format/TestFlinkRowDataReaderContext.java | 247 +++++++++++++++++++++
.../hudi/table/format/TestRecordIterators.java | 140 +++++++++++-
.../table/format/cow/ParquetSplitReaderUtil.java | 7 +-
.../reader/ParquetColumnarRowSplitReader.java | 6 +-
.../format/cow/TestParquetSplitReaderUtil.java | 7 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 7 +-
.../reader/ParquetColumnarRowSplitReader.java | 6 +-
.../format/cow/TestParquetSplitReaderUtil.java | 7 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 7 +-
.../reader/ParquetColumnarRowSplitReader.java | 6 +-
.../format/cow/TestParquetSplitReaderUtil.java | 7 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 7 +-
.../reader/ParquetColumnarRowSplitReader.java | 6 +-
.../format/cow/TestParquetSplitReaderUtil.java | 7 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 7 +-
.../reader/ParquetColumnarRowSplitReader.java | 6 +-
.../format/cow/TestParquetSplitReaderUtil.java | 7 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 7 +-
.../reader/ParquetColumnarRowSplitReader.java | 6 +-
.../format/cow/TestParquetSplitReaderUtil.java | 7 +-
22 files changed, 577 insertions(+), 91 deletions(-)
diff --git
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/ParquetPredicateColumnValidator.java
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/ParquetPredicateColumnValidator.java
new file mode 100644
index 000000000000..1541be0f4628
--- /dev/null
+++
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/ParquetPredicateColumnValidator.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.table.format;
+
+import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.filter2.predicate.Operators;
+import org.apache.parquet.filter2.predicate.UserDefinedPredicate;
+import org.apache.parquet.schema.MessageType;
+
+/** Checks physical column availability without evaluating or rewriting a
Parquet predicate. */
+final class ParquetPredicateColumnValidator implements
FilterPredicate.Visitor<Boolean> {
+ private final MessageType fileSchema;
+
+ private ParquetPredicateColumnValidator(MessageType fileSchema) {
+ this.fileSchema = fileSchema;
+ }
+
+ static boolean hasAllColumns(FilterPredicate predicate, MessageType
fileSchema) {
+ try {
+ return predicate.accept(new ParquetPredicateColumnValidator(fileSchema));
+ } catch (UnsupportedOperationException e) {
+ // Newer Parquet versions can add operators absent from the minimum
supported API.
+ // Omit the whole predicate if its column references cannot be checked.
+ return false;
+ }
+ }
+
+ private boolean contains(Operators.Column<?> column) {
+ return fileSchema.containsPath(column.getColumnPath().toArray());
+ }
+
+ @Override
+ public <T extends Comparable<T>> Boolean visit(Operators.Eq<T> predicate) {
+ return contains(predicate.getColumn());
+ }
+
+ @Override
+ public <T extends Comparable<T>> Boolean visit(Operators.NotEq<T> predicate)
{
+ return contains(predicate.getColumn());
+ }
+
+ @Override
+ public <T extends Comparable<T>> Boolean visit(Operators.Lt<T> predicate) {
+ return contains(predicate.getColumn());
+ }
+
+ @Override
+ public <T extends Comparable<T>> Boolean visit(Operators.LtEq<T> predicate) {
+ return contains(predicate.getColumn());
+ }
+
+ @Override
+ public <T extends Comparable<T>> Boolean visit(Operators.Gt<T> predicate) {
+ return contains(predicate.getColumn());
+ }
+
+ @Override
+ public <T extends Comparable<T>> Boolean visit(Operators.GtEq<T> predicate) {
+ return contains(predicate.getColumn());
+ }
+
+ @Override
+ public <T extends Comparable<T>> Boolean visit(Operators.In<T> predicate) {
+ return contains(predicate.getColumn());
+ }
+
+ @Override
+ public <T extends Comparable<T>> Boolean visit(Operators.NotIn<T> predicate)
{
+ return contains(predicate.getColumn());
+ }
+
+ @Override
+ public Boolean visit(Operators.And predicate) {
+ return predicate.getLeft().accept(this) &&
predicate.getRight().accept(this);
+ }
+
+ @Override
+ public Boolean visit(Operators.Or predicate) {
+ return predicate.getLeft().accept(this) &&
predicate.getRight().accept(this);
+ }
+
+ @Override
+ public Boolean visit(Operators.Not predicate) {
+ return predicate.getPredicate().accept(this);
+ }
+
+ @Override
+ public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> Boolean
visit(Operators.UserDefined<T, U> predicate) {
+ return contains(predicate.getColumn());
+ }
+
+ @Override
+ public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> Boolean
visit(Operators.LogicalNotUserDefined<T, U> predicate) {
+ return predicate.getUserDefined().accept(this);
+ }
+}
diff --git
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/RecordIterators.java
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/RecordIterators.java
index 67665e1b17a6..9e62c1e4a9ef 100644
---
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/RecordIterators.java
+++
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/RecordIterators.java
@@ -40,8 +40,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.filter.UnboundRecordFilter;
import org.apache.parquet.filter2.predicate.FilterPredicate;
import org.apache.parquet.hadoop.BadConfigurationException;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.hadoop.util.ConfigurationUtil;
import org.apache.parquet.hadoop.util.SerializationUtil;
+import org.apache.parquet.schema.MessageType;
import java.io.IOException;
import java.util.LinkedHashMap;
@@ -49,6 +51,8 @@ import java.util.List;
import java.util.Map;
import static org.apache.parquet.filter2.predicate.FilterApi.and;
+import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
+import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
import static org.apache.parquet.hadoop.ParquetInputFormat.FILTER_PREDICATE;
import static
org.apache.parquet.hadoop.ParquetInputFormat.UNBOUND_RECORD_FILTER;
@@ -105,14 +109,19 @@ public abstract class RecordIterators {
long splitStart,
long splitLength,
List<Predicate> predicates) throws IOException {
- FilterPredicate filterPredicate = getFilterPredicate(conf);
- for (Predicate predicate : predicates) {
- FilterPredicate filter = predicate.filter();
- if (filter != null) {
- filterPredicate = filterPredicate == null ? filter :
and(filterPredicate, filter);
- }
+ // Read the footer once for both predicate validation and the split reader.
+ ParquetMetadata footer = readFooter(conf, new
org.apache.hadoop.fs.Path(path.toUri()), range(splitStart, splitStart +
splitLength));
+ MessageType fileSchema = footer.getFileMetaData().getSchema();
+ FilterPredicate configuredFilter = getFilterPredicate(conf);
+ Configuration readConf = conf;
+ if (configuredFilter != null &&
!ParquetPredicateColumnValidator.hasAllColumns(configuredFilter, fileSchema)) {
+ // ParquetFileReader reloads the configured predicate, so remove it from
a per-file copy as well.
+ readConf = new Configuration(conf);
+ readConf.unset(FILTER_PREDICATE);
+ configuredFilter = null;
}
- UnboundRecordFilter recordFilter = getUnboundRecordFilterInstance(conf);
+ FilterPredicate filterPredicate = createFilterPredicate(fileSchema,
predicates, configuredFilter);
+ UnboundRecordFilter recordFilter =
getUnboundRecordFilterInstance(readConf);
InternalSchema mergeSchema =
internalSchemaManager.getMergeSchema(getFileName(path));
if (mergeSchema.isEmptySchema()) {
@@ -120,15 +129,14 @@ public abstract class RecordIterators {
ParquetSplitReaderUtil.genPartColumnarRowReader(
utcTimestamp,
caseSensitive,
- conf,
+ readConf,
fieldNames,
fieldTypes,
partitionSpec,
selectedFields,
batchSize,
path,
- splitStart,
- splitLength,
+ footer,
filterPredicate,
recordFilter));
} else {
@@ -138,15 +146,14 @@ public abstract class RecordIterators {
ParquetSplitReaderUtil.genPartColumnarRowReader(
utcTimestamp,
caseSensitive,
- conf,
+ readConf,
internalSchemaManager.getMergeFieldNames(mergeSchema,
fieldNames), // the reconciled field names
castMap.getFileFieldTypes(),
// the reconciled field types
partitionSpec,
selectedFields,
batchSize,
path,
- splitStart,
- splitLength,
+ footer,
filterPredicate,
recordFilter));
if (castProjection.isPresent()) {
@@ -157,6 +164,21 @@ public abstract class RecordIterators {
}
}
+ static FilterPredicate createFilterPredicate(MessageType fileSchema,
List<Predicate> predicates, FilterPredicate configuredFilter) {
+ FilterPredicate filterPredicate = configuredFilter;
+ for (Predicate predicate : predicates) {
+ // Missing columns are null to Parquet, which can incorrectly prune
native delete records.
+ // Skip the whole predicate if any reference is absent; pruning only one
branch of OR/NOT is unsafe.
+ if (predicate.references().stream().allMatch(column ->
fileSchema.containsPath(column.split("\\.")))) {
+ FilterPredicate filter = predicate.filter();
+ if (filter != null) {
+ filterPredicate = filterPredicate == null ? filter :
and(filterPredicate, filter);
+ }
+ }
+ }
+ return filterPredicate;
+ }
+
private static String getFileName(Path path) {
if (InLineFSUtils.SCHEME.equals(path.toUri().getScheme())) {
return InLineFSUtils.getOuterFilePathFromInlinePath(new
StoragePath(path.toUri())).getName();
diff --git
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestFlinkRowDataReaderContext.java
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestFlinkRowDataReaderContext.java
index f89b367441a4..54f44a626030 100644
---
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestFlinkRowDataReaderContext.java
+++
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestFlinkRowDataReaderContext.java
@@ -19,28 +19,68 @@
package org.apache.hudi.table.format;
+import org.apache.hudi.common.config.RecordMergeMode;
+import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieFileFormat;
+import org.apache.hudi.common.model.HoodieLogFile;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.schema.HoodieSchemaField;
import org.apache.hudi.common.schema.HoodieSchemaType;
+import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.HoodieTableVersion;
import org.apache.hudi.common.table.read.BufferedRecord;
+import org.apache.hudi.common.table.read.HoodieFileGroupReader;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.testutils.reader.HoodieFileSliceTestUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.source.ExpressionPredicates;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.expressions.FieldReferenceExpression;
+import org.apache.flink.table.expressions.ValueLiteralExpression;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.filter2.predicate.FilterApi;
+import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.hadoop.ParquetInputFormat;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.HOODIE_SCHEMA;
+import static
org.apache.hudi.common.testutils.HoodieTestUtils.DEFAULT_WRITE_TOKEN;
+import static
org.apache.hudi.common.testutils.reader.HoodieFileSliceTestUtils.DATA_GEN;
+import static
org.apache.hudi.common.testutils.reader.HoodieFileSliceTestUtils.ROW_KEY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -132,6 +172,213 @@ class TestFlinkRowDataReaderContext {
assertTrue(result.getBoolean(2));
}
+ @Nested
+ class NativeLogFilters {
+ private static final String PARTITION = "partition";
+ private static final String FILE_ID = "file-id";
+
+ @TempDir
+ Path tempDir;
+
+ private HoodieTableMetaClient metaClient;
+ private HoodieStorage storage;
+ private StoragePath partitionPath;
+
+ @BeforeEach
+ void setUpNativeLogs() throws Exception {
+ Properties properties = new Properties();
+ properties.setProperty(HoodieTableConfig.VERSION.key(), "10");
+ properties.setProperty(HoodieTableConfig.POPULATE_META_FIELDS.key(),
"false");
+ properties.setProperty(HoodieTableConfig.RECORDKEY_FIELDS.key(),
ROW_KEY);
+ properties.setProperty(HoodieTableConfig.RECORD_MERGE_MODE.key(),
RecordMergeMode.EVENT_TIME_ORDERING.name());
+ properties.setProperty(HoodieTableConfig.ORDERING_FIELDS.key(),
"timestamp");
+ metaClient = HoodieTestUtils.init(new HadoopStorageConfiguration(new
Configuration()), tempDir.toString(),
+ HoodieTableType.MERGE_ON_READ, properties);
+ assertEquals(HoodieTableVersion.TEN,
metaClient.getTableConfig().getTableVersion());
+ storage = metaClient.getStorage();
+ partitionPath = new StoragePath(metaClient.getBasePath(), PARTITION);
+ storage.createDirectory(partitionPath);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void testPrimaryKeyFilterDoesNotResurrectDeletedRecord(boolean
configuredFilter) throws Exception {
+ HoodieBaseFile baseFile = HoodieFileSliceTestUtils.createBaseFile(
+ new StoragePath(partitionPath, FSUtils.makeBaseFileName("001",
DEFAULT_WRITE_TOKEN, FILE_ID, ".parquet")).toString(),
+ Arrays.asList(record("k", 100L), record("survivor", 100L)),
HOODIE_SCHEMA, "001");
+ HoodieLogFile deleteFile = createDeleteFile();
+
assertTrue(FSUtils.isNativeDeleteLogFile(deleteFile.getPath().getName()));
+ // Establish that the physical delete schema does not contain the
business key column.
+ try (HoodieRowDataParquetReader reader = new
HoodieRowDataParquetReader(storage, deleteFile.getPath())) {
+ assertEquals(Arrays.asList("_hoodie_record_key", "timestamp"),
+
reader.getSchema().getFields().stream().map(HoodieSchemaField::name).collect(Collectors.toList()));
+ }
+ assertEquals(Collections.singletonList("survivor"),
readSnapshot(baseFile, deleteFile, null));
+ assertEquals(Collections.singletonList("survivor"),
readSnapshot(baseFile, deleteFile, "survivor"));
+ if (configuredFilter) {
+
ParquetInputFormat.setFilterPredicate(storage.getConf().unwrapAs(Configuration.class),
keyEquals(ROW_KEY, "k").filter());
+ }
+ assertEquals(Collections.emptyList(), readSnapshot(baseFile, deleteFile,
"k"));
+ }
+
+ @Test
+ void testStoredRecordKeyPredicatePrunesDeleteRowGroups() throws Exception {
+ HoodieLogFile deleteFile = createDeleteFile();
+ assertEquals(Collections.singletonList("k"), readDeleteKeys(deleteFile,
keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "k")));
+ assertEquals(Collections.emptyList(), readDeleteKeys(deleteFile,
keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "other")));
+ }
+
+ @ParameterizedTest(name = "{0}")
+
@MethodSource("org.apache.hudi.table.format.TestFlinkRowDataReaderContext#deleteLogPredicates")
+ void testDeleteLogPredicateSafety(ExpressionPredicates.Predicate
predicate, List<String> expectedKeys) throws Exception {
+ assertEquals(expectedKeys, readDeleteKeys(createDeleteFile(),
predicate));
+ }
+
+ @Test
+ void testNativeDataLogRetainsPrimaryKeyPushdown() throws Exception {
+ HoodieLogFile dataFile =
HoodieFileSliceTestUtils.createNativeDataLogFile(
+ nativeLogPath(".log.parquet"),
Collections.singletonList(record("other", 100L)), HOODIE_SCHEMA, "002");
+ FlinkRowDataReaderContext context = readerContext("k");
+ context.setHasLogFiles(true);
+ try (ClosableIterator<RowData> iterator =
context.getFileRecordIterator(dataFile.getPath(), 0, Long.MAX_VALUE,
+ HOODIE_SCHEMA, HOODIE_SCHEMA, storage)) {
+ assertFalse(iterator.hasNext(), "Primary key pushdown should still
prune nonmatching data log row groups");
+ }
+ }
+
+ @ParameterizedTest(name = "{0}")
+
@MethodSource("org.apache.hudi.table.format.TestFlinkRowDataReaderContext#configuredDeleteLogPredicates")
+ void testConfiguredDeleteLogPredicates(FilterPredicate predicate,
List<String> expectedKeys) throws Exception {
+ HoodieLogFile deleteFile = createDeleteFile();
+ Configuration conf = storage.getConf().unwrapAs(Configuration.class);
+ ParquetInputFormat.setFilterPredicate(conf, predicate);
+ String originalFilter = conf.get(ParquetInputFormat.FILTER_PREDICATE);
+ // No query predicates: exercise the configured filter and its
downstream reload in ParquetFileReader.
+ assertEquals(expectedKeys, readDeleteKeys(deleteFile, null));
+ assertEquals(originalFilter,
conf.get(ParquetInputFormat.FILTER_PREDICATE));
+ }
+
+ @Test
+ void testConfiguredFilterRemainsAvailableForDataFiles() throws Exception {
+ HoodieLogFile deleteFile = createDeleteFile();
+ HoodieLogFile dataFile =
HoodieFileSliceTestUtils.createNativeDataLogFile(
+ nativeLogPath(".log.parquet"),
Collections.singletonList(record("other", 100L)), HOODIE_SCHEMA, "002");
+ Configuration conf = storage.getConf().unwrapAs(Configuration.class);
+ ParquetInputFormat.setFilterPredicate(conf, keyEquals(ROW_KEY,
"k").filter());
+ String originalFilter = conf.get(ParquetInputFormat.FILTER_PREDICATE);
+ assertEquals(Collections.singletonList("k"), readDeleteKeys(deleteFile,
null));
+ assertEquals(originalFilter,
conf.get(ParquetInputFormat.FILTER_PREDICATE));
+ FlinkRowDataReaderContext context =
readerContext(Collections.emptyList());
+ context.setHasLogFiles(true);
+ try (ClosableIterator<RowData> iterator =
context.getFileRecordIterator(dataFile.getPath(), 0, Long.MAX_VALUE,
+ HOODIE_SCHEMA, HOODIE_SCHEMA, storage)) {
+ assertFalse(iterator.hasNext(), "The shared configuration must still
filter files containing the business key");
+ }
+ }
+
+ private HoodieLogFile createDeleteFile() throws Exception {
+ return HoodieFileSliceTestUtils.createNativeDeleteLogFile(storage,
+ nativeLogPath(".deletes.parquet"),
Collections.singletonList(record("k", 200L)), HOODIE_SCHEMA, "002");
+ }
+
+ private List<String> readDeleteKeys(HoodieLogFile deleteFile,
ExpressionPredicates.Predicate predicate) throws Exception {
+ HoodieSchema schema =
HoodieSchemaUtils.createDeleteLogSchema(HOODIE_SCHEMA,
Collections.singletonList("timestamp"));
+ FlinkRowDataReaderContext context = readerContext(predicate == null ?
Collections.emptyList() : Collections.singletonList(predicate));
+ context.setHasLogFiles(true);
+ List<String> keys = new ArrayList<>();
+ try (ClosableIterator<RowData> iterator =
context.getFileRecordIterator(deleteFile.getPath(), 0, Long.MAX_VALUE,
+ schema, schema, storage)) {
+ while (iterator.hasNext()) {
+ keys.add(iterator.next().getString(0).toString());
+ }
+ }
+ return keys;
+ }
+
+ private List<String> readSnapshot(HoodieBaseFile baseFile, HoodieLogFile
deleteFile, String key) throws Exception {
+ HoodieFileGroupReader<RowData> reader =
HoodieFileGroupReader.<RowData>builder()
+ .withReaderContext(readerContext(key))
+ .withHoodieTableMetaClient(metaClient)
+ .withStorage(storage)
+ .withLatestCommitTime("002")
+ .withBaseFileOption(Option.of(baseFile))
+ .withLogFiles(Collections.singletonList(deleteFile).stream())
+ .withPartitionPath(PARTITION)
+ .withDataSchema(HOODIE_SCHEMA)
+ .withRequestedSchema(HOODIE_SCHEMA)
+
.withProps(TypedProperties.copy(metaClient.getTableConfig().getProps()))
+ .withShouldUseRecordPosition(false)
+ .build();
+ List<String> keys = new ArrayList<>();
+ try (ClosableIterator<RowData> iterator = reader.getClosableIterator()) {
+ while (iterator.hasNext()) {
+ String recordKey =
iterator.next().getString(HOODIE_SCHEMA.getField(ROW_KEY).get().pos()).toString();
+ // Parquet pushdown prunes row groups; the SQL engine evaluates the
predicate after merging.
+ if (key == null || key.equals(recordKey)) {
+ keys.add(recordKey);
+ }
+ }
+ }
+ return keys;
+ }
+
+ private FlinkRowDataReaderContext readerContext(String key) {
+ return readerContext(key == null ? Collections.emptyList() :
Collections.singletonList(keyEquals(ROW_KEY, key)));
+ }
+
+ private FlinkRowDataReaderContext
readerContext(List<ExpressionPredicates.Predicate> predicates) {
+ return new FlinkRowDataReaderContext(storage.getConf(),
+ () -> InternalSchemaManager.DISABLED, predicates,
metaClient.getTableConfig(), Option.empty());
+ }
+
+ private String nativeLogPath(String extension) {
+ return new StoragePath(partitionPath, FILE_ID + "_" +
DEFAULT_WRITE_TOKEN + "_002_1" + extension).toString();
+ }
+
+ private IndexedRecord record(String key, long timestamp) {
+ return DATA_GEN.generateGenericRecord(key, PARTITION, "rider." + key,
"driver." + key, timestamp, false, false);
+ }
+ }
+
+ private static Stream<Arguments> configuredDeleteLogPredicates() {
+ FilterPredicate missingKey = keyEquals(ROW_KEY, "k").filter();
+ FilterPredicate storedKey =
keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "other").filter();
+ List<String> retainedKeys = Collections.singletonList("k");
+ return Stream.of(
+ Arguments.of(missingKey, retainedKeys),
+ Arguments.of(FilterApi.and(missingKey, storedKey), retainedKeys),
+ Arguments.of(FilterApi.or(missingKey, storedKey), retainedKeys),
+ Arguments.of(FilterApi.or(storedKey, missingKey), retainedKeys),
+ Arguments.of(FilterApi.not(FilterApi.and(keyEquals(ROW_KEY,
"other").filter(),
+ keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "k").filter())),
retainedKeys),
+ Arguments.of(keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD,
"k").filter(), retainedKeys),
+ Arguments.of(storedKey, Collections.emptyList()));
+ }
+
+ private static Stream<Arguments> deleteLogPredicates() {
+ ExpressionPredicates.Predicate missingKey = keyEquals(ROW_KEY, "k");
+ ExpressionPredicates.Predicate storedKey =
keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "other");
+ List<String> retainedKeys = Collections.singletonList("k");
+ return Stream.of(
+
Arguments.of(ExpressionPredicates.Or.getInstance().bindPredicates(missingKey,
storedKey), retainedKeys),
+
Arguments.of(ExpressionPredicates.Or.getInstance().bindPredicates(storedKey,
missingKey), retainedKeys),
+ Arguments.of(ExpressionPredicates.Not.getInstance().bindPredicate(
+
ExpressionPredicates.And.getInstance().bindPredicates(keyEquals(ROW_KEY,
"other"),
+ keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "k"))),
retainedKeys),
+ Arguments.of(ExpressionPredicates.Or.getInstance().bindPredicates(
+ keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "a"),
keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "b")),
Collections.emptyList()),
+ // Physical presence does not make a non-key ordering predicate safe
for MOR.
+ Arguments.of(ExpressionPredicates.Equals.getInstance()
+ .bindValueLiteral(new ValueLiteralExpression(300L))
+ .bindFieldReference(new FieldReferenceExpression("timestamp",
DataTypes.BIGINT(), 0, 1)), retainedKeys));
+ }
+
+ private static ExpressionPredicates.Predicate keyEquals(String field, String
key) {
+ return ExpressionPredicates.Equals.getInstance()
+ .bindValueLiteral(new ValueLiteralExpression(key))
+ .bindFieldReference(new FieldReferenceExpression(field,
DataTypes.STRING(), 0, 0));
+ }
+
private GenericRowData createBaseRow(int id, String name, boolean active) {
return GenericRowData.of(id, StringData.fromString(name), active);
}
diff --git
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestRecordIterators.java
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestRecordIterators.java
index 8086e1516972..a0fa1778dd19 100644
---
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestRecordIterators.java
+++
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestRecordIterators.java
@@ -39,23 +39,41 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.column.ColumnReader;
import org.apache.parquet.filter.RecordFilter;
import org.apache.parquet.filter.UnboundRecordFilter;
+import org.apache.parquet.filter2.predicate.FilterApi;
import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.filter2.predicate.LogicalInverseRewriter;
+import org.apache.parquet.filter2.predicate.Operators;
+import org.apache.parquet.filter2.predicate.Statistics;
+import org.apache.parquet.filter2.predicate.UserDefinedPredicate;
+import
org.apache.parquet.format.converter.ParquetMetadataConverter.MetadataFilter;
import org.apache.parquet.hadoop.BadConfigurationException;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.metadata.FileMetaData;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.hadoop.util.SerializationUtil;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.MessageTypeParser;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.MockedStatic;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
+import java.util.List;
import static org.apache.parquet.filter2.predicate.FilterApi.eq;
import static org.apache.parquet.filter2.predicate.FilterApi.intColumn;
+import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
import static org.apache.parquet.hadoop.ParquetInputFormat.FILTER_PREDICATE;
import static
org.apache.parquet.hadoop.ParquetInputFormat.UNBOUND_RECORD_FILTER;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -63,11 +81,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -124,6 +142,56 @@ class TestRecordIterators {
assertTrue(exception.getCause() instanceof RuntimeException);
}
+ @Test
+ void testMissingPhysicalColumnSkipsPredicateTranslation() {
+ MessageType schema = MessageTypeParser.parseMessageType("message file {
optional int32 stored; }");
+ Predicate predicate = mock(Predicate.class);
+
when(predicate.references()).thenReturn(Collections.singletonList("missing"));
+ assertNull(RecordIterators.createFilterPredicate(schema,
Collections.singletonList(predicate), null));
+ verify(predicate, never()).filter();
+ }
+
+ @Test
+ void testPhysicalNestedColumnPathIsSupported() {
+ MessageType schema = MessageTypeParser.parseMessageType("message file {
optional group nested { optional int32 id; } }");
+ Predicate predicate = mock(Predicate.class);
+ FilterPredicate expected = eq(intColumn("nested.id"), 7);
+
when(predicate.references()).thenReturn(Collections.singletonList("nested.id"));
+ when(predicate.filter()).thenReturn(expected);
+ assertSame(expected, RecordIterators.createFilterPredicate(schema,
Collections.singletonList(predicate), null));
+
+
when(predicate.references()).thenReturn(Collections.singletonList("nested.missing"));
+ assertNull(RecordIterators.createFilterPredicate(schema,
Collections.singletonList(predicate), null));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"stored", "nested.id", "missing", "nested.missing",
"nested"})
+ void testConfiguredPredicateColumnValidation(String columnName) {
+ MessageType schema = MessageTypeParser.parseMessageType(
+ "message file { optional int32 stored; optional group nested {
optional int32 id; } }");
+ Operators.IntColumn column = intColumn(columnName);
+ FilterPredicate userDefined = FilterApi.userDefined(column,
KeepAllPredicate.class);
+ List<FilterPredicate> predicates = Arrays.asList(
+ eq(column, 7), FilterApi.notEq(column, 7), FilterApi.lt(column, 7),
FilterApi.ltEq(column, 7),
+ FilterApi.gt(column, 7), FilterApi.gtEq(column, 7),
+ FilterApi.in(column, Collections.singleton(7)),
FilterApi.notIn(column, Collections.singleton(7)),
+ FilterApi.and(eq(intColumn("stored"), 1), eq(column, 7)),
+ FilterApi.or(eq(intColumn("stored"), 1), eq(column, 7)),
+ FilterApi.not(eq(column, 7)), userDefined,
LogicalInverseRewriter.rewrite(FilterApi.not(userDefined)));
+ boolean expected = columnName.equals("stored") ||
columnName.equals("nested.id");
+ for (FilterPredicate predicate : predicates) {
+ assertEquals(expected,
ParquetPredicateColumnValidator.hasAllColumns(predicate, schema),
predicate.toString());
+ }
+ }
+
+ @Test
+ void testUnknownConfiguredPredicateIsSkipped() {
+ FilterPredicate predicate = mock(FilterPredicate.class);
+ when(predicate.accept(any())).thenThrow(new
UnsupportedOperationException("Unsupported operator"));
+ MessageType schema = MessageTypeParser.parseMessageType("message file {
optional int32 stored; }");
+ assertFalse(ParquetPredicateColumnValidator.hasAllColumns(predicate,
schema));
+ }
+
@Test
void testUnboundRecordFilterConfiguration() throws Exception {
Configuration conf = new Configuration();
@@ -186,15 +254,17 @@ class TestRecordIterators {
assertEquals(Collections.singletonMap("region", "us"), partitionSpec);
}
- @Test
- void testParquetIteratorBuildsPredicateAndSchemaEvolutionReader() throws
Exception {
- InternalSchema mergeSchema = new InternalSchema(Types.RecordType.get(
- Types.Field.get(0, false, "id", Types.IntType.get())));
+ @ParameterizedTest
+ @CsvSource({"false, false", "false, true", "true, false", "true, true"})
+ void testParquetIteratorReusesFooterAndBuildsPredicate(boolean
schemaEvolution, boolean configuredMissingColumn) throws Exception {
+ InternalSchema mergeSchema = schemaEvolution ? new
InternalSchema(Types.RecordType.get(
+ Types.Field.get(0, false, "id", Types.IntType.get()))) :
InternalSchema.getEmptyInternalSchema();
InternalSchemaManager schemaManager = mock(InternalSchemaManager.class);
CastMap castMap = new CastMap();
DataType[] fieldTypes = {DataTypes.INT()};
castMap.setFileFieldTypes(fieldTypes);
Predicate predicate = mock(Predicate.class);
+ when(predicate.references()).thenReturn(Collections.singletonList("id"));
when(predicate.filter()).thenReturn(eq(intColumn("id"), 7));
when(schemaManager.getMergeSchema(anyString())).thenReturn(mergeSchema);
when(schemaManager.getCastMap(
@@ -208,29 +278,58 @@ class TestRecordIterators {
ParquetColumnarRowSplitReader reader =
mock(ParquetColumnarRowSplitReader.class);
when(reader.reachedEnd()).thenReturn(true);
- try (MockedStatic<ParquetSplitReaderUtil> mocked =
mockStatic(ParquetSplitReaderUtil.class)) {
+ Configuration conf = new Configuration();
+ if (configuredMissingColumn) {
+ SerializationUtil.writeObjectToConfAsBase64(FILTER_PREDICATE,
eq(intColumn("missing"), 1), conf);
+ }
+ String originalFilter = conf.get(FILTER_PREDICATE);
+ org.apache.hadoop.fs.Path path = new
org.apache.hadoop.fs.Path("file:///table.parquet");
+ MessageType fileSchema = MessageTypeParser.parseMessageType("message file
{ optional int32 id; }");
+ ParquetMetadata footer = new ParquetMetadata(new FileMetaData(fileSchema,
Collections.emptyMap(), "test"), Collections.emptyList());
+ try (MockedStatic<ParquetFileReader> parquetReader =
mockStatic(ParquetFileReader.class);
+ MockedStatic<ParquetSplitReaderUtil> mocked =
mockStatic(ParquetSplitReaderUtil.class)) {
+ parquetReader.when(() -> ParquetFileReader.readFooter(
+ org.mockito.ArgumentMatchers.eq(conf),
org.mockito.ArgumentMatchers.eq(path), any(MetadataFilter.class)))
+ .thenAnswer(invocation -> {
+ assertEquals(range(10L, 30L).toString(),
invocation.getArgument(2).toString());
+ return footer;
+ });
mocked.when(() -> ParquetSplitReaderUtil.genPartColumnarRowReader(
anyBoolean(), anyBoolean(), any(Configuration.class),
any(String[].class),
- any(DataType[].class), anyMap(), any(int[].class), anyInt(), any(),
anyLong(),
- anyLong(), any(), any())).thenReturn(reader);
+ any(DataType[].class), anyMap(), any(int[].class), anyInt(), any(),
any(),
+ any(), any())).thenAnswer(invocation -> {
+ assertSame(footer, invocation.getArgument(9));
+ assertEquals(eq(intColumn("id"), 7), invocation.getArgument(10));
+ Configuration readConf = invocation.getArgument(2);
+ assertNull(readConf.get(FILTER_PREDICATE));
+ if (configuredMissingColumn) {
+ assertNotSame(conf, readConf);
+ } else {
+ assertSame(conf, readConf);
+ }
+ return reader;
+ });
ClosableIterator<RowData> iterator =
RecordIterators.getParquetRecordIterator(
schemaManager,
true,
true,
- new Configuration(),
+ conf,
new String[] {"id"},
fieldTypes,
Collections.emptyMap(),
new int[] {0},
16,
- new org.apache.flink.core.fs.Path("file:///table.parquet"),
- 0L,
- 1L,
+ new org.apache.flink.core.fs.Path(path.toUri()),
+ 10L,
+ 20L,
Collections.singletonList(predicate));
assertFalse(iterator.hasNext());
iterator.close();
verify(reader).close();
+ assertEquals(originalFilter, conf.get(FILTER_PREDICATE));
+ parquetReader.verify(() -> ParquetFileReader.readFooter(
+ org.mockito.ArgumentMatchers.eq(conf),
org.mockito.ArgumentMatchers.eq(path), any(MetadataFilter.class)));
}
}
@@ -244,6 +343,23 @@ class TestRecordIterators {
return (T) method.invoke(null, arguments);
}
+ public static class KeepAllPredicate extends UserDefinedPredicate<Integer> {
+ @Override
+ public boolean keep(Integer value) {
+ return true;
+ }
+
+ @Override
+ public boolean canDrop(Statistics<Integer> statistics) {
+ return false;
+ }
+
+ @Override
+ public boolean inverseCanDrop(Statistics<Integer> statistics) {
+ return false;
+ }
+ }
+
public static class ConfigurableRecordFilter
implements UnboundRecordFilter, Configurable {
private Configuration conf;
diff --git
a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
index e702461b5203..f89b2c4ac75a 100644
---
a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
+++
b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
@@ -78,6 +78,7 @@ import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.column.page.PageReader;
import org.apache.parquet.filter.UnboundRecordFilter;
import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.io.ColumnIO;
import org.apache.parquet.io.GroupColumnIO;
import org.apache.parquet.io.MessageColumnIO;
@@ -139,8 +140,7 @@ public class ParquetSplitReaderUtil {
int[] selectedFields,
int batchSize,
Path path,
- long splitStart,
- long splitLength,
+ ParquetMetadata footer,
FilterPredicate filterPredicate,
UnboundRecordFilter recordFilter) throws IOException {
@@ -178,8 +178,7 @@ public class ParquetSplitReaderUtil {
gen,
batchSize,
new org.apache.hadoop.fs.Path(path.toUri()),
- splitStart,
- splitLength,
+ footer,
filterPredicate,
recordFilter);
}
diff --git
a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
index b9f7527a91a8..3454c16e8093 100644
---
a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
+++
b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
@@ -62,8 +62,6 @@ import java.util.stream.IntStream;
import static
org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector;
import static org.apache.parquet.filter2.compat.FilterCompat.get;
import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups;
-import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
-import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
/**
* This reader is used to read a {@link VectorizedColumnBatch} from input
split.
@@ -137,14 +135,12 @@ public class ParquetColumnarRowSplitReader implements
Closeable {
ColumnBatchGenerator generator,
int batchSize,
Path path,
- long splitStart,
- long splitLength,
+ ParquetMetadata footer,
FilterPredicate filterPredicate,
UnboundRecordFilter recordFilter) throws IOException {
this.utcTimestamp = utcTimestamp;
this.batchSize = batchSize;
// then we need to apply the predicate push down filter
- ParquetMetadata footer = readFooter(conf, path, range(splitStart,
splitStart + splitLength));
MessageType fileSchema = footer.getFileMetaData().getSchema();
FilterCompat.Filter filter = get(filterPredicate, recordFilter);
List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(),
fileSchema);
diff --git
a/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
b/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
index 3763003e554d..8258b2d4cb07 100644
---
a/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
+++
b/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
@@ -35,6 +35,7 @@ import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.junit.jupiter.api.Test;
@@ -51,6 +52,8 @@ import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
+import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
+import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
@@ -352,6 +355,7 @@ public class TestParquetSplitReaderUtil {
}
}
+ ParquetMetadata footer = readFooter(conf, path, range(0, new File(tempDir,
"data.parquet").length()));
List<String> values = new ArrayList<>();
try (ParquetColumnarRowSplitReader reader = new
ParquetColumnarRowSplitReader(
true,
@@ -362,8 +366,7 @@ public class TestParquetSplitReaderUtil {
VectorizedColumnBatch::new,
batchSize,
path,
- 0,
- new File(tempDir, "data.parquet").length(),
+ footer,
null,
null)) {
while (!reader.reachedEnd()) {
diff --git
a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
index e702461b5203..f89b2c4ac75a 100644
---
a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
+++
b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
@@ -78,6 +78,7 @@ import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.column.page.PageReader;
import org.apache.parquet.filter.UnboundRecordFilter;
import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.io.ColumnIO;
import org.apache.parquet.io.GroupColumnIO;
import org.apache.parquet.io.MessageColumnIO;
@@ -139,8 +140,7 @@ public class ParquetSplitReaderUtil {
int[] selectedFields,
int batchSize,
Path path,
- long splitStart,
- long splitLength,
+ ParquetMetadata footer,
FilterPredicate filterPredicate,
UnboundRecordFilter recordFilter) throws IOException {
@@ -178,8 +178,7 @@ public class ParquetSplitReaderUtil {
gen,
batchSize,
new org.apache.hadoop.fs.Path(path.toUri()),
- splitStart,
- splitLength,
+ footer,
filterPredicate,
recordFilter);
}
diff --git
a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
index b9f7527a91a8..3454c16e8093 100644
---
a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
+++
b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
@@ -62,8 +62,6 @@ import java.util.stream.IntStream;
import static
org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector;
import static org.apache.parquet.filter2.compat.FilterCompat.get;
import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups;
-import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
-import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
/**
* This reader is used to read a {@link VectorizedColumnBatch} from input
split.
@@ -137,14 +135,12 @@ public class ParquetColumnarRowSplitReader implements
Closeable {
ColumnBatchGenerator generator,
int batchSize,
Path path,
- long splitStart,
- long splitLength,
+ ParquetMetadata footer,
FilterPredicate filterPredicate,
UnboundRecordFilter recordFilter) throws IOException {
this.utcTimestamp = utcTimestamp;
this.batchSize = batchSize;
// then we need to apply the predicate push down filter
- ParquetMetadata footer = readFooter(conf, path, range(splitStart,
splitStart + splitLength));
MessageType fileSchema = footer.getFileMetaData().getSchema();
FilterCompat.Filter filter = get(filterPredicate, recordFilter);
List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(),
fileSchema);
diff --git
a/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
index 3763003e554d..8258b2d4cb07 100644
---
a/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
+++
b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
@@ -35,6 +35,7 @@ import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.junit.jupiter.api.Test;
@@ -51,6 +52,8 @@ import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
+import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
+import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
@@ -352,6 +355,7 @@ public class TestParquetSplitReaderUtil {
}
}
+ ParquetMetadata footer = readFooter(conf, path, range(0, new File(tempDir,
"data.parquet").length()));
List<String> values = new ArrayList<>();
try (ParquetColumnarRowSplitReader reader = new
ParquetColumnarRowSplitReader(
true,
@@ -362,8 +366,7 @@ public class TestParquetSplitReaderUtil {
VectorizedColumnBatch::new,
batchSize,
path,
- 0,
- new File(tempDir, "data.parquet").length(),
+ footer,
null,
null)) {
while (!reader.reachedEnd()) {
diff --git
a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
index e702461b5203..f89b2c4ac75a 100644
---
a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
+++
b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
@@ -78,6 +78,7 @@ import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.column.page.PageReader;
import org.apache.parquet.filter.UnboundRecordFilter;
import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.io.ColumnIO;
import org.apache.parquet.io.GroupColumnIO;
import org.apache.parquet.io.MessageColumnIO;
@@ -139,8 +140,7 @@ public class ParquetSplitReaderUtil {
int[] selectedFields,
int batchSize,
Path path,
- long splitStart,
- long splitLength,
+ ParquetMetadata footer,
FilterPredicate filterPredicate,
UnboundRecordFilter recordFilter) throws IOException {
@@ -178,8 +178,7 @@ public class ParquetSplitReaderUtil {
gen,
batchSize,
new org.apache.hadoop.fs.Path(path.toUri()),
- splitStart,
- splitLength,
+ footer,
filterPredicate,
recordFilter);
}
diff --git
a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
index b9f7527a91a8..3454c16e8093 100644
---
a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
+++
b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
@@ -62,8 +62,6 @@ import java.util.stream.IntStream;
import static
org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector;
import static org.apache.parquet.filter2.compat.FilterCompat.get;
import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups;
-import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
-import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
/**
* This reader is used to read a {@link VectorizedColumnBatch} from input
split.
@@ -137,14 +135,12 @@ public class ParquetColumnarRowSplitReader implements
Closeable {
ColumnBatchGenerator generator,
int batchSize,
Path path,
- long splitStart,
- long splitLength,
+ ParquetMetadata footer,
FilterPredicate filterPredicate,
UnboundRecordFilter recordFilter) throws IOException {
this.utcTimestamp = utcTimestamp;
this.batchSize = batchSize;
// then we need to apply the predicate push down filter
- ParquetMetadata footer = readFooter(conf, path, range(splitStart,
splitStart + splitLength));
MessageType fileSchema = footer.getFileMetaData().getSchema();
FilterCompat.Filter filter = get(filterPredicate, recordFilter);
List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(),
fileSchema);
diff --git
a/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
index 3763003e554d..8258b2d4cb07 100644
---
a/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
+++
b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
@@ -35,6 +35,7 @@ import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.junit.jupiter.api.Test;
@@ -51,6 +52,8 @@ import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
+import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
+import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
@@ -352,6 +355,7 @@ public class TestParquetSplitReaderUtil {
}
}
+ ParquetMetadata footer = readFooter(conf, path, range(0, new File(tempDir,
"data.parquet").length()));
List<String> values = new ArrayList<>();
try (ParquetColumnarRowSplitReader reader = new
ParquetColumnarRowSplitReader(
true,
@@ -362,8 +366,7 @@ public class TestParquetSplitReaderUtil {
VectorizedColumnBatch::new,
batchSize,
path,
- 0,
- new File(tempDir, "data.parquet").length(),
+ footer,
null,
null)) {
while (!reader.reachedEnd()) {
diff --git
a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
index e702461b5203..f89b2c4ac75a 100644
---
a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
+++
b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
@@ -78,6 +78,7 @@ import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.column.page.PageReader;
import org.apache.parquet.filter.UnboundRecordFilter;
import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.io.ColumnIO;
import org.apache.parquet.io.GroupColumnIO;
import org.apache.parquet.io.MessageColumnIO;
@@ -139,8 +140,7 @@ public class ParquetSplitReaderUtil {
int[] selectedFields,
int batchSize,
Path path,
- long splitStart,
- long splitLength,
+ ParquetMetadata footer,
FilterPredicate filterPredicate,
UnboundRecordFilter recordFilter) throws IOException {
@@ -178,8 +178,7 @@ public class ParquetSplitReaderUtil {
gen,
batchSize,
new org.apache.hadoop.fs.Path(path.toUri()),
- splitStart,
- splitLength,
+ footer,
filterPredicate,
recordFilter);
}
diff --git
a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
index b9f7527a91a8..3454c16e8093 100644
---
a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
+++
b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
@@ -62,8 +62,6 @@ import java.util.stream.IntStream;
import static
org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector;
import static org.apache.parquet.filter2.compat.FilterCompat.get;
import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups;
-import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
-import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
/**
* This reader is used to read a {@link VectorizedColumnBatch} from input
split.
@@ -137,14 +135,12 @@ public class ParquetColumnarRowSplitReader implements
Closeable {
ColumnBatchGenerator generator,
int batchSize,
Path path,
- long splitStart,
- long splitLength,
+ ParquetMetadata footer,
FilterPredicate filterPredicate,
UnboundRecordFilter recordFilter) throws IOException {
this.utcTimestamp = utcTimestamp;
this.batchSize = batchSize;
// then we need to apply the predicate push down filter
- ParquetMetadata footer = readFooter(conf, path, range(splitStart,
splitStart + splitLength));
MessageType fileSchema = footer.getFileMetaData().getSchema();
FilterCompat.Filter filter = get(filterPredicate, recordFilter);
List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(),
fileSchema);
diff --git
a/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
index 3763003e554d..8258b2d4cb07 100644
---
a/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
+++
b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
@@ -35,6 +35,7 @@ import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.junit.jupiter.api.Test;
@@ -51,6 +52,8 @@ import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
+import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
+import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
@@ -352,6 +355,7 @@ public class TestParquetSplitReaderUtil {
}
}
+ ParquetMetadata footer = readFooter(conf, path, range(0, new File(tempDir,
"data.parquet").length()));
List<String> values = new ArrayList<>();
try (ParquetColumnarRowSplitReader reader = new
ParquetColumnarRowSplitReader(
true,
@@ -362,8 +366,7 @@ public class TestParquetSplitReaderUtil {
VectorizedColumnBatch::new,
batchSize,
path,
- 0,
- new File(tempDir, "data.parquet").length(),
+ footer,
null,
null)) {
while (!reader.reachedEnd()) {
diff --git
a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
index d74f4d71e1ca..998f22dfc9be 100644
---
a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
+++
b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
@@ -81,6 +81,7 @@ import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.column.page.PageReader;
import org.apache.parquet.filter.UnboundRecordFilter;
import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.io.ColumnIO;
import org.apache.parquet.io.GroupColumnIO;
import org.apache.parquet.io.MessageColumnIO;
@@ -142,8 +143,7 @@ public class ParquetSplitReaderUtil {
int[] selectedFields,
int batchSize,
Path path,
- long splitStart,
- long splitLength,
+ ParquetMetadata footer,
FilterPredicate filterPredicate,
UnboundRecordFilter recordFilter) throws IOException {
@@ -181,8 +181,7 @@ public class ParquetSplitReaderUtil {
gen,
batchSize,
new org.apache.hadoop.fs.Path(path.toUri()),
- splitStart,
- splitLength,
+ footer,
filterPredicate,
recordFilter);
}
diff --git
a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
index b9f7527a91a8..3454c16e8093 100644
---
a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
+++
b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
@@ -62,8 +62,6 @@ import java.util.stream.IntStream;
import static
org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector;
import static org.apache.parquet.filter2.compat.FilterCompat.get;
import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups;
-import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
-import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
/**
* This reader is used to read a {@link VectorizedColumnBatch} from input
split.
@@ -137,14 +135,12 @@ public class ParquetColumnarRowSplitReader implements
Closeable {
ColumnBatchGenerator generator,
int batchSize,
Path path,
- long splitStart,
- long splitLength,
+ ParquetMetadata footer,
FilterPredicate filterPredicate,
UnboundRecordFilter recordFilter) throws IOException {
this.utcTimestamp = utcTimestamp;
this.batchSize = batchSize;
// then we need to apply the predicate push down filter
- ParquetMetadata footer = readFooter(conf, path, range(splitStart,
splitStart + splitLength));
MessageType fileSchema = footer.getFileMetaData().getSchema();
FilterCompat.Filter filter = get(filterPredicate, recordFilter);
List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(),
fileSchema);
diff --git
a/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
index 3763003e554d..8258b2d4cb07 100644
---
a/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
+++
b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
@@ -35,6 +35,7 @@ import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.junit.jupiter.api.Test;
@@ -51,6 +52,8 @@ import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
+import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
+import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
@@ -352,6 +355,7 @@ public class TestParquetSplitReaderUtil {
}
}
+ ParquetMetadata footer = readFooter(conf, path, range(0, new File(tempDir,
"data.parquet").length()));
List<String> values = new ArrayList<>();
try (ParquetColumnarRowSplitReader reader = new
ParquetColumnarRowSplitReader(
true,
@@ -362,8 +366,7 @@ public class TestParquetSplitReaderUtil {
VectorizedColumnBatch::new,
batchSize,
path,
- 0,
- new File(tempDir, "data.parquet").length(),
+ footer,
null,
null)) {
while (!reader.reachedEnd()) {
diff --git
a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
index d74f4d71e1ca..998f22dfc9be 100644
---
a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
+++
b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
@@ -81,6 +81,7 @@ import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.column.page.PageReader;
import org.apache.parquet.filter.UnboundRecordFilter;
import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.io.ColumnIO;
import org.apache.parquet.io.GroupColumnIO;
import org.apache.parquet.io.MessageColumnIO;
@@ -142,8 +143,7 @@ public class ParquetSplitReaderUtil {
int[] selectedFields,
int batchSize,
Path path,
- long splitStart,
- long splitLength,
+ ParquetMetadata footer,
FilterPredicate filterPredicate,
UnboundRecordFilter recordFilter) throws IOException {
@@ -181,8 +181,7 @@ public class ParquetSplitReaderUtil {
gen,
batchSize,
new org.apache.hadoop.fs.Path(path.toUri()),
- splitStart,
- splitLength,
+ footer,
filterPredicate,
recordFilter);
}
diff --git
a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
index eca0fceae6aa..1436a877195d 100644
---
a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
+++
b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java
@@ -62,8 +62,6 @@ import java.util.stream.IntStream;
import static
org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector;
import static org.apache.parquet.filter2.compat.FilterCompat.get;
import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups;
-import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
-import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
/**
* This reader is used to read a {@link VectorizedColumnBatch} from input
split.
@@ -137,14 +135,12 @@ public class ParquetColumnarRowSplitReader implements
Closeable {
ColumnBatchGenerator generator,
int batchSize,
Path path,
- long splitStart,
- long splitLength,
+ ParquetMetadata footer,
FilterPredicate filterPredicate,
UnboundRecordFilter recordFilter) throws IOException {
this.utcTimestamp = utcTimestamp;
this.batchSize = batchSize;
// then we need to apply the predicate push down filter
- ParquetMetadata footer = readFooter(conf, path, range(splitStart,
splitStart + splitLength));
MessageType fileSchema = footer.getFileMetaData().getSchema();
FilterCompat.Filter filter = get(filterPredicate, recordFilter);
List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(),
fileSchema);
diff --git
a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
index 3763003e554d..8258b2d4cb07 100644
---
a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
+++
b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java
@@ -35,6 +35,7 @@ import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.junit.jupiter.api.Test;
@@ -51,6 +52,8 @@ import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
+import static
org.apache.parquet.format.converter.ParquetMetadataConverter.range;
+import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
@@ -352,6 +355,7 @@ public class TestParquetSplitReaderUtil {
}
}
+ ParquetMetadata footer = readFooter(conf, path, range(0, new File(tempDir,
"data.parquet").length()));
List<String> values = new ArrayList<>();
try (ParquetColumnarRowSplitReader reader = new
ParquetColumnarRowSplitReader(
true,
@@ -362,8 +366,7 @@ public class TestParquetSplitReaderUtil {
VectorizedColumnBatch::new,
batchSize,
path,
- 0,
- new File(tempDir, "data.parquet").length(),
+ footer,
null,
null)) {
while (!reader.reachedEnd()) {