This is an automated email from the ASF dual-hosted git repository.
vinothchandar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new afe9e6bcedab fix(common): use GenericData model in parquet-avro
readers to avoid java-class resolution (#20187)
afe9e6bcedab is described below
commit afe9e6bcedab79ce27e156e22d4da2985a4447f5
Author: databird07 <[email protected]>
AuthorDate: Fri Oct 2 08:25:21 2026 -0700
fix(common): use GenericData model in parquet-avro readers to avoid
java-class resolution (#20187)
Dependabot flags org.apache.parquet:parquet-avro < 1.15.1 ("Apache Parquet
Avro Module Vulnerable to Arbitrary Code Execution"). The flagged artifact
comes from ${hive.parquet.version} (1.10.1), and it is shaded into
hudi-hadoop-mr-bundle.
We can't just bump the version. The bundle runs on Hive's own parquet
(1.10.x on Hive 3.1.3), and parquet-avro 1.15.x needs LogicalTypeAnnotation,
ParquetConfiguration and HadoopParquetConfiguration, which that runtime doesn't
have. The Dependabot bump #17654 failed validate-bundles for this reason and
was closed.
The vulnerable path is parquet-avro reading the java-class / java-key-class
schema properties from the file footer and instantiating that class
(AvroRecordConverter.getStringableClass → ClassUtils.forName). It only does
this when the Avro data model is SpecificData/ReflectData. When no model is
set, parquet-avro falls back to SpecificDataSupplier, and that is the default
every Hudi reader was getting.
---
.../org/apache/hudi/common/util/ParquetUtils.java | 8 +-
.../avro/HoodieAvroParquetReaderBuilder.java | 2 +-
.../apache/parquet/avro/HoodieAvroReadSupport.java | 1 +
.../avro/TestAvroStringableClassResolution.java | 108 +++++++++++++++++++++
.../hudi/hadoop/avro/HoodieAvroParquetReader.java | 2 +-
.../apache/hudi/cli/HDFSParquetImporterUtils.java | 2 +
6 files changed, 118 insertions(+), 5 deletions(-)
diff --git
a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
index 31b0e0868fe6..87961f8223cf 100644
---
a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
+++
b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
@@ -47,6 +47,7 @@ import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import lombok.extern.slf4j.Slf4j;
+import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -171,7 +172,7 @@ public class ParquetUtils extends FileFormatUtils {
AvroReadSupport.setRequestedProjection(conf,
HoodieSchemaUtils.getRecordKeySchema().toAvroSchema());
try {
ParquetReaderIterator<GenericRecord> rowIterator = new
ParquetReaderIterator<>(
- AvroParquetReader.<GenericRecord>builder(new
Path(filePath.toUri())).withConf(conf).build());
+ AvroParquetReader.<GenericRecord>builder(new
Path(filePath.toUri())).withDataModel(GenericData.get()).withConf(conf).build());
return new ClosableIterator<String>() {
private long rowPosition = 0;
@@ -241,7 +242,7 @@ public class ParquetUtils extends FileFormatUtils {
AvroReadSupport.setRequestedProjection(conf, readSchema.toAvroSchema());
Set<Pair<String, Long>> rowKeys = new HashSet<>();
long rowPosition = 0;
- try (ParquetReader reader =
AvroParquetReader.builder(filePath).withConf(conf).build()) {
+ try (ParquetReader reader =
AvroParquetReader.builder(filePath).withDataModel(GenericData.get()).withConf(conf).build())
{
Object obj = reader.read();
while (obj != null) {
if (obj instanceof GenericRecord) {
@@ -291,7 +292,7 @@ public class ParquetUtils extends FileFormatUtils {
AvroReadSupport.setAvroReadSchema(conf, readSchema.toAvroSchema());
AvroReadSupport.setRequestedProjection(conf, readSchema.toAvroSchema());
ParquetReader<GenericRecord> reader =
- AvroParquetReader.<GenericRecord>builder(new
Path(filePath.toUri())).withConf(conf).build();
+ AvroParquetReader.<GenericRecord>builder(new
Path(filePath.toUri())).withDataModel(GenericData.get()).withConf(conf).build();
return HoodieKeyIterator.getInstance(new
ParquetReaderIterator<>(reader), keyGeneratorOpt, partitionPath);
} catch (IOException e) {
throw new HoodieIOException("Failed to read from Parquet file " +
filePath, e);
@@ -431,6 +432,7 @@ public class ParquetUtils extends FileFormatUtils {
public List<GenericRecord> readAvroRecords(HoodieStorage storage,
StoragePath filePath) {
List<GenericRecord> records = new ArrayList<>();
try (ParquetReader reader = AvroParquetReader.builder(new
Path(filePath.toUri()))
+ .withDataModel(GenericData.get())
.withConf(storage.getConf().unwrapAs(Configuration.class)).build()) {
Object obj = reader.read();
while (obj != null) {
diff --git
a/hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroParquetReaderBuilder.java
b/hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroParquetReaderBuilder.java
index 9fe32b456ef3..77f9a25a7cd6 100644
---
a/hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroParquetReaderBuilder.java
+++
b/hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroParquetReaderBuilder.java
@@ -36,7 +36,7 @@ import org.apache.parquet.schema.MessageType;
*/
public class HoodieAvroParquetReaderBuilder<T> extends
ParquetReader.Builder<T> {
- private GenericData model = null;
+ private GenericData model = GenericData.get();
private boolean enableCompatibility = true;
private boolean isReflect = true;
private Option<MessageType> tableSchema = Option.empty();
diff --git
a/hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroReadSupport.java
b/hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroReadSupport.java
index 9292baf65dbf..a895db255639 100644
---
a/hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroReadSupport.java
+++
b/hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroReadSupport.java
@@ -50,6 +50,7 @@ public class HoodieAvroReadSupport<T> extends
AvroReadSupport<T> {
}
public HoodieAvroReadSupport() {
+ super(GenericData.get());
tableSchema = Option.empty();
}
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/parquet/avro/TestAvroStringableClassResolution.java
b/hudi-hadoop-common/src/test/java/org/apache/parquet/avro/TestAvroStringableClassResolution.java
new file mode 100644
index 000000000000..5501b4f91f11
--- /dev/null
+++
b/hudi-hadoop-common/src/test/java/org/apache/parquet/avro/TestAvroStringableClassResolution.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.parquet.avro;
+
+import org.apache.hudi.common.util.ParquetUtils;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
+
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+/**
+ * Verifies that Hudi's parquet-avro readers never instantiate classes named
by the
+ * {@code java-class} property of an Avro schema stored in a parquet file
footer.
+ */
+class TestAvroStringableClassResolution {
+
+ @TempDir
+ java.nio.file.Path tempDir;
+
+ private Path filePath;
+
+ public static class StringableProbe {
+ static final AtomicInteger INSTANCES = new AtomicInteger();
+
+ public StringableProbe(String value) {
+ INSTANCES.incrementAndGet();
+ }
+ }
+
+ @BeforeEach
+ void writeFile() throws IOException {
+ Schema stringSchema = Schema.create(Schema.Type.STRING);
+ stringSchema.addProp("java-class", StringableProbe.class.getName());
+ Schema schema =
SchemaBuilder.record("rec").fields().name("f").type(stringSchema).noDefault().endRecord();
+
+ filePath = new Path(tempDir.resolve("stringable.parquet").toUri());
+ try (ParquetWriter<GenericRecord> writer =
AvroParquetWriter.<GenericRecord>builder(filePath)
+ .withSchema(schema)
+ .withDataModel(GenericData.get())
+ .withConf(new Configuration())
+ .build()) {
+ GenericRecord record = new GenericData.Record(schema);
+ record.put("f", "value");
+ writer.write(record);
+ }
+ StringableProbe.INSTANCES.set(0);
+ }
+
+ @Test
+ void hoodieAvroParquetReaderBuilderIgnoresJavaClass() throws IOException {
+ Configuration conf = new Configuration();
+ try (ParquetReader<IndexedRecord> reader =
+ new
HoodieAvroParquetReaderBuilder<IndexedRecord>(HadoopInputFile.fromPath(filePath,
conf))
+ .withConf(conf)
+ .build()) {
+ IndexedRecord record = reader.read();
+ assertEquals("value", record.get(0).toString());
+ assertFalse(record.get(0) instanceof StringableProbe);
+ }
+ assertEquals(0, StringableProbe.INSTANCES.get());
+ }
+
+ @Test
+ void parquetUtilsReadAvroRecordsIgnoresJavaClass() {
+ Configuration conf = new Configuration();
+ conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, false);
+ StoragePath storagePath = new StoragePath(filePath.toUri());
+ List<GenericRecord> records = new ParquetUtils().readAvroRecords(new
HoodieHadoopStorage(filePath, conf), storagePath);
+ assertEquals(1, records.size());
+ assertEquals("value", records.get(0).get("f").toString());
+ assertEquals(0, StringableProbe.INSTANCES.get());
+ }
+}
diff --git
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/avro/HoodieAvroParquetReader.java
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/avro/HoodieAvroParquetReader.java
index e30e1e5c7335..a4bde548268b 100644
---
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/avro/HoodieAvroParquetReader.java
+++
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/avro/HoodieAvroParquetReader.java
@@ -82,7 +82,7 @@ public class HoodieAvroParquetReader extends
RecordReader<Void, ArrayWritable> {
AvroReadSupport.setRequestedProjection(conf,
filterSchema.toAvroSchema());
}
}
- parquetRecordReader = new ParquetRecordReader<>(new AvroReadSupport<>(),
getFilter(conf));
+ parquetRecordReader = new ParquetRecordReader<>(new
AvroReadSupport<>(GenericData.get()), getFilter(conf));
}
@Override
diff --git
a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/HDFSParquetImporterUtils.java
b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/HDFSParquetImporterUtils.java
index 7fc06fc4e380..2b4b5ac55069 100644
---
a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/HDFSParquetImporterUtils.java
+++
b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/HDFSParquetImporterUtils.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.parquet.avro.AvroReadSupport;
+import org.apache.parquet.avro.GenericDataSupplier;
import org.apache.parquet.hadoop.ParquetInputFormat;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
@@ -184,6 +185,7 @@ public class HDFSParquetImporterUtils implements
Serializable {
job.getConfiguration().set(FileInputFormat.LIST_STATUS_NUM_THREADS,
"1024");
AvroReadSupport.setAvroReadSchema(jsc.hadoopConfiguration(),
HoodieSchema.parse(schemaStr).getAvroSchema());
ParquetInputFormat.setReadSupportClass(job, (AvroReadSupport.class));
+ AvroReadSupport.setAvroDataSupplier(job.getConfiguration(),
GenericDataSupplier.class);
HoodieEngineContext context = new HoodieSparkEngineContext(jsc);
context.setJobStatus(this.getClass().getSimpleName(), "Build records for
import: " + this.tableName);
