This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5bd2e79b6a2 HIVE-30035: Upgrade to Parquet 1.18.0 (#6774)
5bd2e79b6a2 is described below
commit 5bd2e79b6a2814f6386040c2905bd04993bdd1b5
Author: Bodor Laszlo <[email protected]>
AuthorDate: Fri Sep 18 13:09:39 2026 +0200
HIVE-30035: Upgrade to Parquet 1.18.0 (#6774)
---
iceberg/pom.xml | 10 +++++++++-
pom.xml | 2 +-
.../hive/ql/io/parquet/TestHiveSchemaConverter.java | 17 ++++++++---------
3 files changed, 18 insertions(+), 11 deletions(-)
diff --git a/iceberg/pom.xml b/iceberg/pom.xml
index f337e0d5dcc..7fcb6387e4f 100644
--- a/iceberg/pom.xml
+++ b/iceberg/pom.xml
@@ -33,7 +33,9 @@
<path.to.iceberg.root>.</path.to.iceberg.root>
<kryo-shaded.version>4.0.3</kryo-shaded.version>
<iceberg.mockito-core.version>5.2.0</iceberg.mockito-core.version>
- <iceberg.avro.version>1.12.0</iceberg.avro.version>
+ <iceberg.avro.version>${avro.version}</iceberg.avro.version>
+ <!-- Must match parquet.version: the bundle embeds org.apache.parquet
unrelocated. -->
+ <iceberg.parquet.version>${parquet.version}</iceberg.parquet.version>
<spotless.maven.plugin.version>2.44.4</spotless.maven.plugin.version>
<google.errorprone.version>2.29.2</google.errorprone.version>
<assertj.version>3.27.3</assertj.version>
@@ -94,6 +96,12 @@
<artifactId>iceberg-parquet</artifactId>
<version>${iceberg.version}</version>
</dependency>
+ <!-- iceberg-data pulls parquet-avro, the root of the Parquet subtree
the bundle embeds. -->
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-avro</artifactId>
+ <version>${iceberg.parquet.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-orc</artifactId>
diff --git a/pom.xml b/pom.xml
index 2313c30e5e2..4f7bdf16d03 100644
--- a/pom.xml
+++ b/pom.xml
@@ -203,7 +203,7 @@
<netty.version>4.1.137.Final</netty.version>
<pac4j-saml.version>4.5.8</pac4j-saml.version>
<paranamer.version>2.8</paranamer.version>
- <parquet.version>1.16.0</parquet.version>
+ <parquet.version>1.18.0</parquet.version>
<pig.version>0.16.0</pig.version>
<plexus.version>1.5.6</plexus.version>
<protobuf.version>3.25.5</protobuf.version>
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestHiveSchemaConverter.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestHiveSchemaConverter.java
index 29395c0bc7a..1f2e34b1716 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestHiveSchemaConverter.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestHiveSchemaConverter.java
@@ -334,7 +334,7 @@ public void testMap() throws Exception {
"map<string,string>",
"message hive_schema {\n"
+ " optional group mapCol (MAP) {\n"
- + " repeated group key_value (MAP_KEY_VALUE) {\n"
+ + " repeated group key_value {\n"
+ " required binary key (UTF8);\n"
+ " optional binary value (UTF8);\n"
+ " }\n"
@@ -348,7 +348,7 @@ public void testMapDecimal() throws Exception {
"map<string,decimal(5,2)>",
"message hive_schema {\n"
+ " optional group mapCol (MAP) {\n"
- + " repeated group key_value (MAP_KEY_VALUE) {\n"
+ + " repeated group key_value {\n"
+ " required binary key (UTF8);\n"
+ " optional fixed_len_byte_array(3) value (DECIMAL(5,2));\n"
+ " }\n"
@@ -362,7 +362,7 @@ public void testMapInts() throws Exception {
"map<smallint,tinyint>",
"message hive_schema {\n"
+ " optional group mapCol (MAP) {\n"
- + " repeated group key_value (MAP_KEY_VALUE) {\n"
+ + " repeated group key_value {\n"
+ " required int32 key (INT_16);\n"
+ " optional int32 value (INT_8);\n"
+ " }\n"
@@ -376,7 +376,7 @@ public void testMapStruct() throws Exception {
"map<string,struct<a:smallint,b:int>>",
"message hive_schema {\n"
+ " optional group mapCol (MAP) {\n"
- + " repeated group key_value (MAP_KEY_VALUE) {\n"
+ + " repeated group key_value {\n"
+ " required binary key (UTF8);\n"
+ " optional group value {\n"
+ " optional int32 a (INT_16);\n"
@@ -393,7 +393,7 @@ public void testMapList() throws Exception {
"map<string,array<string>>",
"message hive_schema {\n"
+ " optional group mapCol (MAP) {\n"
- + " repeated group key_value (MAP_KEY_VALUE) {\n"
+ + " repeated group key_value {\n"
+ " required binary key (UTF8);\n"
+ " optional group value (LIST) {\n"
+ " repeated group bag {\n"
@@ -434,10 +434,9 @@ public void testMapOriginalType() throws Exception {
assertEquals(1, topLevel.asGroupType().getFieldCount());
Type secondLevel = topLevel.asGroupType().getFields().get(0);
- // there is one repeated field for mapCol, the field name is "map" and its
original Type is
- // MAP_KEY_VALUE;
- checkField(secondLevel, "key_value", Repetition.REPEATED,
- LogicalTypeAnnotation.MapKeyValueTypeAnnotation.getInstance());
+ // the repeated key_value group carries no annotation: Parquet's MAP
standard treats
+ // MAP_KEY_VALUE as redundant, and ConversionPatterns.mapType no longer
stamps it.
+ checkField(secondLevel, "key_value", Repetition.REPEATED, null);
}
@Test
