This is an automated email from the ASF dual-hosted git repository.
Abacn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9a1a4b170e4 [Iceberg] pin hadoop to 3.3.6 (#38500)
9a1a4b170e4 is described below
commit 9a1a4b170e4aebbb007db778bc19d7e2f5516b51
Author: Ahmed Abualsaud <[email protected]>
AuthorDate: Thu May 14 15:13:28 2026 -0400
[Iceberg] pin hadoop to 3.3.6 (#38500)
* pin hadoop to 3.3.6
* pin in expansion service too
---
.github/trigger_files/IO_Iceberg_Integration_Tests.json | 2 +-
.../trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json | 2 +-
sdks/java/io/expansion-service/build.gradle | 10 ++++++++++
sdks/java/io/iceberg/build.gradle | 11 ++++++++++-
.../org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java | 2 +-
5 files changed, 23 insertions(+), 4 deletions(-)
diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests.json
b/.github/trigger_files/IO_Iceberg_Integration_Tests.json
index b73af5e61a4..7ab7bcd9a9c 100644
--- a/.github/trigger_files/IO_Iceberg_Integration_Tests.json
+++ b/.github/trigger_files/IO_Iceberg_Integration_Tests.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run.",
- "modification": 1
+ "modification": 2
}
diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json
b/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json
index 5abe02fc09c..3a009261f4f 100644
--- a/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json
+++ b/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run.",
- "modification": 1
+ "modification": 2
}
diff --git a/sdks/java/io/expansion-service/build.gradle
b/sdks/java/io/expansion-service/build.gradle
index 77a062c5b9a..4683b4fd6f5 100644
--- a/sdks/java/io/expansion-service/build.gradle
+++ b/sdks/java/io/expansion-service/build.gradle
@@ -57,6 +57,16 @@ configurations.runtimeClasspath {
// Pin nimbus-jose-jwt to 9.37.4 to fix CVE-2025-53864 (transitive via
hadoop-auth)
resolutionStrategy.force 'com.nimbusds:nimbus-jose-jwt:9.37.4'
+
+ // [iceberg]
+ // bigdataoss:gcs-connector and parquet:parquet-hadoop have conflicts with
global hadoop-common:3.4.2
+ // upgrading gcs-connector to 4.0.0 would be fine, because it uses
hadoop-common 3.4.2
+ // but parquet-hadoop is still at 3.3.0
+ // so for now we need to pin hadoop to 3.3.6 until parquet-hadoop releases a
version that uses hadoop 3.4.2+
+ resolutionStrategy.force 'org.apache.hadoop:hadoop-common:3.3.6'
+ resolutionStrategy.force 'org.apache.hadoop:hadoop-client:3.3.6'
+ resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs:3.3.6'
+ resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs-client:3.3.6'
}
shadowJar {
diff --git a/sdks/java/io/iceberg/build.gradle
b/sdks/java/io/iceberg/build.gradle
index bbd55fee2fc..2dae0335f9c 100644
--- a/sdks/java/io/iceberg/build.gradle
+++ b/sdks/java/io/iceberg/build.gradle
@@ -59,7 +59,7 @@ dependencies {
implementation "org.apache.iceberg:iceberg-parquet:$iceberg_version"
implementation "org.apache.iceberg:iceberg-orc:$iceberg_version"
implementation "org.apache.iceberg:iceberg-data:$iceberg_version"
- implementation library.java.hadoop_common
+ implementation "org.apache.hadoop:hadoop-common:3.3.6"
// TODO(https://github.com/apache/beam/issues/21156): Determine how to
build without this dependency
provided "org.immutables:value:2.8.8"
permitUnusedDeclared "org.immutables:value:2.8.8"
@@ -70,6 +70,7 @@ dependencies {
runtimeOnly "org.apache.iceberg:iceberg-azure:$iceberg_version"
runtimeOnly "org.apache.iceberg:iceberg-azure-bundle:$iceberg_version"
runtimeOnly library.java.bigdataoss_gcs_connector
+ runtimeOnly library.java.bigdataoss_util_hadoop
runtimeOnly library.java.hadoop_client
testImplementation project(":sdks:java:managed")
@@ -117,6 +118,14 @@ dependencies {
configurations.all {
// iceberg-core needs avro:1.12.0
resolutionStrategy.force 'org.apache.avro:avro:1.12.0'
+ // bigdataoss:gcs-connector and parquet:parquet-hadoop have conflicts with
global hadoop-common:3.4.2
+ // upgrading gcs-connector to 4.0.0 would be fine, because it uses
hadoop-common 3.4.2
+ // but parquet-hadoop is still at 3.3.0
+ // so for now we need to pin hadoop to 3.3.6 until parquet-hadoop releases a
version that uses hadoop 3.4.2+
+ resolutionStrategy.force 'org.apache.hadoop:hadoop-common:3.3.6'
+ resolutionStrategy.force 'org.apache.hadoop:hadoop-client:3.3.6'
+ resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs:3.3.6'
+ resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs-client:3.3.6'
}
hadoopVersions.each {kv ->
diff --git
a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java
b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java
index 56f21d2bc8b..45ecc7cf71c 100644
---
a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java
+++
b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java
@@ -19,7 +19,7 @@ package org.apache.beam.sdk.io.iceberg;
import static
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument;
import static
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.hadoop.util.Sets.newHashSet;
+import static
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets.newHashSet;
import com.google.auto.value.AutoValue;
import java.io.Serializable;
