This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new b9865b2d4a [VL] Bump Iceberg/Hudi for Spark-4.1 package (#12168)
b9865b2d4a is described below
commit b9865b2d4a531f56b046ef1c0fd823b981a8a369
Author: Yuan <[email protected]>
AuthorDate: Fri May 29 14:53:21 2026 +0800
[VL] Bump Iceberg/Hudi for Spark-4.1 package (#12168)
This patch bumps the Iceberg/Hudi for Spark-4.1
https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-spark-runtime-4.1
https://mvnrepository.com/artifact/org.apache.hudi/hudi-spark4.1-bundle
This patch also bumps the rsync version to meet apache requirements
---------
Signed-off-by: Yuan <[email protected]>
---
.github/workflows/velox_nightly.yml | 17 +++++++----------
gluten-hudi/pom.xml | 6 ++++++
pom.xml | 4 ++--
3 files changed, 15 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/velox_nightly.yml
b/.github/workflows/velox_nightly.yml
index 0327fe09b9..5dd0afeb3c 100644
--- a/.github/workflows/velox_nightly.yml
+++ b/.github/workflows/velox_nightly.yml
@@ -171,8 +171,7 @@ jobs:
run: |
cd $GITHUB_WORKSPACE/ && \
./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
- # iceberg/hudi support for spark-4.1 is not avaiable, skip it first
- ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Pdelta -DskipTests -Dmaven.source.skip
+ ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -DskipTests
-Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v4
with:
@@ -288,8 +287,7 @@ jobs:
./build/mvn clean install -Pspark-3.4 -Pjava-17 -Pbackends-velox
-Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests
-Dmaven.source.skip
./build/mvn clean install -Pspark-3.5 -Pjava-17 -Pbackends-velox
-Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon -DskipTests
-Dmaven.source.skip
./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
- # iceberg/hudi support for spark-4.1 is not avaiable, skip it first
- ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -Pdelta -DskipTests -Dmaven.source.skip
+ ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-17
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -DskipTests
-Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v4
with:
@@ -324,8 +322,7 @@ jobs:
run: |
cd $GITHUB_WORKSPACE/ && \
./build/mvn clean install -Pspark-4.0 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -Ppaimon
-DskipTests -Dmaven.source.skip
- # iceberg/hudi support for spark-4.1 is not avaiable, skip it first
- ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Pdelta -DskipTests -Dmaven.source.skip
+ ./build/mvn clean install -Pspark-4.1 -Pscala-2.13 -Pjava-21
-Pbackends-velox -Pceleborn -Puniffle -Piceberg -Phudi -Pdelta -DskipTests
-Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v4
with:
@@ -353,7 +350,7 @@ jobs:
name: nightly-gluten-velox-bundle-package-jdk8-x86-${{
steps.date.outputs.date }}
path: package/
- name: rsync to apache nightly
- uses:
burnett01/rsync-deployments@7659d600d8bdd035bb5cdfba1d4bd0dd4a307ca6 # 8.0.3
+ uses:
burnett01/rsync-deployments@66257cad6bfeb2171d3b6bfa6c9a22279dd9c3a1 # 8.0.5
with:
switches: -avzr --delete
path: package/gluten-velox-bundle-*.jar
@@ -382,7 +379,7 @@ jobs:
name: nightly-gluten-velox-bundle-package-jdk17-x86-${{
steps.date.outputs.date }}
path: package/
- name: rsync to apache nightly
- uses:
burnett01/rsync-deployments@7659d600d8bdd035bb5cdfba1d4bd0dd4a307ca6 # 8.0.3
+ uses:
burnett01/rsync-deployments@66257cad6bfeb2171d3b6bfa6c9a22279dd9c3a1 # 8.0.5
with:
switches: -avzr --delete
path: package/gluten-velox-bundle-*.jar
@@ -411,7 +408,7 @@ jobs:
name: nightly-gluten-velox-bundle-package-jdk21-x86-${{
steps.date.outputs.date }}
path: package/
- name: rsync to apache nightly
- uses:
burnett01/rsync-deployments@7659d600d8bdd035bb5cdfba1d4bd0dd4a307ca6 # 8.0.3
+ uses:
burnett01/rsync-deployments@66257cad6bfeb2171d3b6bfa6c9a22279dd9c3a1 # 8.0.5
with:
switches: -avzr --delete
path: package/gluten-velox-bundle-*.jar
@@ -498,7 +495,7 @@ jobs:
path: package/target/gluten-velox-bundle-*.jar
retention-days: 7
- name: rsync to apache nightly
- uses:
burnett01/rsync-deployments@7659d600d8bdd035bb5cdfba1d4bd0dd4a307ca6 # 8.0.3
+ uses:
burnett01/rsync-deployments@66257cad6bfeb2171d3b6bfa6c9a22279dd9c3a1 # 8.0.5
with:
switches: -avzr
path: package/target/gluten-velox-bundle-*.jar
diff --git a/gluten-hudi/pom.xml b/gluten-hudi/pom.xml
index d31f9075c4..a7d27895e9 100755
--- a/gluten-hudi/pom.xml
+++ b/gluten-hudi/pom.xml
@@ -56,6 +56,12 @@
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
<!-- For test -->
<dependency>
diff --git a/pom.xml b/pom.xml
index 6ab7bcddc2..76e99de20d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1350,11 +1350,11 @@
<sparkbundle.version>4.1</sparkbundle.version>
<sparkshim.artifactId>spark-sql-columnar-shims-spark41</sparkshim.artifactId>
<spark.version>4.1.1</spark.version>
- <iceberg.version>1.10.0</iceberg.version>
+ <iceberg.version>1.11.0</iceberg.version>
<delta.package.name>delta-spark_4.1</delta.package.name>
<delta.version>4.1.0</delta.version>
<delta.binary.version>40</delta.binary.version>
- <hudi.version>1.1.0</hudi.version>
+ <hudi.version>1.2.0</hudi.version>
<fasterxml.version>2.18.2</fasterxml.version>
<fasterxml.jackson.version>2.18.2</fasterxml.jackson.version>
<fasterxml.jackson.databind.version>2.18.2</fasterxml.jackson.databind.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
